Merge branch 'develop' of github.com:taosdata/TDengine into develop

Jenkinsfile

@@ -27,6 +27,7 @@ pipeline {
             cd debug
             cmake .. > /dev/null
             make > /dev/null
+            make install > /dev/null
             cd ${WKC}/tests
             #./test-all.sh smoke
             ./test-all.sh pytest

packaging/deb/makedeb.sh

@@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/build/bin/taosdemo                ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/build/bin/taosdump                ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taos                    ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/lib/${libfile}              ${pkg_dir}${install_home_path}/driver 

packaging/rpm/tdengine.spec

@@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh       %{buildroot}%{homepath}/scri
 cp %{_compiledir}/build/bin/taos                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosdemo                %{buildroot}%{homepath}/bin
+cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 cp %{_compiledir}/../src/inc/taos.h                 %{buildroot}%{homepath}/include
 cp %{_compiledir}/../src/inc/taoserror.h            %{buildroot}%{homepath}/include
@@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then
     ${csudo} rm -f ${bin_link_dir}/taos       || :
     ${csudo} rm -f ${bin_link_dir}/taosd      || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo   || :
+    ${csudo} rm -f ${bin_link_dir}/taosdump   || :
     ${csudo} rm -f ${cfg_link_dir}/*          || :
     ${csudo} rm -f ${inc_link_dir}/taos.h     || :
     ${csudo} rm -f ${inc_link_dir}/taoserror.h     || :

packaging/tools/makeclient.sh

@@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then
     strip ${build_dir}/bin/taos
     bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
   else  
-    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
+    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
 else

packaging/tools/makepkg.sh

@@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/taos
   bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
 else 
-  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
+  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
 fi
 
 lib_files="${build_dir}/lib/libtaos.so.${version}"

src/client/src/tscAsync.c

@@ -428,6 +428,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       } else {
         assert(code == TSDB_CODE_SUCCESS);      
       }
+
       // param already freed by other routine and pSql in tscCache when ctrl + c 
       if (atomic_load_ptr(&pSql->param) == NULL) {
         return;
@@ -441,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
           pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
 
+      // tscProcessSql can add error into async res
+      tscProcessSql(pSql);
+      return;
+    } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
+      tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
+      STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+      code = tscGetTableMeta(pSql, pTableMetaInfo);
+      if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+        return;
+      } else {
+        assert(code == TSDB_CODE_SUCCESS);
+      }
+
+      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
       // tscProcessSql can add error into async res
       tscProcessSql(pSql);
       return;
@@ -465,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
           tscResetSqlCmdObj(pCmd, false);
 
           code = tsParseSql(pSql, true);
-
           if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
             return;
           } else if (code != TSDB_CODE_SUCCESS) {

src/client/src/tscLocalMerge.c

@@ -97,6 +97,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
       pCtx->param[2].i64Key = pQueryInfo->order.order;
       pCtx->param[2].nType  = TSDB_DATA_TYPE_BIGINT;
       pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
+    } else if (functionId == TSDB_FUNC_APERCT) {
+      pCtx->param[0].i64Key = pExpr->param[0].i64Key;
+      pCtx->param[0].nType  = pExpr->param[0].nType;
     }
 
     pCtx->interBufBytes = pExpr->interBytes;

src/client/src/tscSQLParser.c

@@ -5311,6 +5311,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
   pMsg->replications = pCreateDb->replica;
   pMsg->quorum = pCreateDb->quorum;
   pMsg->ignoreExist = pCreateDb->ignoreExists;
+  pMsg->update = pCreateDb->update;
 }
 
 int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {

src/client/src/tscSubquery.c

@@ -523,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
   assert(pSqlObj->subState.numOfRemain > 0);
 
   if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
-    tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
+    tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
     freeJoinSubqueryObj(pSqlObj);
   }
 }
@@ -565,7 +565,7 @@ int32_t tagValCompar(const void* p1, const void* p2) {
     return (tag1->len > tag2->len)? 1: -1;
   }
 
-  return strncmp(tag1->data, tag2->data, tag1->len);
+  return memcmp(tag1->data, tag2->data, tag1->len);
 }
 
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@@ -853,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
   }
 
   if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) {  // no results,return.
+    assert(pParentSql->fp != tscJoinQueryCallback);
+
     tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
     freeJoinSubqueryObj(pParentSql);
 
     // set no result command
     pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
+    assert(pParentSql->fp != tscJoinQueryCallback);
+
     (*pParentSql->fp)(pParentSql->param, pParentSql, 0);
   } else {
     // proceed to for ts_comp query

src/common/inc/tdataformat.h

@@ -119,6 +119,33 @@ void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
 int       tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
+// ----------------- Semantic timestamp key definition
+typedef uint64_t TKEY;
+
+#define TKEY_INVALID UINT64_MAX
+#define TKEY_NULL TKEY_INVALID
+#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
+#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
+#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
+
+#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
+#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
+#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
+#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
+#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
+
+static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
+  TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
+  TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
+
+  if (key1 < key2) {
+    return -1;
+  } else if (key1 > key2) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
@@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
  * +----------+----------+---------------------------------+---------------------------------+
  * |   len    | sversion |           First part            |             Second part         |
  * +----------+----------+---------------------------------+---------------------------------+
+ * 
+ * NOTE: timestamp in this row structure is TKEY instead of TSKEY
  */
 typedef void *SDataRow;
 
@@ -137,11 +166,13 @@ typedef void *SDataRow;
 #define dataRowLen(r) (*(uint16_t *)(r))
 #define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
 #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
+#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
+#define dataRowKey(r) tdGetKey(dataRowTKey(r))
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
 #define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
 #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
+#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
 
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
@@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
   int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
   char *  ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
 
-  switch (type) {
+  if (IS_VAR_DATA_TYPE(type)) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
     *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
     memcpy(ptr, value, varDataTLen(value));
     dataRowLen(row) += varDataTLen(value);
-      break;
+  } else {
-    default:
+    if (offset == 0) {
+      ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
+      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
+      memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
+    } else {
       memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
-      break;
+    }
   }
 
   return 0;
@@ -171,11 +204,9 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
 
 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
-  switch (type) {
+  if (IS_VAR_DATA_TYPE(type)) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
     return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
-    default:
+  } else {
     return POINTER_SHIFT(row, offset);
   }
 }
@@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
 void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
@@ -204,27 +234,19 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
 
 // Get the data pointer from a column-wised data
 static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
-  switch (pCol->type) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
     return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
-      break;
+  } else {
-
-    default:
     return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
-      break;
   }
 }
 
 static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
   ASSERT(rows > 0);
 
-  switch (pDataCol->type) {
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
     return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
-      break;
+  } else {
-    default:
     return TYPE_BYTES[pDataCol->type] * rows;
   }
 }
@@ -243,9 +265,14 @@ typedef struct {
 } SDataCols;
 
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
-#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
-#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
+#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
-#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
+#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
+#define dataColsTKeyLast(pCols) \
+  (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
+#define dataColsKeyLast(pCols) \
+  (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
 
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
@@ -253,10 +280,7 @@ int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 void       tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
-void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);  //!!!!
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
-void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
-                              int limit2, int tRows);
 
 // ----------------- K-V data row structure
 /*

src/common/inc/tglobal.h

@@ -88,6 +88,7 @@ extern int16_t tsWAL;
 extern int32_t tsFsyncPeriod;
 extern int32_t tsReplications;
 extern int32_t tsQuorum;
+extern int32_t tsUpdate;
 
 // balance
 extern int32_t tsEnableBalance;

src/common/src/tdataformat.c

@@ -18,6 +18,9 @@
 #include "tcoding.h"
 #include "wchar.h"
 
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows);
+
 /**
  * Duplicate the schema and return a new object
  */
@@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
   pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
 
   pDataCol->len = 0;
-  if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
     pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
     pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
     pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
   }
 }
 
+// value from timestamp should be TKEY here instead of TSKEY
 void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
 
-  switch (pCol->type) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
     // set offset
     pCol->dataOff[numOfRows] = pCol->len;
     // Copy data
     memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
     // Update the length
     pCol->len += varDataTLen(value);
-      break;
+  } else {
-    default:
     ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
     memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
     pCol->len += pCol->bytes;
-      break;
-  }
-}
-
-void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
-  int pointsLeft = numOfRows - pointsToPop;
-
-  ASSERT(pointsLeft > 0);
-
-  if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
-    ASSERT(pCol->len > 0);
-    VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
-    pCol->len = pCol->len - toffset;
-    ASSERT(pCol->len > 0);
-    memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
-    dataColSetOffset(pCol, pointsLeft);
-  } else {
-    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
-    pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
-    memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
   }
 }
 
 bool isNEleNull(SDataCol *pCol, int nEle) {
-  switch (pCol->type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
   for (int i = 0; i < nEle; i++) {
     if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
   }
   return true;
-    default:
-      for (int i = 0; i < nEle; i++) {
-        if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
-      }
-      return true;
-  }
 }
 
 void dataColSetNullAt(SDataCol *pCol, int index) {
@@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
     pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
     pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
 
-    if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+    if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
       ASSERT(pDataCols->cols[i].dataOff != NULL);
       pRet->cols[i].dataOff =
           (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
       pRet->cols[i].len = pDataCols->cols[i].len;
       if (pDataCols->cols[i].len > 0) {
         memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
-        if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+        if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
           memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
         }
       }
@@ -420,11 +392,21 @@ void tdResetDataCols(SDataCols *pCols) {
 }
 
 void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
-  ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
+  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
 
   int rcol = 0;
   int dcol = 0;
 
+  if (dataRowDeleted(row)) {
+    for (; dcol < pCols->numOfCols; dcol++) {
+      SDataCol *pDataCol = &(pCols->cols[dcol]);
+      if (dcol == 0) {
+        dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
+      } else {
+        dataColSetNullAt(pDataCol, pCols->numOfRows);
+      }
+    }
+  } else {
     while (dcol < pCols->numOfCols) {
       SDataCol *pDataCol = &(pCols->cols[dcol]);
       if (rcol >= schemaNCols(pSchema)) {
@@ -435,7 +417,7 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
 
       STColumn *pRowCol = schemaColAt(pSchema, rcol);
       if (pRowCol->colId == pDataCol->colId) {
-      void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
+        void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
         dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
         dcol++;
         rcol++;
@@ -446,32 +428,18 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
         dcol++;
       }
     }
+  }
   pCols->numOfRows++;
 }
 
-// Pop pointsToPop points from the SDataCols
-void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
-  int pointsLeft = pCols->numOfRows - pointsToPop;
-  if (pointsLeft <= 0) {
-    tdResetDataCols(pCols);
-    return;
-  }
-
-  for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
-    SDataCol *pCol = pCols->cols + iCol;
-    dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
-  }
-  pCols->numOfRows = pointsLeft;
-}
-
 int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
-  ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
   ASSERT(target->numOfCols == source->numOfCols);
 
   SDataCols *pTarget = NULL;
 
   if (dataColsKeyLast(target) < dataColsKeyFirst(source)) {  // No overlap
+    ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
     for (int i = 0; i < rowsToMerge; i++) {
       for (int j = 0; j < source->numOfCols; j++) {
         if (source->cols[j].len > 0) {
@@ -499,17 +467,23 @@ _err:
   return -1;
 }
 
-void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
+// src2 data has more priority than src1
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows) {
   tdResetDataCols(target);
   ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
 
   while (target->numOfRows < tRows) {
     if (*iter1 >= limit1 && *iter2 >= limit2) break;
 
-    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
+    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
-    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
+    TKEY  tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
+    TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
 
-    if (key1 <= key2) {
+    ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
+
+    if (key1 < key2) {
       for (int i = 0; i < src1->numOfCols; i++) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
         if (src1->cols[i].len > 0) {
@@ -520,8 +494,8 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
 
       target->numOfRows++;
       (*iter1)++;
-      if (key1 == key2) (*iter2)++;
+    } else if (key1 >= key2) {
-    } else {
+      if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
         for (int i = 0; i < src2->numOfCols; i++) {
           ASSERT(target->cols[i].type == src2->cols[i].type);
           if (src2->cols[i].len > 0) {
@@ -529,10 +503,14 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
                              target->maxPoints);
           }
         }
-
         target->numOfRows++;
-      (*iter2)++;
       }
+
+      (*iter2)++;
+      if (key1 == key2) (*iter1)++;
+    }
+
+    ASSERT(target->numOfRows <= target->maxPoints);
   }
 }
 

src/common/src/tglobal.c

@@ -120,6 +120,7 @@ int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
 int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
+int32_t tsUpdate        = TSDB_DEFAULT_DB_UPDATE_OPTION;
 int32_t tsMaxVgroupsPerDb  = 0;
 int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@@ -786,6 +787,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "update";
+  cfg.ptr = &tsUpdate;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_DB_UPDATE;
+  cfg.maxValue = TSDB_MAX_DB_UPDATE;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "mqttHostName";
   cfg.ptr = tsMqttHostName;
   cfg.valType = TAOS_CFG_VTYPE_STRING;

src/dnode/src/dnodeVRead.c

@@ -92,33 +92,23 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
     pHead->vgId = htonl(pHead->vgId);
     pHead->contLen = htonl(pHead->contLen);
 
-    taos_queue queue = vnodeAcquireRqueue(pHead->vgId);
+    void *pVnode = vnodeAcquire(pHead->vgId);
-
+    if (pVnode != NULL) {
-    if (queue == NULL) {
+      int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
-      leftLen -= pHead->contLen;
+      if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
-      pCont -= pHead->contLen;
+      vnodeRelease(pVnode);
-      continue;
     }
 
-    // put message into queue
-    SVReadMsg *pRead = taosAllocateQitem(sizeof(SVReadMsg));
-    pRead->rpcMsg = *pMsg;
-    pRead->pCont = pCont;
-    pRead->contLen = pHead->contLen;
-
-    // next vnode
     leftLen -= pHead->contLen;
     pCont -= pHead->contLen;
-    queuedMsgNum++;
-
-    taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
   }
 
   if (queuedMsgNum == 0) {
     SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
     rpcSendResponse(&rpcRsp);
-    rpcFreeCont(pMsg->pCont);
   }
+
+  rpcFreeCont(pMsg->pCont);
 }
 
 void *dnodeAllocVReadQueue(void *pVnode) {
@@ -162,50 +152,48 @@ void dnodeFreeVReadQueue(void *rqueue) {
 
 void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
   SRpcMsg rpcRsp = {
-    .handle  = pRead->rpcMsg.handle,
+    .handle  = pRead->rpcHandle,
     .pCont   = pRead->rspRet.rsp,
     .contLen = pRead->rspRet.len,
     .code    = code,
   };
 
   rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pRead->rpcMsg.pCont);
   vnodeRelease(pVnode);
 }
 
 void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) {
-  rpcFreeCont(pRead->rpcMsg.pCont);
   vnodeRelease(pVnode);
 }
 
 static void *dnodeProcessReadQueue(void *param) {
-  SVReadMsg *pReadMsg;
+  SVReadMsg *pRead;
   int32_t    qtype;
   void *     pVnode;
 
   while (1) {
-    if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pReadMsg, &pVnode) == 0) {
+    if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) {
       dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset);
       break;
     }
 
-    dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle,
+    dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead,
-           taosMsg[pReadMsg->rpcMsg.msgType], qtype, pReadMsg);
+           taosMsg[pRead->msgType], qtype);
 
-    int32_t code = vnodeProcessRead(pVnode, pReadMsg);
+    int32_t code = vnodeProcessRead(pVnode, pRead);
 
     if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
-      dnodeSendRpcVReadRsp(pVnode, pReadMsg, code);
+      dnodeSendRpcVReadRsp(pVnode, pRead, code);
     } else {
       if (code == TSDB_CODE_QRY_HAS_RSP) {
-        dnodeSendRpcVReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code);
+        dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code);
       } else {  // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
-        assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5));
+        assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
-        dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+        dnodeDispatchNonRspMsg(pVnode, pRead, code);
       }
     }
 
-    taosFreeQitem(pReadMsg);
+    taosFreeQitem(pRead);
   }
 
   return NULL;

src/inc/taosdef.h

@@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_DATA_SMALLINT_NULL         0x8000
 #define TSDB_DATA_INT_NULL              0x80000000L
 #define TSDB_DATA_BIGINT_NULL           0x8000000000000000L
+#define TSDB_DATA_TIMESTAMP_NULL        TSDB_DATA_BIGINT_NULL
 
 #define TSDB_DATA_FLOAT_NULL            0x7FF00000              // it is an NAN
 #define TSDB_DATA_DOUBLE_NULL           0x7FFFFF0000000000L     // an NAN
@@ -363,6 +364,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_MAX_WAL_LEVEL              2
 #define TSDB_DEFAULT_WAL_LEVEL          1
 
+#define TSDB_MIN_DB_UPDATE              0
+#define TSDB_MAX_DB_UPDATE              1
+#define TSDB_DEFAULT_DB_UPDATE_OPTION   0
+
 #define TSDB_MIN_FSYNC_PERIOD           0
 #define TSDB_MAX_FSYNC_PERIOD           180000   // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD       3000     // three second

src/inc/taosmsg.h

@@ -544,6 +544,8 @@ typedef struct {
   int8_t   replications;
   int8_t   quorum;
   int8_t   ignoreExist;
+  int8_t   update;
+  int8_t   reserve[9];
 } SCreateDbMsg, SAlterDbMsg;
 
 typedef struct {
@@ -654,7 +656,8 @@ typedef struct {
   int8_t   replications;
   int8_t   wals;
   int8_t   quorum;
-  int8_t   reserved[16];
+  int8_t   update;
+  int8_t   reserved[15];
 } SVnodeCfg;
 
 typedef struct {

src/inc/tsdb.h

@@ -65,6 +65,7 @@ typedef struct {
   int32_t maxRowsPerFileBlock;  // maximum rows per file block
   int8_t  precision;
   int8_t  compression;
+  int8_t  update;
 } STsdbCfg;
 
 // --------- TSDB REPOSITORY USAGE STATISTICS

src/inc/ttokendef.h

@@ -114,114 +114,115 @@
 #define TK_FSYNC                           95
 #define TK_COMP                            96
 #define TK_PRECISION                       97
-#define TK_LP                              98
+#define TK_UPDATE                          98
-#define TK_RP                              99
+#define TK_LP                              99
-#define TK_TAGS                           100
+#define TK_RP                             100
-#define TK_USING                          101
+#define TK_TAGS                           101
-#define TK_AS                             102
+#define TK_USING                          102
-#define TK_COMMA                          103
+#define TK_AS                             103
-#define TK_NULL                           104
+#define TK_COMMA                          104
-#define TK_SELECT                         105
+#define TK_NULL                           105
-#define TK_UNION                          106
+#define TK_SELECT                         106
-#define TK_ALL                            107
+#define TK_UNION                          107
-#define TK_FROM                           108
+#define TK_ALL                            108
-#define TK_VARIABLE                       109
+#define TK_FROM                           109
-#define TK_INTERVAL                       110
+#define TK_VARIABLE                       110
-#define TK_FILL                           111
+#define TK_INTERVAL                       111
-#define TK_SLIDING                        112
+#define TK_FILL                           112
-#define TK_ORDER                          113
+#define TK_SLIDING                        113
-#define TK_BY                             114
+#define TK_ORDER                          114
-#define TK_ASC                            115
+#define TK_BY                             115
-#define TK_DESC                           116
+#define TK_ASC                            116
-#define TK_GROUP                          117
+#define TK_DESC                           117
-#define TK_HAVING                         118
+#define TK_GROUP                          118
-#define TK_LIMIT                          119
+#define TK_HAVING                         119
-#define TK_OFFSET                         120
+#define TK_LIMIT                          120
-#define TK_SLIMIT                         121
+#define TK_OFFSET                         121
-#define TK_SOFFSET                        122
+#define TK_SLIMIT                         122
-#define TK_WHERE                          123
+#define TK_SOFFSET                        123
-#define TK_NOW                            124
+#define TK_WHERE                          124
-#define TK_RESET                          125
+#define TK_NOW                            125
-#define TK_QUERY                          126
+#define TK_RESET                          126
-#define TK_ADD                            127
+#define TK_QUERY                          127
-#define TK_COLUMN                         128
+#define TK_ADD                            128
-#define TK_TAG                            129
+#define TK_COLUMN                         129
-#define TK_CHANGE                         130
+#define TK_TAG                            130
-#define TK_SET                            131
+#define TK_CHANGE                         131
-#define TK_KILL                           132
+#define TK_SET                            132
-#define TK_CONNECTION                     133
+#define TK_KILL                           133
-#define TK_STREAM                         134
+#define TK_CONNECTION                     134
-#define TK_COLON                          135
+#define TK_STREAM                         135
-#define TK_ABORT                          136
+#define TK_COLON                          136
-#define TK_AFTER                          137
+#define TK_ABORT                          137
-#define TK_ATTACH                         138
+#define TK_AFTER                          138
-#define TK_BEFORE                         139
+#define TK_ATTACH                         139
-#define TK_BEGIN                          140
+#define TK_BEFORE                         140
-#define TK_CASCADE                        141
+#define TK_BEGIN                          141
-#define TK_CLUSTER                        142
+#define TK_CASCADE                        142
-#define TK_CONFLICT                       143
+#define TK_CLUSTER                        143
-#define TK_COPY                           144
+#define TK_CONFLICT                       144
-#define TK_DEFERRED                       145
+#define TK_COPY                           145
-#define TK_DELIMITERS                     146
+#define TK_DEFERRED                       146
-#define TK_DETACH                         147
+#define TK_DELIMITERS                     147
-#define TK_EACH                           148
+#define TK_DETACH                         148
-#define TK_END                            149
+#define TK_EACH                           149
-#define TK_EXPLAIN                        150
+#define TK_END                            150
-#define TK_FAIL                           151
+#define TK_EXPLAIN                        151
-#define TK_FOR                            152
+#define TK_FAIL                           152
-#define TK_IGNORE                         153
+#define TK_FOR                            153
-#define TK_IMMEDIATE                      154
+#define TK_IGNORE                         154
-#define TK_INITIALLY                      155
+#define TK_IMMEDIATE                      155
-#define TK_INSTEAD                        156
+#define TK_INITIALLY                      156
-#define TK_MATCH                          157
+#define TK_INSTEAD                        157
-#define TK_KEY                            158
+#define TK_MATCH                          158
-#define TK_OF                             159
+#define TK_KEY                            159
-#define TK_RAISE                          160
+#define TK_OF                             160
-#define TK_REPLACE                        161
+#define TK_RAISE                          161
-#define TK_RESTRICT                       162
+#define TK_REPLACE                        162
-#define TK_ROW                            163
+#define TK_RESTRICT                       163
-#define TK_STATEMENT                      164
+#define TK_ROW                            164
-#define TK_TRIGGER                        165
+#define TK_STATEMENT                      165
-#define TK_VIEW                           166
+#define TK_TRIGGER                        166
-#define TK_COUNT                          167
+#define TK_VIEW                           167
-#define TK_SUM                            168
+#define TK_COUNT                          168
-#define TK_AVG                            169
+#define TK_SUM                            169
-#define TK_MIN                            170
+#define TK_AVG                            170
-#define TK_MAX                            171
+#define TK_MIN                            171
-#define TK_FIRST                          172
+#define TK_MAX                            172
-#define TK_LAST                           173
+#define TK_FIRST                          173
-#define TK_TOP                            174
+#define TK_LAST                           174
-#define TK_BOTTOM                         175
+#define TK_TOP                            175
-#define TK_STDDEV                         176
+#define TK_BOTTOM                         176
-#define TK_PERCENTILE                     177
+#define TK_STDDEV                         177
-#define TK_APERCENTILE                    178
+#define TK_PERCENTILE                     178
-#define TK_LEASTSQUARES                   179
+#define TK_APERCENTILE                    179
-#define TK_HISTOGRAM                      180
+#define TK_LEASTSQUARES                   180
-#define TK_DIFF                           181
+#define TK_HISTOGRAM                      181
-#define TK_SPREAD                         182
+#define TK_DIFF                           182
-#define TK_TWA                            183
+#define TK_SPREAD                         183
-#define TK_INTERP                         184
+#define TK_TWA                            184
-#define TK_LAST_ROW                       185
+#define TK_INTERP                         185
-#define TK_RATE                           186
+#define TK_LAST_ROW                       186
-#define TK_IRATE                          187
+#define TK_RATE                           187
-#define TK_SUM_RATE                       188
+#define TK_IRATE                          188
-#define TK_SUM_IRATE                      189
+#define TK_SUM_RATE                       189
-#define TK_AVG_RATE                       190
+#define TK_SUM_IRATE                      190
-#define TK_AVG_IRATE                      191
+#define TK_AVG_RATE                       191
-#define TK_TBID                           192
+#define TK_AVG_IRATE                      192
-#define TK_SEMI                           193
+#define TK_TBID                           193
-#define TK_NONE                           194
+#define TK_SEMI                           194
-#define TK_PREV                           195
+#define TK_NONE                           195
-#define TK_LINEAR                         196
+#define TK_PREV                           196
-#define TK_IMPORT                         197
+#define TK_LINEAR                         197
-#define TK_METRIC                         198
+#define TK_IMPORT                         198
-#define TK_TBNAME                         199
+#define TK_METRIC                         199
-#define TK_JOIN                           200
+#define TK_TBNAME                         200
-#define TK_METRICS                        201
+#define TK_JOIN                           201
-#define TK_STABLE                         202
+#define TK_METRICS                        202
-#define TK_INSERT                         203
+#define TK_STABLE                         203
-#define TK_INTO                           204
+#define TK_INSERT                         204
-#define TK_VALUES                         205
+#define TK_INTO                           205
+#define TK_VALUES                         206
 
 #define TK_SPACE                          300
 #define TK_COMMENT                        301

src/inc/vnode.h

@@ -37,10 +37,15 @@ typedef struct {
 } SRspRet;
 
 typedef struct {
-  SRspRet rspRet;
+  int32_t code;
-  void *  pCont;
   int32_t contLen;
-  SRpcMsg rpcMsg;
+  void *  rpcHandle;
+  void *  rpcAhandle;
+  void *  qhandle;
+  int8_t  qtype;
+  int8_t  msgType;
+  SRspRet rspRet;
+  char    pCont[];
 } SVReadMsg;
 
 typedef struct {
@@ -62,13 +67,11 @@ int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeClose(int32_t vgId);
 
 void*   vnodeAcquire(int32_t vgId);        // add refcount
-void*   vnodeAcquireRqueue(int32_t vgId);  // add refCount, get read queue 
 void    vnodeRelease(void *pVnode);        // dec refCount
 void*   vnodeGetWal(void *pVnode);
 
 int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam);
 int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam);
-int32_t vnodeCheckWrite(void *pVnode);
 int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
 void    vnodeBuildStatusMsg(void *param);
 void    vnodeConfirmForward(void *param, uint64_t version, int32_t code);
@@ -77,8 +80,8 @@ void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
 int32_t vnodeInitResources();
 void    vnodeCleanupResources();
 
-int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pReadMsg);
+int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
-int32_t vnodeCheckRead(void *pVnode);
+int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeDef.h

@@ -172,7 +172,8 @@ typedef struct {
   int8_t  walLevel;
   int8_t  replications;
   int8_t  quorum;
-  int8_t  reserved[12];
+  int8_t  update;
+  int8_t  reserved[11];
 } SDbCfg;
 
 typedef struct SDbObj {

src/mnode/src/mnodeDb.c

@@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
   }
 #endif
 
+  if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
+    mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
+    return TSDB_CODE_MND_INVALID_DB_OPTION;
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
   if (pCfg->replications < 0) pCfg->replications = tsReplications;
   if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
+  if (pCfg->update < 0) pCfg->update = tsUpdate;
 }
 
 static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@@ -391,7 +397,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
     .compression         = pCreate->compression,
     .walLevel            = pCreate->walLevel,
     .replications        = pCreate->replications,
-    .quorum              = pCreate->quorum
+    .quorum              = pCreate->quorum,
+    .update              = pCreate->update
   };
 
   mnodeSetDefaultDbCfg(&pDb->cfg);
@@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  pShow->bytes[cols] = 1;
+  pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
+  strcpy(pSchema[cols].name, "update");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "status");
@@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
     STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
     cols++;
 
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int8_t *)pWrite = pDb->cfg.update;
+    cols++;
+
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     if (pDb->status == TSDB_DB_STATUS_READY) {
       const char *src = "ready";
@@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
   int8_t  replications   = pAlter->replications;
   int8_t  quorum         = pAlter->quorum;
   int8_t  precision      = pAlter->precision;
+  int8_t  update         = pAlter->update;
   
   terrno = TSDB_CODE_SUCCESS;
 
@@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
     newCfg.quorum = quorum;
   }
 
+  if (update >= 0 && update != pDb->cfg.update) {
+#if 0
+    mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update);
+    newCfg.update = update;
+#else
+    mError("db:%s, can't alter update option", pDb->name);
+    terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+#endif
+  }
+
   return newCfg;
 }
 

src/mnode/src/mnodeVgroup.c

@@ -850,6 +850,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
   pCfg->replications        = (int8_t) pVgroup->numOfVnodes;
   pCfg->wals                = 3;
   pCfg->quorum              = pDb->cfg.quorum;
+  pCfg->update              = pDb->cfg.update;
   
   SVnodeDesc *pNodes = pVnode->nodes;
   for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {

src/query/inc/qSqlparser.h

@@ -129,6 +129,7 @@ typedef struct SCreateDBInfo {
   int32_t   compressionLevel;
   SStrToken precision;
   bool      ignoreExists;
+  int8_t    update; 
   
   tVariantList *keep;
 } SCreateDBInfo;

src/query/inc/sql.y

@@ -239,6 +239,7 @@ wal(Y)     ::= WAL INTEGER(X).                { Y = X; }
 fsync(Y)   ::= FSYNC INTEGER(X).              { Y = X; }
 comp(Y)    ::= COMP INTEGER(X).               { Y = X; }
 prec(Y)    ::= PRECISION STRING(X).           { Y = X; }
+update(Y)  ::= UPDATE INTEGER(X).             { Y = X; }     
 
 %type db_optr {SCreateDBInfo}
 db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
@@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X).          { Y = Z; Y.fsyncPeriod = strtol(X.z
 db_optr(Y) ::= db_optr(Z) comp(X).           { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
 db_optr(Y) ::= db_optr(Z) prec(X).           { Y = Z; Y.precision = X; }
 db_optr(Y) ::= db_optr(Z) keep(X).           { Y = Z; Y.keep = X; }
+db_optr(Y) ::= db_optr(Z) update(X).         { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 
 %type alter_db_optr {SCreateDBInfo}
 alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
@@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X).      { Y = Z; Y.numOfBlocks = s
 alter_db_optr(Y) ::= alter_db_optr(Z) comp(X).        { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) wal(X).         { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X).       { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) update(X).       { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 
 %type typename {TAOS_FIELD}
 typename(A) ::= ids(X). { 

src/query/inc/tsqlfunction.h

@@ -149,7 +149,7 @@ typedef struct SResultRowCellInfo {
   int8_t   hasResult;       // result generated, not NULL value
   bool     initialized;     // output buffer has been initialized
   bool     complete;        // query has completed
-  uint16_t numOfRes;        // num of output result in current buffer
+  uint32_t numOfRes;        // num of output result in current buffer
 } SResultRowCellInfo;
 
 #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))

src/query/src/qAst.c

@@ -32,7 +32,6 @@
 #include "tstoken.h"
 #include "ttokendef.h"
 #include "tulog.h"
-#include "tutil.h"
 
 /*
  *
@@ -415,9 +414,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   }
 
   if (cond.start != NULL) {
-    iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
+    iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC);
   } else {
-    iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+    iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC);
   }
   
   if (cond.start != NULL) {
@@ -432,7 +431,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
           break;
         }
 
-        STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+        STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
         taosArrayPush(result, &info);
       }
     } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
@@ -450,7 +449,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
         if (ret == 0 && optr == TSDB_RELATION_GREATER) {
           continue;
         } else {
-          STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+          STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
           taosArrayPush(result, &info);
           comp = false;
         }
@@ -465,14 +464,14 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
           continue;
         }
 
-        STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+        STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
         taosArrayPush(result, &info);
       }
       
       tSkipListDestroyIter(iter);
   
       comp = true;
-      iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
+      iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC);
       while(tSkipListIterNext(iter)) {
         SSkipListNode* pNode = tSkipListIterGet(iter);
         comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
@@ -480,7 +479,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
           continue;
         }
 
-        STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+        STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
         taosArrayPush(result, &info);
       }
   
@@ -504,7 +503,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
         if (ret == 0 && optr == TSDB_RELATION_LESS) {
           continue;
         } else {
-          STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+          STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
           taosArrayPush(result, &info);
           comp = false;  // no need to compare anymore
         }
@@ -518,7 +517,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
         bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
         if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
             (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
-          STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
+          STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
           taosArrayPush(result, &info);
         }
       }
@@ -682,7 +681,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
   while (tSkipListIterNext(iter)) {
     SSkipListNode *pNode = tSkipListIterGet(iter);
     if (filterItem(pExpr, pNode, param)) {
-      taosArrayPush(pResult, SL_GET_NODE_DATA(pNode));
+      taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode)));
     }
   }
   tSkipListDestroyIter(iter);
@@ -697,7 +696,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
     SSkipListNode *pNode = tSkipListIterGet(iter);
     char *         pData = SL_GET_NODE_DATA(pNode);
 
-    tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
+    tstr *name = (tstr*) tsdbGetTableName((void*) pData);
 
     // todo speed up by using hash
     if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
@@ -711,7 +710,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
     }
 
     if (addToResult) {
-      STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL};
+      STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
       taosArrayPush(res, &info);
     }
   }

src/query/src/qExecutor.c

@@ -1609,7 +1609,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
 
   pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
   pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t));
-  pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);//calloc(1, sizeof(SResultRow));
+  pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);
 
   if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) {
     goto _clean;
@@ -1745,6 +1745,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 
   pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
   taosTFree(pRuntimeEnv->keyBuf);
+  taosTFree(pRuntimeEnv->rowCellInfoOffset);
 
   taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
   pRuntimeEnv->pResultRowHashTable = NULL;

src/query/src/qFill.c

@@ -170,7 +170,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
   int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
 
   TSKEY ekey1 = ekey;
-  if (pFillInfo->order != TSDB_ORDER_ASC) {
+  if (!FILL_IS_ASC_FILL(pFillInfo)) {
     pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
   }
 

src/query/src/qHistogram.c

@@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
       (*pHisto)->numOfEntries += 1;
     }
   } else { /* insert a new slot */
-    if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) {
+    if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
       if (idx > 0) {
         assert((*pHisto)->elems[idx - 1].val <= val);
       }

src/query/src/qParserImpl.c

@@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
   pDBInfo->quorum = -1;
   pDBInfo->keep = NULL;
 
+  pDBInfo->update = -1;
   memset(&pDBInfo->precision, 0, sizeof(SStrToken));
 }

src/query/src/qTokenizer.c

@@ -155,6 +155,7 @@ static SKeyword keywordTable[] = {
     {"INSERT",       TK_INSERT},
     {"INTO",         TK_INTO},
     {"VALUES",       TK_VALUES},
+    {"UPDATE",       TK_UPDATE},
     {"RESET",        TK_RESET},
     {"QUERY",        TK_QUERY},
     {"ADD",          TK_ADD},

src/rpc/src/rpcMain.c

@@ -1622,7 +1622,7 @@ static void rpcDecRef(SRpcInfo *pRpc)
 
     int count = atomic_sub_fetch_32(&tsRpcNum, 1);
     if (count == 0) {
-      taosCloseRef(tsRpcRefId);
+      // taosCloseRef(tsRpcRefId);
       // tsRpcInit = PTHREAD_ONCE_INIT;    // windows compliling error  
     }
   }

src/tsdb/inc/tsdbMain.h

@@ -320,6 +320,15 @@ typedef struct {
   void*      compBuffer;  // Buffer for temperary compress/decompress purpose
 } SRWHelper;
 
+typedef struct {
+  int   rowsInserted;
+  int   rowsUpdated;
+  int   rowsDeleteSucceed;
+  int   rowsDeleteFailed;
+  int   nOperations;
+  TSKEY keyFirst;
+  TSKEY keyLast;
+} SMergeInfo;
 // ------------------ tsdbScan.c
 typedef struct {
   SFileGroup fGroup;
@@ -422,7 +431,7 @@ void          tsdbCloseBufPool(STsdbRepo* pRepo);
 SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
 
 // ------------------ tsdbMemTable.c
-int   tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
+int   tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
 int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
 int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
 int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
@@ -430,7 +439,7 @@ void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem)
 void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
 int   tsdbAsyncCommit(STsdbRepo* pRepo);
 int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
-                            TSKEY* filterKeys, int nFilterKeys);
+                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
 
 static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
   if (pIter == NULL) return NULL;
@@ -438,16 +447,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
   SSkipListNode* node = tSkipListIterGet(pIter);
   if (node == NULL) return NULL;
 
-  return *(SDataRow *)SL_GET_NODE_DATA(node);
+  return (SDataRow)SL_GET_NODE_DATA(node);
 }
 
 static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
   SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return -1;
+  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
 
   return dataRowKey(row);
 }
 
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+  SDataRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TKEY_NULL;
+
+  return dataRowTKey(row);
+}
+
 static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
   ASSERT(pRepo != NULL);
   if (pRepo->mem == NULL) return NULL;

src/tsdb/src/tsdbMain.c

@@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
     }
   }
 
+  // update check
+  if (pCfg->update != 0) pCfg->update = 1;
+
   return 0;
 
 _err:
@@ -762,7 +765,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY
       return -1;
     }
 
-    if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1;
+    if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
 
     (*affectedrows)++;
     points++;
@@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
   tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
   tlen += taosEncodeFixedI8(buf, pCfg->precision);
   tlen += taosEncodeFixedI8(buf, pCfg->compression);
+  tlen += taosEncodeFixedI8(buf, pCfg->update);
 
   return tlen;
 }
@@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
   buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
   buf = taosDecodeFixedI8(buf, &(pCfg->precision));
   buf = taosDecodeFixedI8(buf, &(pCfg->compression));
+  buf = taosDecodeFixedI8(buf, &(pCfg->update));
 
   return buf;
 }

src/tsdb/src/tsdbMemTable.c

@@ -32,43 +32,40 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
 static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
 static void         tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
 static int          tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
+static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
 
 // ---------------- INTERNAL FUNCTIONS ----------------
-int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
+int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
   STsdbCfg *  pCfg = &pRepo->config;
   STsdbMeta * pMeta = pRepo->tsdbMeta;
-  int32_t     level = 0;
+  TKEY        tkey = dataRowTKey(row);
-  int32_t     headSize = 0;
   TSKEY       key = dataRowKey(row);
   SMemTable * pMemTable = pRepo->mem;
   STableData *pTableData = NULL;
-  SSkipList * pSList = NULL;
+  bool        isRowDelete = TKEY_IS_DELETED(tkey);
 
-  if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL &&
+  if (isRowDelete) {
-      pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) {
+    if (!pCfg->update) {
-    pTableData = pMemTable->tData[TABLE_TID(pTable)];
+      tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
-    pSList = pTableData->pData;
+      terrno = TSDB_CODE_TDB_INVALID_ACTION;
+      return -1;
     }
 
-  tSkipListNewNodeInfo(pSList, &level, &headSize);
+    if (key > TABLE_LASTKEY(pTable)) {
-
+      tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
-  SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *));
+                REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
-  if (pNode == NULL) {
+      return 0;
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    }
-    return -1;
   }
 
   void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
   if (pRow == NULL) {
     tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
               REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
-    free(pNode);
     return -1;
   }
 
-  pNode->level = level;
   dataRowCpy(pRow, row);
-  *(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow;
 
   // Operations above may change pRepo->mem, retake those values
   ASSERT(pRepo->mem != NULL);
@@ -77,7 +74,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
   if (TABLE_TID(pTable) >= pMemTable->maxTables) {
     if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
       tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
-      free(pNode);
       return -1;
     }
   }
@@ -97,7 +93,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
                 " to table %s while create new table data object since %s",
                 REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
       tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
-      free(pNode);
       return -1;
     }
 
@@ -106,24 +101,31 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
 
   ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
 
-  if (tSkipListPut(pTableData->pData, pNode) == NULL) {
+  int64_t oldSize = SL_SIZE(pTableData->pData);
+  if (tSkipListPut(pTableData->pData, pRow) == NULL) {
     tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
-    free(pNode);
+  } else {
+    int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
+    if (isRowDelete) {
+      if (TABLE_LASTKEY(pTable) == key) {
+        // TODO: need to update table last key here (may from file)
+      }
     } else {
       if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
+    }
+
     if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
     if (pMemTable->keyLast < key) pMemTable->keyLast = key;
-    pMemTable->numOfRows++;
+    pMemTable->numOfRows += deltaSize;
 
     if (pTableData->keyFirst > key) pTableData->keyFirst = key;
     if (pTableData->keyLast < key) pTableData->keyLast = key;
-    pTableData->numOfRows++;
+    pTableData->numOfRows += deltaSize;
-
-    ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
   }
 
-  tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
+  tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
-            TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key);
+            isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
+            key);
 
   return 0;
 }
@@ -295,63 +297,124 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
   return 0;
 }
 
+/**
+ * This is an important function to load data or try to load data from memory skiplist iterator.
+ * 
+ * This function load memory data until:
+ * 1. iterator ends
+ * 2. data key exceeds maxKey
+ * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
+ * 4. operations in pCols not exceeds its max capacity if pCols is given
+ * 
+ * The function tries to procceed AS MUSH AS POSSIBLE.
+ */
 int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
-                          TSKEY *filterKeys, int nFilterKeys) {
+                          TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
-  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
+  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
   if (pIter == NULL) return 0;
   STSchema *pSchema = NULL;
-  int       numOfRows = 0;
+  TSKEY     rowKey = 0;
-  TSKEY     keyNext = 0;
+  TSKEY     fKey = 0;
+  bool      isRowDel = false;
   int       filterIter = 0;
+  SDataRow  row = NULL;
 
-  if (nFilterKeys != 0) { // for filter purpose
+  memset(pMergeInfo, 0, sizeof(*pMergeInfo));
-    ASSERT(filterKeys != NULL);
+  pMergeInfo->keyFirst = INT64_MAX;
-    keyNext = tsdbNextIterKey(pIter);
+  pMergeInfo->keyLast = INT64_MIN;
-    if (keyNext < 0 || keyNext > maxKey) return numOfRows;
+  if (pCols) tdResetDataCols(pCols);
-    void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
-    filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)));
-  }
 
-  do {
+  row = tsdbNextIterRow(pIter);
-    SDataRow row = tsdbNextIterRow(pIter);
+  if (row == NULL || dataRowKey(row) > maxKey) {
-    if (row == NULL) break;
+    rowKey = INT64_MAX;
-
+    isRowDel = false;
-    keyNext = dataRowKey(row);
-    if (keyNext > maxKey) break;
-
-    bool keyFiltered = false;
-    if (nFilterKeys != 0) {
-      while (true) {
-        if (filterIter >= nFilterKeys) break;
-        if (keyNext == filterKeys[filterIter]) {
-          keyFiltered = true;
-          filterIter++;
-          break;
-        } else if (keyNext < filterKeys[filterIter]) {
-          break;
   } else {
+    rowKey = dataRowKey(row);
+    isRowDel = dataRowDeleted(row);
+  }
+
+  if (filterIter >= nFilterKeys) {
+    fKey = INT64_MAX;
+  } else {
+    fKey = tdGetKey(filterKeys[filterIter]);
+  }
+
+  while (true) {
+    if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
+
+    if (fKey < rowKey) {
+      pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+      pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
+
       filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
+      }
+    } else if (fKey > rowKey) {
+      if (isRowDel) {
+        pMergeInfo->rowsDeleteFailed++;
+      } else {
+        if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+        pMergeInfo->rowsInserted++;
+        pMergeInfo->nOperations++;
+        pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+        pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+      }
+
+      tSkipListIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || dataRowKey(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = dataRowKey(row);
+        isRowDel = dataRowDeleted(row);
+      }
+    } else {
+      if (isRowDel) {
+        ASSERT(!keepDup);
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+        pMergeInfo->rowsDeleteSucceed++;
+        pMergeInfo->nOperations++;
+        tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+      } else {
+        if (keepDup) {
+          if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+          pMergeInfo->rowsUpdated++;
+          pMergeInfo->nOperations++;
+          pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
+          pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
+          tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
+        } else {
+          pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
+          pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
+        }
+      }
+
+      tSkipListIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || dataRowKey(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = dataRowKey(row);
+        isRowDel = dataRowDeleted(row);
+      }
+
+      filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
       }
     }
   }
 
-    if (!keyFiltered) {
+  return 0;
-      if (numOfRows >= maxRowsToRead) break;
-      if (pCols) {
-        if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
-          pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
-          if (pSchema == NULL) {
-            ASSERT(0);
-          }
-        }
-
-        tdAppendDataRowToDataCol(row, pSchema, pCols);
-      }
-      numOfRows++;
-    }
-  } while (tSkipListIterNext(pIter));
-
-  return numOfRows;
 }
 
 // ---------------- LOCAL FUNCTIONS ----------------
@@ -440,8 +503,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
   pTableData->keyLast = 0;
   pTableData->numOfRows = 0;
 
-  pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP,
+  pTableData->pData =
-                                      TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey);
+      tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
+                      tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey);
   if (pTableData->pData == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     goto _err;
@@ -461,7 +525,7 @@ static void tsdbFreeTableData(STableData *pTableData) {
   }
 }
 
-static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); }
+static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
 
 static void *tsdbCommitData(void *arg) {
   STsdbRepo *  pRepo = (STsdbRepo *)arg;
@@ -583,7 +647,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo) {
 static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
   for (int i = 0; i < nIters; i++) {
     TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
-    if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+    if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
   }
   return 0;
 }
@@ -781,3 +845,19 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
 
   return 0;
 }
+
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
+  if (pCols) {
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
+      if (*ppSchema == NULL) {
+        ASSERT(false);
+        return -1;
+      }
+    }
+
+    tdAppendDataRowToDataCol(row, *ppSchema, pCols);
+  }
+
+  return 0;
+}

src/tsdb/src/tsdbMeta.c

@@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
   if (pTable != NULL) {
     tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
               TABLE_TID(pTable), TABLE_UID(pTable));
-    return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+    terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+    goto _err;
   }
 
   if (pCfg->type == TSDB_CHILD_TABLE) {
@@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) {
 }
 
 static char *getTagIndexKey(const void *pData) {
-  STable *pTable = *(STable **)pData;
+  STable *pTable = (STable *)pData;
 
   STSchema *pSchema = tsdbGetTableTagSchema(pTable);
   STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
@@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
     }
     pTable->tagVal = NULL;
     STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
-    pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
+    pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey);
     if (pTable->pIndex == NULL) {
       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
       goto _err;
@@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
 
   pTable->pSuper = pSTable;
 
-  int32_t level = 0;
+  tSkipListPut(pSTable->pIndex, (void *)pTable);
-  int32_t headSize = 0;
 
-  tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
-
-  // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
-  // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
-  SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
-  if (pNode == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-  pNode->level = level;
-
-  memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
-
-  tSkipListPut(pSTable->pIndex, pNode);
   if (refSuper) T_REF_INC(pSTable);
   return 0;
 }
@@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
     SSkipListNode *pNode = taosArrayGetP(res, i);
 
     // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
-    if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) {  // this is the exact what we need
+    if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) {  // this is the exact what we need
       tSkipListRemoveNode(pSTable->pIndex, pNode);
     }
   }
@@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
     if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
       buf = tdDecodeSchema(buf, &(pTable->tagSchema));
       STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
-      pTable->pIndex =
+      pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL,
-          tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
+                                       SL_ALLOW_DUP_KEY, getTagIndexKey);
       if (pTable->pIndex == NULL) {
         terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
         tsdbFreeTable(pTable);
@@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
     tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM);
   } else {
     if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
-      tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1));
+      tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1));
     } else {
       tlen = sizeof(SListNode) + sizeof(SActObj);
     }
@@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) {
     }
 
     while (tSkipListIterNext(pIter)) {
-      STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+      STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
       ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
       pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable);
     }
@@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) {
     tsdbWLockRepoMeta(pRepo);
 
     while (tSkipListIterNext(pIter)) {
-      STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
+      STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
       tsdbRemoveTableFromMeta(pRepo, tTable, false, false);
     }
 

src/tsdb/src/tsdbRWHelper.c

@@ -25,6 +25,7 @@
 #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
 #define TSDB_KEY_COL_OFFSET 0
 #define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
+#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
 
 static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
 static int  tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
@@ -32,7 +33,7 @@ static int  tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD
 static int  compareKeyBlock(const void *arg1, const void *arg2);
 static int  tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
 static int  tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static int  tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
+static int  tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
 static int  tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
 static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
 static int  tsdbInitHelperFile(SRWHelper *pHelper);
@@ -59,8 +60,10 @@ static int   tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp
 static int   tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
 static int   tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
                                     int *blkIdx);
-static int   tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+static void  tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
-                                       TSKEY maxKey, int maxRows);
+                                       TSKEY maxKey, int maxRows, int8_t update);
+static bool  tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
+static int   tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
 
 // ---------------------- INTERNAL FUNCTIONS ----------------------
 int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
@@ -277,7 +280,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols
   while (true) {
     ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
     TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
-    if (keyFirst < 0 || keyFirst > maxKey) break;  // iter over
+    if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break;  // iter over
 
     if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
       if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
@@ -923,7 +926,7 @@ _err:
   return -1;
 }
 
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) {
+static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
   ASSERT(pCompBlock->numOfSubBlocks == 0);
 
   SCompIdx *pIdx = &(pHelper->curCompIdx);
@@ -956,9 +959,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
     pSCompBlock->numOfSubBlocks++;
     ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
     pSCompBlock->len += sizeof(SCompBlock);
-    pSCompBlock->numOfRows += rowsAdded;
+    pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
-    pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
+    pSCompBlock->keyFirst = pMergeInfo->keyFirst;
-    pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
+    pSCompBlock->keyLast = pMergeInfo->keyLast;
     pIdx->len += sizeof(SCompBlock);
   } else {  // Need to create two sub-blocks
     void *ptr = NULL;
@@ -987,11 +990,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
     ((SCompBlock *)ptr)[1] = *pCompBlock;
 
     pSCompBlock->numOfSubBlocks = 2;
-    pSCompBlock->numOfRows += rowsAdded;
+    pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
     pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
     pSCompBlock->len = sizeof(SCompBlock) * 2;
-    pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
+    pSCompBlock->keyFirst = pMergeInfo->keyFirst;
-    pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast);
+    pSCompBlock->keyLast = pMergeInfo->keyLast;
 
     pIdx->len += (sizeof(SCompBlock) * 2);
   }
@@ -1045,6 +1048,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
   return 0;
 }
 
+static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
+  SCompIdx *pCompIdx = &(pHelper->curCompIdx);
+
+  ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
+
+  SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
+  SCompBlock  compBlock = *pCompBlock;
+  ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
+
+  if (pCompIdx->numOfBlocks == 1) {
+    memset(pCompIdx, 0, sizeof(*pCompIdx));
+  } else {
+    int tsize = 0;
+
+    if (compBlock.numOfSubBlocks > 1) {
+      tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
+
+      ASSERT(tsize > 0);
+      memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
+              POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
+              tsize);
+
+      pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
+    }
+
+    tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
+    ASSERT(tsize > 0);
+    memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
+
+    pCompIdx->len -= sizeof(SCompBlock);
+
+    pCompIdx->numOfBlocks--;
+    pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
+    pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
+  }
+
+  return 0;
+}
+
 static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
   pHelper->idxH.numOfIdx = 0;
   pHelper->idxH.curIdx = 0;
@@ -1443,20 +1485,28 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
   TSKEY       keyFirst = tsdbNextIterKey(pCommitIter->pIter);
   int         defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
   SCompBlock  compBlock = {0};
+  SMergeInfo  mergeInfo = {0};
+  SMergeInfo *pMergeInfo = &mergeInfo;
 
   ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
   if (pIdx->hasLast) {  // append to with last block
     ASSERT(pIdx->len > 0);
     SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
     ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
-    tdResetDataCols(pDataCols);
+    tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
-    int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows,
+                          NULL, 0, pCfg->update, pMergeInfo);
-                                         pDataCols, NULL, 0);
+
-    ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
+    ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
-    if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
+
+    if (pDataCols->numOfRows > 0) {
+      ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
+
+      if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
           pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
         if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
-      if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1;
+        pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
+        pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
+        if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
       } else {
         if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
         ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
@@ -1469,19 +1519,22 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
       }
 
       if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+    }
   } else {
     ASSERT(!pHelper->hasOldLastBlock);
-    tdResetDataCols(pDataCols);
+    tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
-    int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0);
+    ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
-    ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
 
+    if (pDataCols->numOfRows > 0) {
+      ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
       if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
       if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
     }
+  }
 
 #ifndef NDEBUG
   TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
-  ASSERT(keyNext < 0 || keyNext > pIdx->maxKey);
+  ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
 #endif
 
   return 0;
@@ -1495,7 +1548,10 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
   SCompBlock  compBlock = {0};
   TSKEY       keyFirst = tsdbNextIterKey(pCommitIter->pIter);
   int         defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
-  SDataCols *pDataCols0 = pHelper->pDataCols[0];
+  SDataCols * pDataCols0 = pHelper->pDataCols[0];
+  SMergeInfo  mergeInfo = {0};
+  SMergeInfo *pMergeInfo = &mergeInfo;
+  SCompBlock  oBlock = {0};
 
   SSkipListIterator slIter = {0};
 
@@ -1505,112 +1561,75 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
                                        pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
   ASSERT(pCompBlock != NULL);
   int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
+  oBlock = *pCompBlock;
 
-  if (pCompBlock->last) {
+  ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
-    ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1);
-    int16_t colId = 0;
-    slIter = *(pCommitIter->pIter);
-    if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
-    ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows);
 
-    int rows1 = defaultRowsInBlock - pCompBlock->numOfRows;
+  if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
-    int rows2 =
-        tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows);
-    if (rows2 == 0) {  // all data filtered out
-      *(pCommitIter->pIter) = slIter;
-    } else {
-      if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock &&
-          pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
-        tdResetDataCols(pDataCols);
-        int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols,
-                                             pDataCols0->cols[0].pData, pDataCols0->numOfRows);
-        ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows);
-        if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
-        if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
-        tblkIdx++;
-      } else {
-        if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
-        int round = 0;
-        int dIter = 0;
     while (true) {
-          tdResetDataCols(pDataCols);
+      tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
-          int rowsRead =
+                            pCfg->update, pMergeInfo);
-              tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock);
+      ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
-          if (rowsRead == 0) break;
+      if (pDataCols->numOfRows == 0) break;
 
-          if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
-          if (round == 0) {
-            if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
-          } else {
-            if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
-          }
-
-          tblkIdx++;
-          round++;
-        }
-      }
-      if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
-    }
-  } else {
-    TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1);
-    TSKEY blkKeyFirst = pCompBlock->keyFirst;
-    TSKEY blkKeyLast = pCompBlock->keyLast;
-
-    if (keyFirst < blkKeyFirst) {
-      while (true) {
-        tdResetDataCols(pDataCols);
-        int rowsRead =
-            tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0);
-        if (rowsRead == 0) break;
-
-        ASSERT(rowsRead == pDataCols->numOfRows);
       if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
       if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
       tblkIdx++;
     }
-      ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
+    ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
                             tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
   } else {
-      ASSERT(keyFirst <= blkKeyLast);
     int16_t colId = 0;
-      if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
+    if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
+
+    TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
 
     slIter = *(pCommitIter->pIter);
-      int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows);
+    tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
-      int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData,
+                          pCfg->update, pMergeInfo);
-                                        pDataCols0->numOfRows);
 
-      if (rows2 == 0) {  // all filtered out
+    if (pMergeInfo->nOperations == 0) {
+      // Do nothing
+      ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
       *(pCommitIter->pIter) = slIter;
-        ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
-                                tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
-      } else {
-        int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2;
-
-        if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) {
-          int rows = (rows1 >= rows3) ? rows3 : rows2;
-          tdResetDataCols(pDataCols);
-          int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols,
-                                               pDataCols0->cols[0].pData, pDataCols0->numOfRows);
-          ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows);
-          if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0)
-            return -1;
-          if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
       tblkIdx++;
-          ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
+    } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
-                                  tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
+      // Delete the block and do some stuff
+      ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
+      if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
+      *pCommitIter->pIter = slIter;
+      if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
+    } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
+      // Append as a sub-block of the searched block
+      tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
+                            pDataCols0->numOfRows, pCfg->update, pMergeInfo);
+      ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
+      if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
+                               &compBlock, oBlock.last, false) < 0) {
+        return -1;
+      }
+      if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
+        return -1;
+      }
+      tblkIdx++;
     } else {
-          if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
+      // load the block data, merge with the memory data
+      if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
       int round = 0;
       int dIter = 0;
       while (true) {
-            int rowsRead =
+        tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
-                tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock);
+                                  pCfg->update);
-            if (rowsRead == 0) break;
+
+        if (pDataCols->numOfRows == 0) break;
+        if (tblkIdx == pIdx->numOfBlocks - 1) {
+          if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
+        } else {
+          if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
+        }
 
-            if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0)
-              return -1;
         if (round == 0) {
+          if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
           if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
         } else {
           if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
@@ -1619,10 +1638,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
         round++;
         tblkIdx++;
       }
-          ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
-                                  tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
-        }
-      }
     }
   }
 
@@ -1630,9 +1645,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
   return 0;
 }
 
-static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
-                                     TSKEY maxKey, int maxRows) {
+                                     TSKEY maxKey, int maxRows, int8_t update) {
-  int       numOfRows = 0;
   TSKEY     key1 = INT64_MAX;
   TSKEY     key2 = INT64_MAX;
   STSchema *pSchema = NULL;
@@ -1642,35 +1656,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte
 
   while (true) {
     key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
+    bool isRowDel = false;
     SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
-    key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row);
+    if (row == NULL || dataRowKey(row) > maxKey) {
+      key2 = INT64_MAX;
+    } else {
+      key2 = dataRowKey(row);
+      isRowDel = dataRowDeleted(row);
+    }
 
     if (key1 == INT64_MAX && key2 == INT64_MAX) break;
 
-    if (key1 <= key2) {
+    if (key1 < key2) {
       for (int i = 0; i < pDataCols->numOfCols; i++) {
         dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
                          pTarget->maxPoints);
       }
+
       pTarget->numOfRows++;
       (*iter)++;
-      if (key1 == key2) tSkipListIterNext(pCommitIter->pIter);
+    } else if (key1 > key2) {
-    } else {
+      if (!isRowDel) {
         if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
           pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
           ASSERT(pSchema != NULL);
         }
 
         tdAppendDataRowToDataCol(row, pSchema, pTarget);
+      }
+
+      tSkipListIterNext(pCommitIter->pIter);
+    } else {
+      if (update) {
+        if (!isRowDel) {
+          if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+            pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+            ASSERT(pSchema != NULL);
+          }
+
+          tdAppendDataRowToDataCol(row, pSchema, pTarget);
+        }
+      } else {
+        ASSERT(!isRowDel);
+
+        for (int i = 0; i < pDataCols->numOfCols; i++) {
+          dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
+                           pTarget->maxPoints);
+        }
+
+        pTarget->numOfRows++;
+      }
+      (*iter)++;
       tSkipListIterNext(pCommitIter->pIter);
     }
 
-    numOfRows++;
+    if (pTarget->numOfRows >= maxRows) break;
-    if (numOfRows >= maxRows) break;
-    ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints);
   }
-
-  return numOfRows;
 }
 
 static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
@@ -1693,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols,
 
   return 0;
 }
+
+static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
+  STsdbCfg *pCfg = &(pHelper->pRepo->config);
+  int       mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
+
+  ASSERT(mergeRows > 0);
+
+  if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
+    if (pCompBlock->last) {
+      if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
+    } else {
+      if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
+    }
+  }
+
+  return false;
+}

src/tsdb/src/tsdbRead.c

@@ -457,7 +457,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     assert(node != NULL);
 
-    SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
+    SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@@ -479,7 +479,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     assert(node != NULL);
 
-    SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
+    SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@@ -504,19 +504,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
   tSkipListDestroyIter(pCheckInfo->iiter);
 }
 
-static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) {
+static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
   SDataRow rmem = NULL, rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
-      rmem = *(SDataRow *)SL_GET_NODE_DATA(node);
+      rmem = (SDataRow)SL_GET_NODE_DATA(node);
     }
   }
 
   if (pCheckInfo->iiter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
-      rimem = *(SDataRow *)SL_GET_NODE_DATA(node);
+      rimem = (SDataRow)SL_GET_NODE_DATA(node);
     }
   }
 
@@ -538,9 +538,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
   TSKEY r2 = dataRowKey(rimem);
 
   if (r1 == r2) { // data ts are duplicated, ignore the data in mem
+    if (!update) {
       tSkipListIterNext(pCheckInfo->iter);
       pCheckInfo->chosen = 1;
       return rimem;
+    } else {
+      tSkipListIterNext(pCheckInfo->iiter);
+      pCheckInfo->chosen = 0;
+      return rmem;
+    }
   } else {
     if (ASCENDING_TRAVERSE(order)) {
       if (r1 < r2) {
@@ -594,6 +600,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
 }
 
 static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
+  STsdbCfg *pCfg = &pHandle->pTsdb->config;
   size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
   assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
   pHandle->cur.fid = -1;
@@ -607,7 +614,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
     initTableMemIterator(pHandle, pCheckInfo);
   }
 
-  SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order);
+  SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
   if (row == NULL) {
     return false;
   }
@@ -827,11 +834,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
 
 static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
   SQueryFilePos* cur = &pQueryHandle->cur;
+  STsdbCfg*      pCfg = &pQueryHandle->pTsdb->config;
   SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
   int32_t code = TSDB_CODE_SUCCESS;
 
   /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
-  SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+  SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
 
   assert(cur->pos >= 0 && cur->pos <= binfo.rows);
 
@@ -1316,6 +1324,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
 static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
+  STsdbCfg*      pCfg = &pQueryHandle->pTsdb->config;
 
   initTableMemIterator(pQueryHandle, pCheckInfo);
 
@@ -1353,7 +1362,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
     do {
-      SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+      SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
       if (row == NULL) {
         break;
       }
@@ -1383,7 +1392,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
         moveToNextRowInMem(pCheckInfo);
       } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
+        if (pCfg->update) {
+          copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
+          numOfRows += 1;
+          if (cur->win.skey == TSKEY_INITIAL_VAL) {
+            cur->win.skey = key;
+          }
+
+          cur->win.ekey = key;
+          cur->lastKey = key + step;
+          cur->mixBlock = true;
+
           moveToNextRowInMem(pCheckInfo);
+          pos += step;
+        } else {
+          moveToNextRowInMem(pCheckInfo);
+        }
       } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
                   (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -1394,7 +1418,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         assert(end != -1);
 
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
+          if (!pCfg->update) {
             moveToNextRowInMem(pCheckInfo);
+          } else {
+            end -= step;
+          }
         }
 
         int32_t qstart = 0, qend = 0;
@@ -1414,8 +1442,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
        * copy them all to result buffer, since it may be overlapped with file data block.
        */
       if (node == NULL ||
-          ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+          ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-          ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
+          ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         // no data in cache or data in cache is greater than the ekey of time window, load data from file block
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
@@ -1862,13 +1890,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
                                  STsdbQueryHandle* pQueryHandle) {
   int     numOfRows = 0;
   int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns);
+  STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
   win->skey = TSKEY_INITIAL_VAL;
 
   int64_t st = taosGetTimestampUs();
   STable* pTable = pCheckInfo->pTableObj;
 
   do {
-    SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
+    SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
     if (row == NULL) {
       break;
     }
@@ -1919,9 +1948,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
 
-    STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
+    STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode);
 
-    STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL};
+    STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL};
     taosArrayPush(list, &info);
   }
 
@@ -2464,7 +2493,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
 static bool indexedNodeFilterFp(const void* pNode, void* param) {
   tQueryInfo* pInfo = (tQueryInfo*) param;
 
-  STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
+  STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
   char*  val = NULL;
 

src/util/CMakeLists.txt

@@ -8,7 +8,7 @@ TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
   
 IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(tutil m rt)
-  ADD_SUBDIRECTORY(tests)
+  # ADD_SUBDIRECTORY(tests)
 
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)

src/util/inc/tskiplist.h

@@ -27,33 +27,25 @@ extern "C" {
 #define MAX_SKIP_LIST_LEVEL 15
 #define SKIP_LIST_RECORD_PERFORMANCE 0
 
+// For key property setting
+#define SL_ALLOW_DUP_KEY (uint8_t)0x0    // Allow duplicate key exists (for tag index usage)
+#define SL_DISCARD_DUP_KEY (uint8_t)0x1  // Discard duplicate key (for data update=0 case)
+#define SL_UPDATE_DUP_KEY (uint8_t)0x2   // Update duplicate key by remove/insert (for data update=1 case)
+// For thread safety setting
+#define SL_THREAD_SAFE (uint8_t)0x4
+
 typedef char *SSkipListKey;
 typedef char *(*__sl_key_fn_t)(const void *);
 
-/**
- * the skip list node is located in a consecutive memory area,
- * the format of skip list node is as follows:
- * +------------+-----------------------+------------------------+-----+------+
- * | node level | forward pointer array | backward pointer array | key | data |
- * +------------+-----------------------+------------------------+-----+------+
- */
 typedef struct SSkipListNode {
   uint8_t        level;
+  void *         pData;
+  struct SSkipListNode *forwards[];
 } SSkipListNode;
 
-#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
+#define SL_GET_NODE_DATA(n) (n)->pData
-
+#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)]
-#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
+#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
-#define SL_GET_BACKWARD_POINTER(n, _l) \
-  ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
-
-#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
-#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
-
-#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
-#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
-
-#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
 
 /*
  * @version 0.3
@@ -103,21 +95,16 @@ typedef struct tSkipListState {
   uint64_t nTotalElapsedTimeForInsert;
 } tSkipListState;
 
-typedef struct SSkipListKeyInfo {
-  uint8_t dupKey : 2;  // if allow duplicated key in the skip list
-  uint8_t type : 4;    // key type
-  uint8_t freeNode:2;  // free node when destroy the skiplist
-  uint8_t len;         // maximum key length, used in case of string key
-} SSkipListKeyInfo;
-
 typedef struct SSkipList {
   __compar_fn_t     comparFn;
   __sl_key_fn_t     keyFn;
-  uint32_t          size;
-  uint8_t           maxLevel;
-  uint8_t           level;
-  SSkipListKeyInfo  keyInfo;
   pthread_rwlock_t *lock;
+  uint16_t          len;
+  uint8_t           maxLevel;
+  uint8_t           flags;
+  uint8_t           type;  // static info above
+  uint8_t           level;
+  uint32_t          size;
   SSkipListNode *   pHead;  // point to the first element
   SSkipListNode *   pTail;  // point to the last element
 #if SKIP_LIST_RECORD_PERFORMANCE
@@ -125,12 +112,6 @@ typedef struct SSkipList {
 #endif
 } SSkipList;
 
-/*
- * iterate the skiplist
- * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
- * continue iterating the skiplist, so add the reference count for skiplist
- * TODO add the ref for skip list when one iterator is created
- */
 typedef struct SSkipListIterator {
   SSkipList *    pSkipList;
   SSkipListNode *cur;
@@ -139,113 +120,25 @@ typedef struct SSkipListIterator {
   SSkipListNode *next;          // next points to the true qualified node in skip list
 } SSkipListIterator;
 
-/**
+#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE)
- *
+#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1))
- * @param nMaxLevel   maximum skip list level
+#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
- * @param keyType     type of key
+#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
- * @param dupKey      allow the duplicated key in the skip list
+#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
- * @return
+#define SL_SIZE(s) (s)->size
- */
-SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
-    uint8_t freeNode, __sl_key_fn_t fn);
 
-/**
+SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
- *
+                           __sl_key_fn_t fn);
- * @param pSkipList
+void       tSkipListDestroy(SSkipList *pSkipList);
- * @return                NULL will always be returned
+SSkipListNode *    tSkipListPut(SSkipList *pSkipList, void *pData);
- */
+SArray *           tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
-void *tSkipListDestroy(SSkipList *pSkipList);
-
-/**
- *
- * @param pSkipList
- * @param level
- * @param headSize
- */
-void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
-
-/**
- * put the skip list node into the skip list.
- * If failed, NULL will be returned, otherwise, the pNode will be returned.
- *
- * @param pSkipList
- * @param pNode
- * @return
- */
-SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
-
-/**
- * get *all* nodes which key are equivalent to pKey
- *
- * @param pSkipList
- * @param pKey
- * @return
- */
-SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
-
-/**
- * get the size of skip list
- * @param pSkipList
- * @return
- */
-size_t tSkipListGetSize(const SSkipList *pSkipList);
-
-/**
- * display skip list of the given level, for debug purpose only
- * @param pSkipList
- * @param nlevel
- */
 void               tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
-
-/**
- * create skiplist iterator
- * @param pSkipList
- * @return
- */
 SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
-
+SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
-/**
- * create skip list iterator from the given node and specified the order
- * @param pSkipList
- * @param pNode     start position, instead of the first node in skip list
- * @param order     traverse order of the iterator
- * @return
- */
-SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order);
-
-/**
- * forward the skip list iterator
- * @param iter
- * @return
- */
 bool               tSkipListIterNext(SSkipListIterator *iter);
-
+SSkipListNode *    tSkipListIterGet(SSkipListIterator *iter);
-/**
+void *             tSkipListDestroyIter(SSkipListIterator *iter);
- * get the element of skip list node
- * @param iter
- * @return
- */
-SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
-
-/**
- * destroy the skip list node
- * @param iter
- * @return
- */
-void *tSkipListDestroyIter(SSkipListIterator *iter);
-
-/*
- * remove nodes of the pKey value.
- * If more than one node has the same value, all will be removed
- *
- * @Return
- * the count of removed nodes
- */
 uint32_t           tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
-
-/*
- * remove the specified node in parameters
- */
 void               tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
 
 #ifdef __cplusplus

src/util/src/tcompare.c

@@ -367,7 +367,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_BOOL:       DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
     case TSDB_DATA_TYPE_NCHAR: {
-      int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE);
+      tstr* t1 = (tstr*) f1;
+      tstr* t2 = (tstr*) f2;
+
+      if (t1->len != t2->len) {
+        return t1->len > t2->len? 1:-1;
+      }
+
+      int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE);
       if (ret == 0) {
         return ret;
       }

src/util/tests/skiplistTest.cpp

@@ -247,7 +247,7 @@ void skiplistPerformanceTest() {
   printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size);
   printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
 
-  assert(tSkipListGetSize(pSkipList) == size);
+  assert(SL_GET_SIZE(pSkipList) == size);
 
   //  printf("the level of skiplist is:\n");
   //
@@ -273,7 +273,7 @@ void skiplistPerformanceTest() {
 
   int64_t et = taosGetTimestampMs();
   printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
-  assert(tSkipListGetSize(pSkipList) == size);
+  assert(SL_GET_SIZE(pSkipList) == size);
 
   tSkipListDestroy(pSkipList);
   taosTFree(total);

src/vnode/src/vnodeMain.c

@@ -140,6 +140,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
   tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
   tsdbCfg.precision           = pVnodeCfg->cfg.precision;
   tsdbCfg.compression         = pVnodeCfg->cfg.compression;
+  tsdbCfg.update              = pVnodeCfg->cfg.update;
 
   char tsdbDir[TSDB_FILENAME_LEN] = {0};
   sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
@@ -468,21 +469,6 @@ void *vnodeAcquire(int32_t vgId) {
   return *ppVnode;
 }
 
-void *vnodeAcquireRqueue(int32_t vgId) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) return NULL;
-
-  int32_t code = vnodeCheckRead(pVnode);
-  if (code != TSDB_CODE_SUCCESS) {
-    terrno = code;
-    vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]);
-    vnodeRelease(pVnode);
-    return NULL;
-  }
-
-  return pVnode->rqueue;
-}
-
 void *vnodeGetWal(void *pVnode) {
   return ((SVnodeObj *)pVnode)->wal;
 }

src/vnode/src/vnodeRead.c

@@ -46,7 +46,7 @@ void vnodeInitReadFp(void) {
 //
 int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) {
   SVnodeObj *pVnode = (SVnodeObj *)param;
-  int msgType = pReadMsg->rpcMsg.msgType;
+  int32_t    msgType = pReadMsg->msgType;
 
   if (vnodeProcessReadMsgFp[msgType] == NULL) {
     vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
@@ -56,7 +56,7 @@ int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) {
   return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
 }
 
-int32_t vnodeCheckRead(void *param) {
+static int32_t vnodeCheckRead(void *param) {
   SVnodeObj *pVnode = param;
   if (pVnode->status != TAOS_VN_STATUS_READY) {
     vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
@@ -78,24 +78,58 @@ int32_t vnodeCheckRead(void *param) {
 
   return TSDB_CODE_SUCCESS;
 }
-static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
+
+int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) {
+  SVnodeObj *pVnode = vparam;
+
+  if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) {
     int32_t code = vnodeCheckRead(pVnode);
     if (code != TSDB_CODE_SUCCESS) return code;
+  }
 
-  SVReadMsg *pRead = (SVReadMsg *)taosAllocateQitem(sizeof(SVReadMsg));
+  int32_t size = sizeof(SVReadMsg) + contLen;
-  pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+  SVReadMsg *pRead = taosAllocateQitem(size);
-  pRead->pCont = qhandle;
+  if (pRead == NULL) {
-  pRead->contLen = 0;
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  pRead->rpcMsg.ahandle = ahandle;
+  }
+
+  if (rparam != NULL) {
+    SRpcMsg *pRpcMsg = rparam;
+    pRead->rpcHandle = pRpcMsg->handle;
+    pRead->rpcAhandle = pRpcMsg->ahandle;
+    pRead->msgType = pRpcMsg->msgType;
+    pRead->code = pRpcMsg->code;
+  }
+
+  if (contLen != 0) {
+    pRead->contLen = contLen;
+    memcpy(pRead->pCont, pCont, contLen);
+  } else {
+    pRead->qhandle = pCont;
+  }
+
+  pRead->qtype = qtype;
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
+  vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
 
-  vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead);
+  taosWriteQitem(pVnode->rqueue, qtype, pRead);
-  taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
-
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+  rpcMsg.ahandle = ahandle;
+
+  int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
+  if (code == TSDB_CODE_SUCCESS) {
+    vDebug("QInfo:%p add to vread queue for exec query", *qhandle);
+  }
+
+  return code;
+}
+
 /**
  *
  * @param pRet         response message object
@@ -155,18 +189,18 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   memset(pRet, 0, sizeof(SRspRet));
 
   // qHandle needs to be freed correctly
-  if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+  if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
     SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
     killQueryMsg->free = htons(killQueryMsg->free);
     killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
 
-    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
+    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle);
-    assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
+    assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1);
 
     void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
     if (qhandle == NULL || *qhandle == NULL) {
       vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
-            pReadMsg->rpcMsg.handle);
+            pReadMsg->rpcHandle);
     } else {
       assert(*qhandle == (void *)killQueryMsg->qhandle);
 
@@ -208,9 +242,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
       }
 
       if (handle != NULL &&
-          vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+          vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
         vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
-               pReadMsg->rpcMsg.handle);
+               pReadMsg->rpcHandle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
         qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
         return pRsp->code;
@@ -221,7 +255,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
 
     if (handle != NULL) {
       vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
-      code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle);
+      code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle);
       if (code != TSDB_CODE_SUCCESS) {
         pRsp->code = code;
         qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -230,7 +264,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
     }
   } else {
     assert(pCont != NULL);
-    void **qhandle = (void **)pCont;
+    void **qhandle = (void **)pReadMsg->qhandle;
 
     vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
 
@@ -242,14 +276,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
     // build query rsp, the retrieve request has reached here already
     if (buildRes) {
       // update the connection info according to the retrieve connection
-      pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle);
+      pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle);
-      assert(pReadMsg->rpcMsg.handle != NULL);
+      assert(pReadMsg->rpcHandle != NULL);
 
       vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
-             pReadMsg->rpcMsg.handle);
+             pReadMsg->rpcHandle);
 
       // set the real rsp error code
-      pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle);
+      pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle);
 
       // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
       code = TSDB_CODE_QRY_HAS_RSP;
@@ -283,7 +317,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
 
   vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
-         pRetrieve->free, pReadMsg->rpcMsg.handle);
+         pRetrieve->free, pReadMsg->rpcHandle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
@@ -298,7 +332,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   }
 
   if (code != TSDB_CODE_SUCCESS) {
-    vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
+    vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
     vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
@@ -314,9 +348,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   }
 
   // register the qhandle to connect to quit query immediate if connection is broken
-  if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+  if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
+    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle);
-           pReadMsg->rpcMsg.handle);
     code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -326,7 +359,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   bool freeHandle = true;
   bool buildRes = false;
 
-  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
+  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle);
   if (code != TSDB_CODE_SUCCESS) {
     // TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@@ -337,7 +370,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
     assert(buildRes == true);
 #if _NON_BLOCKING_RETRIEVE
     if (!buildRes) {
-      assert(pReadMsg->rpcMsg.handle != NULL);
+      assert(pReadMsg->rpcHandle != NULL);
 
       qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
       return TSDB_CODE_QRY_NOT_READY;
@@ -345,7 +378,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
 #endif
 
     // ahandle is the sqlObj pointer
-    code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle);
+    code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle);
   }
 
   // If qhandle is not added into vread queue, the query should be completed already or paused with error.

src/vnode/src/vnodeWrite.c

@@ -97,7 +97,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
   return syncCode;
 }
 
-int32_t vnodeCheckWrite(void *param) {
+static int32_t vnodeCheckWrite(void *param) {
   SVnodeObj *pVnode = param;
   if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
     vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);

tests/examples/c/apitest.c

@@ -9,26 +9,40 @@
 
 
 static void prepare_data(TAOS* taos) {
-  taos_query(taos, "drop database if exists test;");
+  TAOS_RES *result;
+  result = taos_query(taos, "drop database if exists test;");
+  taos_free_result(result);
   usleep(100000);
-  taos_query(taos, "create database test;");
+  result = taos_query(taos, "create database test;");
+  taos_free_result(result);
   usleep(100000);
   taos_select_db(taos, "test");
 
-  taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
+  result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
+  taos_free_result(result);
 
-  taos_query(taos, "create table t0 using meters tags(0);");
+  result = taos_query(taos, "create table t0 using meters tags(0);");
-  taos_query(taos, "create table t1 using meters tags(1);");
+  taos_free_result(result);
-  taos_query(taos, "create table t2 using meters tags(2);");
+  result = taos_query(taos, "create table t1 using meters tags(1);");
-  taos_query(taos, "create table t3 using meters tags(3);");
+  taos_free_result(result);
-  taos_query(taos, "create table t4 using meters tags(4);");
+  result = taos_query(taos, "create table t2 using meters tags(2);");
-  taos_query(taos, "create table t5 using meters tags(5);");
+  taos_free_result(result);
-  taos_query(taos, "create table t6 using meters tags(6);");
+  result = taos_query(taos, "create table t3 using meters tags(3);");
-  taos_query(taos, "create table t7 using meters tags(7);");
+  taos_free_result(result);
-  taos_query(taos, "create table t8 using meters tags(8);");
+  result = taos_query(taos, "create table t4 using meters tags(4);");
-  taos_query(taos, "create table t9 using meters tags(9);");
+  taos_free_result(result);
+  result = taos_query(taos, "create table t5 using meters tags(5);");
+  taos_free_result(result);
+  result = taos_query(taos, "create table t6 using meters tags(6);");
+  taos_free_result(result);
+  result = taos_query(taos, "create table t7 using meters tags(7);");
+  taos_free_result(result);
+  result = taos_query(taos, "create table t8 using meters tags(8);");
+  taos_free_result(result);
+  result = taos_query(taos, "create table t9 using meters tags(9);");
+  taos_free_result(result);
 
-  TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
+  result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
     " ('2020-01-01 00:01:00.000', 0)"
     " ('2020-01-01 00:02:00.000', 0)"
     " t1 values('2020-01-01 00:00:00.000', 0)"
@@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) {
     " t7 values('2020-01-01 00:01:02.000', 0)"
     " t8 values('2020-01-01 00:01:02.000', 0)"
     " t9 values('2020-01-01 00:01:02.000', 0)");
-  int affected = taos_affected_rows(res);
+  int affected = taos_affected_rows(result);
   if (affected != 18) {
     printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected);
   }
+  taos_free_result(result);
   // super tables subscription
   usleep(1000000);
 }
@@ -135,6 +150,7 @@ static void verify_query(TAOS* taos) {
 
   res = taos_query(taos, "select * from meters");
   taos_stop_query(res);
+  taos_free_result(res);
 }
 
 
@@ -153,23 +169,30 @@ static void verify_subscribe(TAOS* taos) {
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 0);
 
-  taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
+  TAOS_RES *result;
-  taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
+  result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
+  taos_free_result(result);
+  result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
+  taos_free_result(result);
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 2);
 
-  taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
+  result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
-  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
+  taos_free_result(result);
+  result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
+  taos_free_result(result);
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 2);
 
-  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
+  result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
+  taos_free_result(result);
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 1);
 
   // keep progress information and restart subscription
   taos_unsubscribe(tsub, 1);
-  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
+  result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
+  taos_free_result(result);
   tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0);
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 24);
@@ -196,7 +219,8 @@ static void verify_subscribe(TAOS* taos) {
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 0);
 
-  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
+  result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
+  taos_free_result(result);
   res = taos_consume(tsub);
   check_row_count(__LINE__, res, 1);
 
@@ -205,7 +229,8 @@ static void verify_subscribe(TAOS* taos) {
   int blockFetch = 0;
   tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000);
   usleep(2000000);
-  taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
+  result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
+  taos_free_result(result);
   usleep(2000000);
   taos_unsubscribe(tsub, 0);
 }
@@ -213,8 +238,9 @@ static void verify_subscribe(TAOS* taos) {
 
 void verify_prepare(TAOS* taos) {
   TAOS_RES* result = taos_query(taos, "drop database if exists test;");
+  taos_free_result(result);
   usleep(100000);
-  taos_query(taos, "create database test;");
+  result = taos_query(taos, "create database test;");
 
   int code = taos_errno(result);
   if (code != 0) {
@@ -429,7 +455,8 @@ void verify_stream(TAOS* taos) {
     NULL);
   printf("waiting for stream data\n"); 
   usleep(100000);
-  taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
+  TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
+  taos_free_result(result);
   usleep(200000000);
   taos_close_stream(strm);
 }

tests/examples/c/asyncdemo.c

@@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
 void taos_select_call_back(void *param, TAOS_RES *tres, int code);
 void taos_error(TAOS *taos);
 
+static void queryDB(TAOS *taos, char *command) {
+  int i;
+  TAOS_RES *pSql = NULL;
+  int32_t   code = -1;
+
+  for (i = 0; i < 5; i++) {
+    if (NULL != pSql) {
+      taos_free_result(pSql);
+      pSql = NULL;
+    }
+    
+    pSql = taos_query(taos, command);
+    code = taos_errno(pSql);
+    if (0 == code) {
+      break;
+    }    
+  }
+
+  if (code != 0) {
+    fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
+    taos_free_result(pSql);
+    taos_close(taos);
+    exit(EXIT_FAILURE);
+  }
+
+  taos_free_result(pSql);
+}
+
 int main(int argc, char *argv[])
 {
   TAOS   *taos;
@@ -78,16 +106,14 @@ int main(int argc, char *argv[])
 
   printf("success to connect to server\n");
 
-  sprintf(sql, "drop database %s", db);
+  sprintf(sql, "drop database if exists %s", db);
-  taos_query(taos, sql);
+  queryDB(taos, sql);
 
   sprintf(sql, "create database %s", db);
-  if (taos_query(taos, sql) != 0)
+  queryDB(taos, sql);
-    taos_error(taos);
 
   sprintf(sql, "use %s", db);
-  if (taos_query(taos, sql) != 0)
+  queryDB(taos, sql);
-    taos_error(taos);
 
   strcpy(prefix, "asytbl_");
   for (i = 0; i < numOfTables; ++i) {
@@ -95,8 +121,7 @@ int main(int argc, char *argv[])
     tableList[i].taos = taos;
     sprintf(tableList[i].name, "%s%d", prefix, i);
     sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i);
-    if (taos_query(taos, sql) != 0)
+    queryDB(taos, sql);
-      taos_error(taos);
   }  
 
   gettimeofday(&systemTime, NULL);

tests/examples/c/demo.c

@@ -22,6 +22,34 @@
 #include <inttypes.h>
 #include <taos.h>  // TAOS header file
 
+static void queryDB(TAOS *taos, char *command) {
+  int i;
+  TAOS_RES *pSql = NULL;
+  int32_t   code = -1;
+
+  for (i = 0; i < 5; i++) {
+    if (NULL != pSql) {
+      taos_free_result(pSql);
+      pSql = NULL;
+    }
+    
+    pSql = taos_query(taos, command);
+    code = taos_errno(pSql);
+    if (0 == code) {
+      break;
+    }    
+  }
+
+  if (code != 0) {
+    fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
+    taos_free_result(pSql);
+    taos_close(taos);
+    exit(EXIT_FAILURE);
+  }
+
+  taos_free_result(pSql);
+}
+
 int main(int argc, char *argv[]) {
   TAOS *    taos;
   char      qstr[1024];
@@ -44,22 +72,26 @@ int main(int argc, char *argv[]) {
   printf("success to connect to server\n");
 
 
-  taos_query(taos, "drop database demo");
+  //taos_query(taos, "drop database demo");
+  queryDB(taos, "drop database if exists demo");
 
-  result = taos_query(taos, "create database demo");
+  //result = taos_query(taos, "create database demo");
-  if (result == NULL) {
+  //if (result == NULL) {
-    printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/);
+  //  printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/);
-    exit(1);
+  //  exit(1);
-  }
+  //}
+  queryDB(taos, "create database demo");
   printf("success to create database\n");
 
-  taos_query(taos, "use demo");
+  //taos_query(taos, "use demo");
+  queryDB(taos, "use demo");
 
   // create table
-  if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
+  //if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
-    printf("failed to create table, reason:%s\n", taos_errstr(result));
+  //  printf("failed to create table, reason:%s\n", taos_errstr(result));
-    exit(1);
+  //  exit(1);
-  }
+  //}
+  queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))");
   printf("success to create table\n");
 
   // sleep for one second to make sure table is created on data node
@@ -80,8 +112,10 @@ int main(int argc, char *argv[]) {
       printf("insert row: %i\n", i);
     } else {
       printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/);
+      taos_free_result(result);
       exit(1);
     }
+    taos_free_result(result);
 
     //sleep(1);
   }
@@ -92,6 +126,7 @@ int main(int argc, char *argv[]) {
   result = taos_query(taos, qstr);
   if (result == NULL || taos_errno(result) != 0) {
     printf("failed to select, reason:%s\n", taos_errstr(result));    
+    taos_free_result(result);
     exit(1);
   }
 

tests/pytest/alter/db_update_options.py

@@ -0,0 +1,71 @@
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+import subprocess
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+    def run(self):
+        tdLog.debug("check database")
+        tdSql.prepare()
+
+        # check default update value 
+        sql = "create database if not exists db"
+        tdSql.execute(sql)
+        tdSql.query('show databases')
+        tdSql.checkRows(1)
+        tdSql.checkData(0,16,0)
+
+        sql = "alter database db update 1"
+
+        # check update value 
+        tdSql.execute(sql)
+        tdSql.query('show databases')
+        tdSql.checkRows(1)
+        tdSql.checkData(0,16,1)
+
+
+        sql = "alter database db update 0"
+        tdSql.execute(sql)
+        tdSql.query('show databases')
+        tdSql.checkRows(1)
+        tdSql.checkData(0,16,0)
+
+        sql = "alter database db update -1"
+        tdSql.error(sql)
+
+        sql = "alter database db update 100"
+        tdSql.error(sql)
+
+        tdSql.query('show databases')
+        tdSql.checkRows(1)
+        tdSql.checkData(0,16,0)
+
+        tdSql.execute('drop database db') 
+        tdSql.error('create database db update 100')
+        tdSql.error('create database db update -1')
+
+        tdSql.execute('create database db update 1')
+
+        tdSql.query('show databases')
+        tdSql.checkRows(1)
+        tdSql.checkData(0,16,1)
+
+        tdSql.execute('drop database db') 
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/crash_gen/crash_gen_main.py

@@ -38,9 +38,9 @@ import resource
 from guppy import hpy
 import gc
 
-from .service_manager import ServiceManager, TdeInstance
+from crash_gen.service_manager import ServiceManager, TdeInstance
-from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress
+from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress
-from .db import DbConn, MyTDSql, DbConnNative, DbManager
+from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager
 
 import taos
 import requests
@@ -243,7 +243,7 @@ class WorkerThread:
 
 
 class ThreadCoordinator:
-    WORKER_THREAD_TIMEOUT = 180 # one minute
+    WORKER_THREAD_TIMEOUT = 120  # Normal: 120
 
     def __init__(self, pool: ThreadPool, dbManager: DbManager):
         self._curStep = -1  # first step is 0
@@ -388,9 +388,9 @@ class ThreadCoordinator:
                 self._syncAtBarrier() # For now just cross the barrier
                 Progress.emit(Progress.END_THREAD_STEP)
             except threading.BrokenBarrierError as err:
-                Logging.info("Main loop aborted, caused by worker thread time-out")
+                Logging.info("Main loop aborted, caused by worker thread(s) time-out")
                 self._execStats.registerFailure("Aborted due to worker thread timeout")
-                print("\n\nWorker Thread time-out detected, important thread info:")
+                print("\n\nWorker Thread time-out detected, TAOS related threads are:")
                 ts = ThreadStacks()
                 ts.print(filterInternal=True)
                 workerTimeout = True
@@ -435,7 +435,7 @@ class ThreadCoordinator:
         Logging.debug("\r\n\n--> Main thread ready to finish up...")
         Logging.debug("Main thread joining all threads")
         self._pool.joinAll()  # Get all threads to finish
-        Logging.info("\nAll worker threads finished")
+        Logging.info(". . . All worker threads finished") # No CR/LF before
         self._execStats.endExec()
 
     def cleanup(self): # free resources
@@ -1072,17 +1072,18 @@ class Database:
         t3 = datetime.datetime(2012, 1, 1)  # default "keep" is 10 years
         t4 = datetime.datetime.fromtimestamp(
             t3.timestamp() + elSec2)  # see explanation above
-        Logging.info("Setting up TICKS to start from: {}".format(t4))
+        Logging.debug("Setting up TICKS to start from: {}".format(t4))
         return t4
 
     @classmethod
     def getNextTick(cls):        
         with cls._clsLock:  # prevent duplicate tick
-            if cls._lastLaggingTick==0:
+            if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized
                 # 10k at 1/20 chance, should be enough to avoid overlaps
-                cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000)                 
+                tick = cls.setupLastTick()
-            if cls._lastTick==0: # should be quite a bit into the future
+                cls._lastTick = tick
-                cls._lastTick = cls.setupLastTick()  
+                cls._lastLaggingTick = tick + datetime.timedelta(0, -10000)                 
+                # if : # should be quite a bit into the future
 
             if Dice.throw(20) == 0:  # 1 in 20 chance, return lagging tick
                 cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds
@@ -1177,6 +1178,8 @@ class Task():
         instead. But a task is always associated with a DB
     '''
     taskSn = 100
+    _lock = threading.Lock()
+    _tableLocks: Dict[str, threading.Lock] = {}
 
     @classmethod
     def allocTaskNum(cls):
@@ -1198,6 +1201,8 @@ class Task():
         self._execStats = execStats
         self._db = db # A task is always associated/for a specific DB
 
+        
+
     def isSuccess(self):
         return self._err is None
 
@@ -1237,6 +1242,7 @@ class Task():
                 0x0B,  # Unable to establish connection, more details in TD-1648
                 0x200, # invalid SQL， TODO: re-examine with TD-934
                 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776
+                0x213, # "Disconnected from service", result of "kill connection ???"
                 0x217, # "db not selected", client side defined error code
                 # 0x218, # "Table does not exist" client side defined error code
                 0x360, # Table already exists
@@ -1318,7 +1324,7 @@ class Task():
                 self._err = err
                 self._aborted = True
         except Exception as e:
-            self.logInfo("Non-TAOS exception encountered")
+            Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__))
             self._err = e
             self._aborted = True
             traceback.print_exc()
@@ -1351,6 +1357,24 @@ class Task():
     def getQueryResult(self, wt: WorkerThread):  # execute an SQL on the worker thread
         return wt.getQueryResult()
 
+    def lockTable(self, ftName): # full table name
+        # print(" <<" + ftName + '_', end="", flush=True)
+        with Task._lock:
+            if not ftName in Task._tableLocks:
+                Task._tableLocks[ftName] = threading.Lock()
+        
+        Task._tableLocks[ftName].acquire()
+
+    def unlockTable(self, ftName):
+        # print('_' + ftName + ">> ", end="", flush=True)
+        with Task._lock:
+            if not ftName in self._tableLocks:
+                raise RuntimeError("Corrupt state, no such lock")
+            lock = Task._tableLocks[ftName]
+            if not lock.locked():
+                raise RuntimeError("Corrupte state, already unlocked")
+        lock.release()
+
 
 class ExecutionStats:
     def __init__(self):
@@ -1461,7 +1485,7 @@ class StateTransitionTask(Task):
 
     _baseTableNumber = None
 
-    _endState = None
+    _endState = None # TODO: no longter used?
 
     @classmethod
     def getInfo(cls):  # each sub class should supply their own information
@@ -1486,7 +1510,7 @@ class StateTransitionTask(Task):
 
     @classmethod
     def getRegTableName(cls, i):
-        if ( StateTransitionTask._baseTableNumber is None):
+        if ( StateTransitionTask._baseTableNumber is None): # Set it one time
             StateTransitionTask._baseTableNumber = Dice.throw(
                 999) if gConfig.dynamic_db_table_names else 0
         return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i)
@@ -1544,8 +1568,11 @@ class TaskCreateSuperTable(StateTransitionTask):
 
         sTable = self._db.getFixedSuperTable() # type: TdSuperTable
         # wt.execSql("use db")    # should always be in place
+
         sTable.create(wt.getDbConn(), self._db.getName(), 
-            {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'})
+            {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'},
+            dropIfExists = True
+            )
         # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
         # No need to create the regular tables, INSERT will do that
         # automatically
@@ -1558,13 +1585,40 @@ class TdSuperTable:
     def getName(self):
         return self._stName
 
+    def drop(self, dbc, dbName, skipCheck = False):
+        if self.exists(dbc, dbName) : # if myself exists
+            fullTableName = dbName + '.' + self._stName                
+            dbc.execute("DROP TABLE {}".format(fullTableName))
+        else:
+            if not skipCheck:
+                raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName))
+
+    def exists(self, dbc, dbName):
+        dbc.execute("USE " + dbName)
+        return dbc.existsSuperTable(self._stName)
+
     # TODO: odd semantic, create() method is usually static?
-    def create(self, dbc, dbName, cols: dict, tags: dict):
+    def create(self, dbc, dbName, cols: dict, tags: dict,
+        dropIfExists = False
+        ):
+
         '''Creating a super table'''
-        sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format(
+        dbc.execute("USE " + dbName)
-            dbName,
+        fullTableName = dbName + '.' + self._stName       
-            self._stName,
+        if dbc.existsSuperTable(self._stName):
-            ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]),
+            if dropIfExists: 
+                dbc.execute("DROP TABLE {}".format(fullTableName))
+            else: # error
+                raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName))
+                 
+        # Now let's create
+        sql = "CREATE TABLE {} ({})".format(
+            fullTableName,
+            ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]))
+        if tags is None :
+            sql += " TAGS (dummy int) "
+        else:
+            sql += " TAGS ({})".format(
                 ",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
             )
         dbc.execute(sql)        
@@ -1583,14 +1637,25 @@ class TdSuperTable:
     def hasRegTables(self, dbc: DbConn, dbName: str):
         return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0
 
-    def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str):
+    def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str):
         sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName)
         if dbc.query(sql) >= 1 : # reg table exists already
             return
-        sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format(
+
-            dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
+        # acquire a lock first, so as to be able to *verify*. More details in TD-1471
+        fullTableName = dbName + '.' + regTableName      
+        if task is not None:  # optional lock
+            task.lockTable(fullTableName)
+        Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table
+        # print("(" + fullTableName[-3:] + ")", end="", flush=True)  
+        try:
+            sql = "CREATE TABLE {} USING {}.{} tags ({})".format(
+                fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
             )
             dbc.execute(sql)
+        finally:
+            if task is not None:
+                task.unlockTable(fullTableName) # no matter what
 
     def _getTagStrForSql(self, dbc, dbName: str) :
         tags = self._getTags(dbc, dbName)
@@ -1809,7 +1874,7 @@ class TaskRestartService(StateTransitionTask):
 
         with self._classLock:
             if self._isRunning:
-                print("Skipping restart task, another running already")
+                Logging.info("Skipping restart task, another running already")
                 return
             self._isRunning = True
 
@@ -1847,24 +1912,21 @@ class TaskAddData(StateTransitionTask):
     def canBeginFrom(cls, state: AnyState):
         return state.canAddData()
 
-    def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
+    def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor): 
-        # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
+        numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS        
-        db = self._db
+        fullTableName = db.getName() + '.' + regTableName
-        dbc = wt.getDbConn()
-        tblSeq = list(range(
-                self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))
-        random.shuffle(tblSeq)
-        for i in tblSeq:
-            if (i in self.activeTable):  # wow already active
-                print("x", end="", flush=True) # concurrent insertion
-            else:
-                self.activeTable.add(i)  # marking it active
 
-            sTable = db.getFixedSuperTable()
+        sql = "insert into {} values ".format(fullTableName)
-            regTableName = self.getRegTableName(i)  # "db.reg_table_{}".format(i)
+        for j in range(numRecords):  # number of records per table
-            sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName)  # Ensure the table exists           
+            nextInt = db.getNextInt()
+            nextTick = db.getNextTick()
+            sql += "('{}', {});".format(nextTick, nextInt)
+        dbc.execute(sql)
 
-            for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS):  # number of records per table
+    def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches
+        numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS        
+
+        for j in range(numRecords):  # number of records per table
             nextInt = db.getNextInt()
             nextTick = db.getNextTick()
             if gConfig.record_ops:
@@ -1872,27 +1934,29 @@ class TaskAddData(StateTransitionTask):
                 self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
                 self.fAddLogReady.flush()
                 os.fsync(self.fAddLogReady)
-                sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {})
+                
-                    db.getName(),
+            # TODO: too ugly trying to lock the table reliably, refactor...
-                    regTableName,
+            fullTableName = db.getName() + '.' + regTableName
+            if gConfig.verify_data:
+                self.lockTable(fullTableName) 
+                # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written
+
+            try:
+                sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
+                    fullTableName,
                     # ds.getFixedSuperTableName(),
                     # ds.getNextBinary(), ds.getNextFloat(),
                     nextTick, nextInt)
                 dbc.execute(sql)
-                # Successfully wrote the data into the DB, let's record it
+            except: # Any exception at all
-                # somehow
+                if gConfig.verify_data:
-                te.recordDataMark(nextInt)
+                    self.unlockTable(fullTableName)     
-                if gConfig.record_ops:
+                raise
-                    self.fAddLogDone.write(
-                        "Wrote {} to {}\n".format(
-                            nextInt, regTableName))
-                    self.fAddLogDone.flush()
-                    os.fsync(self.fAddLogDone)
 
             # Now read it back and verify, we might encounter an error if table is dropped
             if gConfig.verify_data: # only if command line asks for it
                 try:
-                        readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts= '{}'".
+                    readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'".
                         format(db.getName(), regTableName, nextTick))
                     if readBack != nextInt :
                         raise taos.error.ProgrammingError(
@@ -1905,16 +1969,52 @@ class TaskAddData(StateTransitionTask):
                             "Failed to read back same data for tick: {}, wrote: {}, read: {}"
                             .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"),
                             errno)
-                        # Re-throw no matter what
+                    elif errno in [0x218, 0x362]: # table doesn't exist
+                        # do nothing
+                        dummy = 0
+                    else:
+                        # Re-throw otherwise
                         raise
+                finally:
+                    self.unlockTable(fullTableName) # Unlock the table no matter what
 
+            # Successfully wrote the data into the DB, let's record it somehow
+            te.recordDataMark(nextInt)
+
+            if gConfig.record_ops:
+                self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName))
+                self.fAddLogDone.flush()
+                os.fsync(self.fAddLogDone)
+
+    def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
+        # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
+        db = self._db
+        dbc = wt.getDbConn()
+        numTables  = self.LARGE_NUMBER_OF_TABLES  if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES
+        numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
+        tblSeq = list(range(numTables ))
+        random.shuffle(tblSeq) # now we have random sequence
+        for i in tblSeq:
+            if (i in self.activeTable):  # wow already active
+                print("x", end="", flush=True) # concurrent insertion
+            else:
+                self.activeTable.add(i)  # marking it active
+            
+            sTable = db.getFixedSuperTable()
+            regTableName = self.getRegTableName(i)  # "db.reg_table_{}".format(i)            
+            fullTableName = db.getName() + '.' + regTableName
+            # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked"
+            sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName)  # Ensure the table exists           
+            # self._unlockTable(fullTableName)
+           
+            if Dice.throw(1) == 0: # 1 in 2 chance
+                self._addData(db, dbc, regTableName, te)
+            else:
+                self._addDataInBatch(db, dbc, regTableName, te)
 
             self.activeTable.discard(i)  # not raising an error, unlike remove
 
 
-
-
-
 class ThreadStacks: # stack info for all threads
     def __init__(self):
         self._allStacks = {}
@@ -1936,17 +2036,18 @@ class ThreadStacks: # stack info for all threads
                     '__init__']: # the thread that extracted the stack
                     continue # ignore
             # Now print
-            print("\n<----- Thread Info for ID: {}".format(thNid))
+            print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid))
+            stackFrame = 0
             for frame in stack:
                 # print(frame)
-                print("File {filename}, line {lineno}, in {name}".format(
+                print("[{sf}] File {filename}, line {lineno}, in {name}".format(
-                    filename=frame.filename, lineno=frame.lineno, name=frame.name))
+                    sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
                 print("    {}".format(frame.line))
-            print("-----> End of Thread Info\n")
+            print("-----> End of Thread Info ----->\n")
 
 class ClientManager:
     def __init__(self):
-        print("Starting service manager")
+        Logging.info("Starting service manager")
         # signal.signal(signal.SIGTERM, self.sigIntHandler)
         # signal.signal(signal.SIGINT, self.sigIntHandler)
 
@@ -2048,7 +2149,7 @@ class ClientManager:
         thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps)
         self.tc = ThreadCoordinator(thPool, dbManager)
         
-        print("Starting client instance to: {}".format(tInst))
+        Logging.info("Starting client instance: {}".format(tInst))
         self.tc.run()
         # print("exec stats: {}".format(self.tc.getExecStats()))
         # print("TC failed = {}".format(self.tc.isFailed()))

tests/pytest/crash_gen/db.py

@@ -95,6 +95,11 @@ class DbConn:
         # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName)))
         return dbName in dbs # TODO: super weird type mangling seen, once here
 
+    def existsSuperTable(self, stName):
+        self.query("show stables")
+        sts = [v[0] for v in self.getQueryResult()]
+        return stName in sts
+
     def hasTables(self):
         return self.query("show tables") > 0
 
@@ -240,6 +245,7 @@ class MyTDSql:
 
     def _execInternal(self, sql):
         startTime = time.time() 
+        # Logging.debug("Executing SQL: " + sql)
         ret = self._cursor.execute(sql)
         # print("\nSQL success: {}".format(sql))
         queryTime =  time.time() - startTime

tests/pytest/crash_gen/misc.py

@@ -27,7 +27,7 @@ class LoggingFilter(logging.Filter):
 
 class MyLoggingAdapter(logging.LoggerAdapter):
     def process(self, msg, kwargs):
-        return "[{}] {}".format(threading.get_ident() % 10000, msg), kwargs
+        return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs
         # return '[%s] %s' % (self.extra['connid'], msg), kwargs
 
 
@@ -51,7 +51,7 @@ class Logging:
         _logger.addHandler(ch)
 
         # Logging adapter, to be used as a logger
-        print("setting logger variable")
+        # print("setting logger variable")
         # global logger
         cls.logger = MyLoggingAdapter(_logger, [])
 
@@ -166,6 +166,9 @@ class Progress:
     SERVICE_RECONNECT_START     = 4
     SERVICE_RECONNECT_SUCCESS   = 5
     SERVICE_RECONNECT_FAILURE   = 6
+    SERVICE_START_NAP           = 7
+    CREATE_TABLE_ATTEMPT        = 8
+
     tokens = {
         STEP_BOUNDARY:      '.',
         BEGIN_THREAD_STEP:  '[',
@@ -174,6 +177,8 @@ class Progress:
         SERVICE_RECONNECT_START:    '<r.',
         SERVICE_RECONNECT_SUCCESS:  '.r>',
         SERVICE_RECONNECT_FAILURE:  '.xr>',
+        SERVICE_START_NAP:           '_zz',
+        CREATE_TABLE_ATTEMPT: '_c',
     }
 
     @classmethod

tests/pytest/crash_gen/service_manager.py

@@ -47,6 +47,17 @@ class TdeInstance():
                 .format(selfPath, projPath))
         return buildPath
 
+    @classmethod
+    def prepareGcovEnv(cls, env):
+        # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html
+        bPath = cls._getBuildPath() # build PATH
+        numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3
+        numSegments = numSegments - 1 # DEBUG only
+        env['GCOV_PREFIX'] = bPath + '/svc_gcov'
+        env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings
+        Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format(
+            numSegments, env['GCOV_PREFIX'] ))
+
     def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030):
         self._buildDir  = self._getBuildPath()
         self._subdir    = '/' + subdir # TODO: tolerate "/"
@@ -217,6 +228,11 @@ class TdeSubProcess:
         #     raise CrashGenError("Empty instance not allowed in TdeSubProcess")
         # self._tInst = tInst # Default create at ServiceManagerThread
 
+    def __repr__(self):
+        if self.subProcess is None:
+            return '[TdeSubProc: Empty]'
+        return '[TdeSubProc: pid = {}]'.format(self.getPid())
+
     def getStdOut(self):
         return self.subProcess.stdout
 
@@ -236,16 +252,29 @@ class TdeSubProcess:
         if self.subProcess:  # already there
             raise RuntimeError("Corrupt process state")
 
+        # Prepare environment variables for coverage information
+        # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment
+        myEnv = os.environ.copy()
+        TdeInstance.prepareGcovEnv(myEnv)
+
+        # print(myEnv)
+        # print(myEnv.items())
+        # print("Starting TDengine via Shell: {}".format(cmdLineStr))
+
+        useShell = True    
         self.subProcess = subprocess.Popen(
-            cmdLine,
+            ' '.join(cmdLine) if useShell else cmdLine,
-            shell=False,
+            shell=useShell,
             # svcCmdSingle, shell=True, # capture core dump?
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             # bufsize=1, # not supported in binary mode
-            close_fds=ON_POSIX
+            close_fds=ON_POSIX,
+            env=myEnv
             )  # had text=True, which interferred with reading EOF
 
+    STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process?
+
     def stop(self):
         """
         Stop a sub process, and try to return a meaningful return code.
@@ -267,7 +296,7 @@ class TdeSubProcess:
         SIGUSR2         12
         """
         if not self.subProcess:
-            print("Sub process already stopped")
+            Logging.error("Sub process already stopped")
             return  # -1
 
         retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N)
@@ -278,20 +307,25 @@ class TdeSubProcess:
             return retCode
 
         # process still alive, let's interrupt it
-        print("Terminate running process, send SIG_INT and wait...")
+        Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL))
         # sub process should end, then IPC queue should end, causing IO thread to end        
-        # sig = signal.SIGINT
+        topSubProc = psutil.Process(self.subProcess.pid)
-        sig = signal.SIGKILL
+        for child in topSubProc.children(recursive=True):  # or parent.children() for recursive=False
-        self.subProcess.send_signal(sig) # SIGNINT or SIGKILL
+            child.send_signal(self.STOP_SIGNAL)
+            time.sleep(0.2) # 200 ms
+        # topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell)
+
+        self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell)
         self.subProcess.wait(20)
         retCode = self.subProcess.returncode # should always be there
         # May throw subprocess.TimeoutExpired exception above, therefore
         # The process is guranteed to have ended by now
         self.subProcess = None        
         if retCode != 0: # != (- signal.SIGINT):
-            Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(sig, retCode))
+            Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(
+                self.STOP_SIGNAL, retCode))
         else:
-            Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(sig))
+            Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL))
         return - retCode
 
 class ServiceManager:
@@ -439,7 +473,7 @@ class ServiceManager:
                     
             time.sleep(self.PAUSE_BETWEEN_IPC_CHECK)  # pause, before next round
         # raise CrashGenError("dummy")
-        print("Service Manager Thread (with subprocess) ended, main thread exiting...")
+        Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...")
 
     def _getFirstInstance(self):
         return self._tInsts[0]
@@ -452,7 +486,7 @@ class ServiceManager:
             # Find if there's already a taosd service, and then kill it
             for proc in psutil.process_iter():
                 if proc.name() == 'taosd':
-                    print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt")
+                    Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt")
                     time.sleep(2.0)
                     proc.kill()
                 # print("Process: {}".format(proc.name()))
@@ -559,7 +593,8 @@ class ServiceManagerThread:
         for i in range(0, 100):
             time.sleep(1.0)
             # self.procIpcBatch() # don't pump message during start up
-            print("_zz_", end="", flush=True)
+            Progress.emit(Progress.SERVICE_START_NAP)
+            # print("_zz_", end="", flush=True)
             if self._status.isRunning():
                 Logging.info("[] TDengine service READY to process requests")
                 Logging.info("[] TAOS service started: {}".format(self))
@@ -595,12 +630,12 @@ class ServiceManagerThread:
 
     def stop(self):
         # can be called from both main thread or signal handler
-        print("Terminating TDengine service running as the sub process...")
+        Logging.info("Terminating TDengine service running as the sub process...")
         if self.getStatus().isStopped():
-            print("Service already stopped")
+            Logging.info("Service already stopped")
             return
         if self.getStatus().isStopping():
-            print("Service is already being stopped")
+            Logging.info("Service is already being stopped")
             return
         # Linux will send Control-C generated SIGINT to the TDengine process
         # already, ref:
@@ -616,10 +651,10 @@ class ServiceManagerThread:
             if retCode == signal.SIGSEGV : # SGV
                 Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)")
         except subprocess.TimeoutExpired as err:
-            print("Time out waiting for TDengine service process to exit")
+            Logging.info("Time out waiting for TDengine service process to exit")
         else:    
             if self._tdeSubProcess.isRunning():  # still running, should now never happen
-                print("FAILED to stop sub process, it is still running... pid = {}".format(
+                Logging.error("FAILED to stop sub process, it is still running... pid = {}".format(
                     self._tdeSubProcess.getPid()))
             else:
                 self._tdeSubProcess = None  # not running any more
@@ -683,9 +718,9 @@ class ServiceManagerThread:
                 return  # we are done with THIS BATCH
             else:  # got line, printing out
                 if forceOutput:
-                    Logging.info(line)
+                    Logging.info('[TAOSD] ' + line)
                 else:
-                    Logging.debug(line)
+                    Logging.debug('[TAOSD] ' + line)
         print(">", end="", flush=True)
 
     _ProgressBars = ["--", "//", "||", "\\\\"]
@@ -728,11 +763,11 @@ class ServiceManagerThread:
 
             # queue.put(line)
         # meaning sub process must have died
-        Logging.info("\nEnd of stream detected for TDengine STDOUT: {}".format(self))
+        Logging.info("EOF for TDengine STDOUT: {}".format(self))
         out.close()
 
     def svcErrorReader(self, err: IO, queue):
         for line in iter(err.readline, b''):
-            print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line))
+            Logging.info("TDengine STDERR: {}".format(line))
-        Logging.info("\nEnd of stream detected for TDengine STDERR: {}".format(self))
+        Logging.info("EOF for TDengine STDERR: {}".format(self))
         err.close()

tests/pytest/crash_gen_bootstrap.py

@@ -11,7 +11,7 @@
 ###################################################################
 
 import sys
-from crash_gen.crash_gen import MainExec
+from crash_gen.crash_gen_main import MainExec
 
 if __name__ == "__main__":
     

tests/pytest/query/queryLike.py

@@ -0,0 +1,45 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()         
+
+        tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))")
+        tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');")
+        tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'")
+        tdSql.checkRows(0)
+
+        tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))")
+        tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');")
+        tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'")        
+        tdSql.checkRows(1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/allow_update-0.py

@@ -0,0 +1,170 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+        self.numOfRecords = 10
+        self.ts = 1604295582000    
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.execute("use udb")
+
+    def run(self):
+        tdSql.prepare()
+        startTs = self.ts
+
+        print("==============step1")        
+        tdSql.execute("create database udb update 0")
+        tdSql.execute("use udb")
+        tdSql.execute("create table t (ts timestamp, a int)")
+        tdSql.execute("insert into t values (%d, 1)" % (startTs))
+        tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
+        tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 1)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 1)
+
+        print("==============step2")
+        tdSql.execute("insert into t values (%d, 2)" % (startTs))
+        tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
+        tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 1)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 1)
+
+        print("==============step3")
+        tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
+        tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
+        tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
+        tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(7)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 3)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 3)
+        tdSql.checkData(3, 0, 1)
+        tdSql.checkData(4, 0, 3)
+        tdSql.checkData(5, 0, 1)
+        tdSql.checkData(6, 0, 3)
+        
+        print("==============step4")
+        tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
+        tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
+        tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
+        tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(7)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 3)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 3)
+        tdSql.checkData(3, 0, 1)
+        tdSql.checkData(4, 0, 3)
+        tdSql.checkData(5, 0, 1)
+        tdSql.checkData(6, 0, 3)
+        
+        print("==============step5")
+        tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
+        tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(9)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 3)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 3)        
+        tdSql.checkData(3, 0, 5)
+        tdSql.checkData(4, 0, 1)
+        tdSql.checkData(5, 0, 5)        
+        tdSql.checkData(6, 0, 3)
+        tdSql.checkData(7, 0, 1)
+        tdSql.checkData(8, 0, 3)
+        
+        print("==============step6")
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(9)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 3)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 3)        
+        tdSql.checkData(3, 0, 5)
+        tdSql.checkData(4, 0, 1)
+        tdSql.checkData(5, 0, 5)        
+        tdSql.checkData(6, 0, 3)
+        tdSql.checkData(7, 0, 1)
+        tdSql.checkData(8, 0, 3)
+        
+        # restart taosd to commit, and check
+        self.restartTaosd();
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(9)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 3)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 3)        
+        tdSql.checkData(3, 0, 5)
+        tdSql.checkData(4, 0, 1)
+        tdSql.checkData(5, 0, 5)        
+        tdSql.checkData(6, 0, 3)
+        tdSql.checkData(7, 0, 1)
+        tdSql.checkData(8, 0, 3)        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/allow_update.py

@@ -0,0 +1,266 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+        self.numOfRecords = 10
+        self.ts = 1604295582000    
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.execute("use udb")
+
+    def run(self):
+        tdSql.prepare()
+        startTs = self.ts
+
+        tdSql.execute("create database udb update 1")
+        tdSql.execute("use udb")
+        tdSql.execute("create table t (ts timestamp, a int)")
+        
+        print("==============step1")
+        tdSql.execute("insert into t values (%d, 1)" % (startTs))
+        tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
+        tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 1)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 1)
+
+        print("==============step2")
+        tdSql.execute("insert into t values (%d, 2)" % (startTs))
+        tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
+        tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 2)
+        tdSql.checkData(1, 0, 2)
+        tdSql.checkData(2, 0, 2)
+
+        print("==============step3")
+        tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
+        tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
+        tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
+        tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(7)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 3)
+        tdSql.checkData(1, 0, 2)
+        tdSql.checkData(2, 0, 3)
+        tdSql.checkData(3, 0, 2)
+        tdSql.checkData(4, 0, 3)
+        tdSql.checkData(5, 0, 2)
+        tdSql.checkData(6, 0, 3)
+        
+        print("==============step4")
+        tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
+        tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
+        tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
+        tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(7)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 4)
+        tdSql.checkData(1, 0, 2)
+        tdSql.checkData(2, 0, 4)
+        tdSql.checkData(3, 0, 2)
+        tdSql.checkData(4, 0, 4)
+        tdSql.checkData(5, 0, 2)
+        tdSql.checkData(6, 0, 4)
+        
+        print("==============step5")
+        tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
+        tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(9)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 4)
+        tdSql.checkData(1, 0, 2)
+        tdSql.checkData(2, 0, 4)        
+        tdSql.checkData(3, 0, 5)
+        tdSql.checkData(4, 0, 2)
+        tdSql.checkData(5, 0, 5)        
+        tdSql.checkData(6, 0, 4)
+        tdSql.checkData(7, 0, 2)
+        tdSql.checkData(8, 0, 4)
+        
+        print("==============step6")
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
+        tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(9)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 6)
+        tdSql.checkData(1, 0, 6)
+        tdSql.checkData(2, 0, 6)        
+        tdSql.checkData(3, 0, 6)
+        tdSql.checkData(4, 0, 6)
+        tdSql.checkData(5, 0, 6)        
+        tdSql.checkData(6, 0, 6)
+        tdSql.checkData(7, 0, 6)
+        tdSql.checkData(8, 0, 6)
+        
+        # restart taosd to commit, and check
+        self.restartTaosd();
+        
+        tdSql.query("select * from t")
+        tdSql.checkRows(9)
+        
+        tdSql.query("select a from t")
+        tdSql.checkData(0, 0, 6)
+        tdSql.checkData(1, 0, 6)
+        tdSql.checkData(2, 0, 6)        
+        tdSql.checkData(3, 0, 6)
+        tdSql.checkData(4, 0, 6)
+        tdSql.checkData(5, 0, 6)        
+        tdSql.checkData(6, 0, 6)
+        tdSql.checkData(7, 0, 6)
+        tdSql.checkData(8, 0, 6)
+        
+        tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
+        
+        print("==============step7")
+        tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
+        tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
+        tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
+        
+        tdSql.query("select * from subt")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a,b,c,d from subt")
+        tdSql.checkData(0, 0, 1)
+        tdSql.checkData(1, 0, 1)
+        tdSql.checkData(2, 0, 1)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(1, 1, None)
+        tdSql.checkData(2, 1, None)
+        tdSql.checkData(0, 2, 'c-3')
+        tdSql.checkData(1, 2, 'c+0')
+        tdSql.checkData(2, 2, 'c+3')
+        tdSql.checkData(0, 3, None)
+        tdSql.checkData(1, 3, None)
+        tdSql.checkData(2, 3, None)
+
+        print("==============step8")
+        tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
+        tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
+        tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
+        
+        tdSql.query("select * from subt")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a,b,c,d from subt")
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(1, 0, None)
+        tdSql.checkData(2, 0, None)
+        tdSql.checkData(0, 1, 2.0)
+        tdSql.checkData(1, 1, 2.0)
+        tdSql.checkData(2, 1, 2.0)
+        tdSql.checkData(0, 2, None)
+        tdSql.checkData(1, 2, None)
+        tdSql.checkData(2, 2, None)
+        tdSql.checkData(0, 3, 1)
+        tdSql.checkData(1, 3, 1)
+        tdSql.checkData(2, 3, 0)
+        
+        # restart taosd to commit, and check
+        self.restartTaosd();
+        
+        tdSql.query("select * from subt")
+        tdSql.checkRows(3)
+        
+        tdSql.query("select a,b,c,d from subt")
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(1, 0, None)
+        tdSql.checkData(2, 0, None)
+        tdSql.checkData(0, 1, 2.0)
+        tdSql.checkData(1, 1, 2.0)
+        tdSql.checkData(2, 1, 2.0)
+        tdSql.checkData(0, 2, None)
+        tdSql.checkData(1, 2, None)
+        tdSql.checkData(2, 2, None)
+        tdSql.checkData(0, 3, 1)
+        tdSql.checkData(1, 3, 1)
+        tdSql.checkData(2, 3, 0)
+        
+        
+        
+        tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
+        
+        print("==============step9")        
+        insertRows = 20000
+        for i in range(0, insertRows):
+          tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
+        
+        tdSql.query("select * from ct")
+        tdSql.checkRows(insertRows)        
+        
+        for i in range(0, insertRows):
+          tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
+        
+        tdSql.query("select * from ct")
+        tdSql.checkRows(insertRows)
+        
+        tdSql.query("select a,b from ct limit 3")
+        tdSql.checkData(0, 0, insertRows+0)
+        tdSql.checkData(1, 0, insertRows+1)
+        tdSql.checkData(2, 0, insertRows+2)
+        
+        tdSql.checkData(0, 1, insertRows+0)
+        tdSql.checkData(1, 1, insertRows+1)
+        tdSql.checkData(2, 1, insertRows+2)
+        
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/append_commit_data-0.py

@@ -0,0 +1,84 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        print("==========step1")
+        print("create table && insert data")
+        s = 'reset query cache'
+        tdSql.execute(s)
+        s = 'drop database if exists db'
+        tdSql.execute(s)
+        s = 'create database db'
+        tdSql.execute(s)
+        s = 'use db'
+        tdSql.execute(s)
+        ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+        insertRows = 200
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(0, insertRows):
+          ret = tdSql.execute(
+              'insert into t1 values (%d , 1)' %
+              (t0+i))
+        print("==========step2")
+        print("restart to commit ")
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from db.t1")
+        tdSql.checkRows(insertRows)
+        for k in range(0,100):
+          tdLog.info("insert %d rows" % (insertRows))
+          for i in range (0,insertRows):
+            ret = tdSql.execute(
+                'insert into db.t1 values(%d,1)' %
+                (t0+k*200+i)
+            )
+          tdDnodes.stop(1)
+          tdDnodes.start(1)
+          tdSql.query("select * from db.t1")
+          tdSql.checkRows(insertRows+200*k)
+        print("==========step2")
+        print("insert into another table ")
+        s = 'use db'
+        tdSql.execute(s)
+        ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+        insertRows = 20000
+        for i in range(0, insertRows):
+          ret = tdSql.execute(
+              'insert into t2 values (%d, 1)' %
+              (t0+i))
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/append_commit_data.py

@@ -0,0 +1,84 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        print("==========step1")
+        print("create table && insert data")
+        s = 'reset query cache'
+        tdSql.execute(s)
+        s = 'drop database if exists db'
+        tdSql.execute(s)
+        s = 'create database db  update 1'
+        tdSql.execute(s)
+        s = 'use db'
+        tdSql.execute(s)
+        ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+        insertRows = 200
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(0, insertRows):
+          ret = tdSql.execute(
+              'insert into t1 values (%d , 1)' %
+              (t0+i))
+        print("==========step2")
+        print("restart to commit ")
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from db.t1")
+        tdSql.checkRows(insertRows)
+        for k in range(0,100):
+          tdLog.info("insert %d rows" % (insertRows))
+          for i in range (0,insertRows):
+            ret = tdSql.execute(
+                'insert into db.t1 values(%d,1)' %
+                (t0+k*200+i)
+            )
+          tdDnodes.stop(1)
+          tdDnodes.start(1)
+          tdSql.query("select * from db.t1")
+          tdSql.checkRows(insertRows+200*k)
+        print("==========step2")
+        print("insert into another table ")
+        s = 'use db'
+        tdSql.execute(s)
+        ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+        insertRows = 20000
+        for i in range(0, insertRows):
+          ret = tdSql.execute(
+              'insert into t2 values (%d, 1)' %
+              (t0+i))
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/append_commit_last-0.py

@@ -0,0 +1,90 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.ts = 1604298064000
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use db")
+
+    def run(self):            
+        tdSql.prepare()
+
+        print("==============step1")        
+        tdSql.execute("create table t1 (ts timestamp, a int)")
+
+        for i in range(10):
+            tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
+            self.restartTaosd()
+            tdSql.query("select * from t1")
+            tdSql.checkRows(i + 1)
+            tdSql.query("select sum(a) from t1")
+            tdSql.checkData(0, 0, i + 1)
+
+        print("==============step2")
+        tdSql.execute("create table t2 (ts timestamp, a int)")
+        tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1)
+
+        for i in range(1, 151):
+            tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(151)
+        tdSql.query("select sum(a) from t2")
+        tdSql.checkData(0, 0, 151)
+
+
+        print("==============step3")
+        tdSql.execute("create table t3 (ts timestamp, a int)")
+        tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(1)        
+        tdSql.checkData(0, 1, 1)
+
+        for i in range(8):
+            for j in range(1, 11):
+                tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(81)
+        tdSql.query("select sum(a) from t3")
+        tdSql.checkData(0, 0, 81)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/append_commit_last.py

@@ -0,0 +1,85 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.ts = 1604298064000
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use udb")
+
+    def run(self):            
+        tdSql.prepare()
+
+        print("==============step1")
+        tdSql.execute("create database udb update 1")        
+        tdSql.execute("use udb")
+        tdSql.execute("create table t1 (ts timestamp, a int)")
+
+        for i in range(10):
+            tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
+            self.restartTaosd()
+            tdSql.query("select * from t1")
+            tdSql.checkRows(i + 1)
+
+
+        print("==============step2")
+        tdSql.execute("create table t2 (ts timestamp, a int)")
+        tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(1)
+
+        for i in range(1, 151):
+            tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(151)
+
+
+        print("==============step3")
+        tdSql.execute("create table t3 (ts timestamp, a int)")
+        tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(1)
+
+        for i in range(8):
+            for j in range(1, 11):
+                tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(81)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/merge_commit_data-0.py

@@ -0,0 +1,351 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        print("==========step1")
+        print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
+        s = 'reset query cache'
+        tdSql.execute(s)
+        s = 'drop database if exists db'
+        tdSql.execute(s)
+        s = 'create database db days 30'
+        tdSql.execute(s)
+        s = 'use db'
+        tdSql.execute(s)
+        ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+        insertRows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(0, insertRows):
+          ret = tdSql.execute(
+            'insert into t1 values (%d , 1)' %
+            (t0 + i))
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t1")
+        tdSql.checkRows(insertRows)
+
+        for k in range(0,10):
+          for i in range (0,insertRows):
+            ret = tdSql.execute(
+              'insert into t1 values(%d,1)' %
+              (t0+i)
+            )
+          tdSql.query("select * from t1")
+          tdSql.checkRows(insertRows)
+          tdDnodes.stop(1)
+          tdDnodes.start(1)
+          tdSql.query("select * from t1")
+          tdSql.checkRows(insertRows)
+        print("==========step2")
+        print("PREPEND DATA ")
+        ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t2 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows)
+        for i in range(-100,0):
+          ret = tdSql.execute(
+                'insert into t2 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows+100)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows+100)
+        print("==========step3")
+        print("PREPEND MASSIVE DATA ")
+        ret = tdSql.execute('create table t3 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t3 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows)
+        for i in range(-6000,0):
+          ret = tdSql.execute(
+                'insert into t3 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows+6000)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows+6000)
+        print("==========step4")
+        print("APPEND DATA")
+        ret = tdSql.execute('create table t4 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t4 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows)
+        for i in range(0,100):
+          ret = tdSql.execute(
+                'insert into t4 values (%d , 1)' %
+                (t0+200+i))
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows+100)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows+100)
+        print("==========step5")
+        print("APPEND DATA")
+        ret = tdSql.execute('create table t5 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t5 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows)
+        for i in range(0,6000):
+          ret = tdSql.execute(
+                'insert into t5 values (%d , 1)' %
+                (t0+200+i))
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows+6000)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows+6000)
+        print("==========step6")
+        print("UPDATE BLOCK IN TWO STEP")
+        ret = tdSql.execute('create table t6 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t6 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        for i in range(0,100):
+          ret = tdSql.execute(
+                'insert into t6 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'200')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'200')
+        for i in range(0,200):
+          ret = tdSql.execute(
+                'insert into t6 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'200')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'200')
+        print("==========step7")
+        print("UPDATE LAST HALF AND INSERT LITTLE DATA")
+        ret = tdSql.execute('create table t7 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t7 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t7")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t7")
+        tdSql.checkRows(insertRows)
+        for i in range(100,300):
+          ret = tdSql.execute(
+                'insert into t7 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t7")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0,0,'400')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t7")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0,0,'400')
+        print("==========step8")
+        print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
+        ret = tdSql.execute('create table t8 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t8 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t8")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t8")
+        tdSql.checkRows(insertRows)
+        for i in range(6000):
+          ret = tdSql.execute(
+                'insert into t8 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0,0,'11800')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0,0,'11800')
+        print("==========step9")
+        print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
+        ret = tdSql.execute('create table t9 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t9 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t9")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t9")
+        tdSql.checkRows(insertRows)
+        for i in range(-100,100):
+          ret = tdSql.execute(
+                'insert into t9 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t9")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0,0,'400')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t9")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0,0,'400')
+        print("==========step10")
+        print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
+        ret = tdSql.execute('create table t10 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t10 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t10")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t10")
+        tdSql.checkRows(insertRows)
+        for i in range(-6000,100):
+          ret = tdSql.execute(
+                'insert into t10 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t10")
+        tdSql.checkRows(6200)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0,0,'12200')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t10")
+        tdSql.checkRows(6200)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0,0,'12200')
+        print("==========step11")
+        print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
+        ret = tdSql.execute('create table t11 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t11 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t11")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t11")
+        tdSql.checkRows(insertRows)
+        for i in range(100):
+          ret = tdSql.execute(
+                'insert into t11 values (%d , 2)' %
+                (t0+i))
+        for i in range(200,6000):
+          ret = tdSql.execute(
+                'insert into t11 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t11")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t11")
+        tdSql.checkData(0,0,'11800')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t11")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t11")
+        tdSql.checkData(0,0,'11800')
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/merge_commit_data.py

@@ -0,0 +1,351 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        print("==========step1")
+        print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
+        s = 'reset query cache'
+        tdSql.execute(s)
+        s = 'drop database if exists db'
+        tdSql.execute(s)
+        s = 'create database db update 1 days 30'
+        tdSql.execute(s)
+        s = 'use db'
+        tdSql.execute(s)
+        ret = tdSql.execute('create table t1 (ts timestamp, a int)')
+
+        insertRows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(0, insertRows):
+          ret = tdSql.execute(
+            'insert into t1 values (%d , 1)' %
+            (t0 + i))
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t1")
+        tdSql.checkRows(insertRows)
+
+        for k in range(0,10):
+          for i in range (0,insertRows):
+            ret = tdSql.execute(
+              'insert into t1 values(%d,1)' %
+              (t0+i)
+            )
+          tdSql.query("select * from t1")
+          tdSql.checkRows(insertRows)
+          tdDnodes.stop(1)
+          tdDnodes.start(1)
+          tdSql.query("select * from t1")
+          tdSql.checkRows(insertRows)
+        print("==========step2")
+        print("PREPEND DATA ")
+        ret = tdSql.execute('create table t2 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t2 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows)
+        for i in range(-100,0):
+          ret = tdSql.execute(
+                'insert into t2 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows+100)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t2")
+        tdSql.checkRows(insertRows+100)
+        print("==========step3")
+        print("PREPEND MASSIVE DATA ")
+        ret = tdSql.execute('create table t3 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t3 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows)
+        for i in range(-6000,0):
+          ret = tdSql.execute(
+                'insert into t3 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows+6000)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t3")
+        tdSql.checkRows(insertRows+6000)
+        print("==========step4")
+        print("APPEND DATA")
+        ret = tdSql.execute('create table t4 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t4 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows)
+        for i in range(0,100):
+          ret = tdSql.execute(
+                'insert into t4 values (%d , 1)' %
+                (t0+200+i))
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows+100)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t4")
+        tdSql.checkRows(insertRows+100)
+        print("==========step5")
+        print("APPEND DATA")
+        ret = tdSql.execute('create table t5 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t5 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows)
+        for i in range(0,6000):
+          ret = tdSql.execute(
+                'insert into t5 values (%d , 1)' %
+                (t0+200+i))
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows+6000)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t5")
+        tdSql.checkRows(insertRows+6000)
+        print("==========step6")
+        print("UPDATE BLOCK IN TWO STEP")
+        ret = tdSql.execute('create table t6 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t6 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        for i in range(0,100):
+          ret = tdSql.execute(
+                'insert into t6 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'300')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'300')
+        for i in range(0,200):
+          ret = tdSql.execute(
+                'insert into t6 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'400')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t6")
+        tdSql.checkRows(insertRows)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0,0,'400')
+        print("==========step7")
+        print("UPDATE LAST HALF AND INSERT LITTLE DATA")
+        ret = tdSql.execute('create table t7 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t7 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t7")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t7")
+        tdSql.checkRows(insertRows)
+        for i in range(100,300):
+          ret = tdSql.execute(
+                'insert into t7 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t7")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0,0,'500')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t7")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0,0,'500')
+        print("==========step8")
+        print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
+        ret = tdSql.execute('create table t8 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t8 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t8")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t8")
+        tdSql.checkRows(insertRows)
+        for i in range(6000):
+          ret = tdSql.execute(
+                'insert into t8 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0,0,'12000')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0,0,'12000')
+        print("==========step9")
+        print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
+        ret = tdSql.execute('create table t9 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t9 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t9")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t9")
+        tdSql.checkRows(insertRows)
+        for i in range(-100,100):
+          ret = tdSql.execute(
+                'insert into t9 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t9")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0,0,'500')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t9")
+        tdSql.checkRows(300)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0,0,'500')
+        print("==========step10")
+        print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
+        ret = tdSql.execute('create table t10 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t10 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t10")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t10")
+        tdSql.checkRows(insertRows)
+        for i in range(-6000,100):
+          ret = tdSql.execute(
+                'insert into t10 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t10")
+        tdSql.checkRows(6200)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0,0,'12300')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t10")
+        tdSql.checkRows(6200)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0,0,'12300')
+        print("==========step11")
+        print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
+        ret = tdSql.execute('create table t11 (ts timestamp, a int)')
+        insertRows = 200
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into t11 values (%d , 1)' %
+                (t0+i))
+        tdSql.query("select * from t11")
+        tdSql.checkRows(insertRows)
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t11")
+        tdSql.checkRows(insertRows)
+        for i in range(100):
+          ret = tdSql.execute(
+                'insert into t11 values (%d , 2)' %
+                (t0+i))
+        for i in range(200,6000):
+          ret = tdSql.execute(
+                'insert into t11 values (%d , 2)' %
+                (t0+i))
+        tdSql.query("select * from t11")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t11")
+        tdSql.checkData(0,0,'11900')
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+        tdSql.query("select * from t11")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t11")
+        tdSql.checkData(0,0,'11900')
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/merge_commit_data2.py

@@ -0,0 +1,352 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import time
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+    
+    
+    def restart_taosd(self,db):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use %s;" % db)
+
+    def date_to_timestamp_microseconds(self, date):
+        datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
+        obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
+        return obj_stamp
+
+    def timestamp_microseconds_to_date(self, timestamp):
+        d = datetime.datetime.fromtimestamp(timestamp/1000)
+        str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
+        return str1
+
+
+
+    def run(self):
+        print("==========step1")
+        print("create table && insert data")
+        sql = 'reset query cache'
+        tdSql.execute(sql)
+        sql = 'drop database if exists db'
+        tdSql.execute(sql)
+        sql = 'create database db update 1 days 30;'
+        tdSql.execute(sql)
+        sql = 'use db;'
+        tdSql.execute(sql)
+        tdSql.execute('create table t1 (ts timestamp, a int)')
+
+
+        print("==================================1 start")
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
+        print("==========step2")
+        print("restart to commit ")
+        self.restart_taosd('db')
+
+        print('check query result after restart')
+        tdSql.query('select * from db.t1;')
+        for i in range(insert_rows):
+            tdSql.checkData(i, 1, 1)
+
+        print("==========step3")
+        print('insert data')
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+        print('check query result before restart')
+        tdSql.query('select * from db.t1;')
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        self.restart_taosd('db')
+        print('check query result after restart')
+        tdSql.query('select * from db.t1;')
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        print("==========step4")
+        print('insert data')
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t1;')
+        print(tdSql.queryResult)
+        for i in range(insert_rows):
+            tdSql.checkData(i, 1, 2)
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t1;')
+        # print(tdSql.queryResult)
+        for i in range(insert_rows):
+            tdSql.checkData(i, 1, 2)
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        print("==================================2 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t2 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+        for i in range(insert_rows):
+            tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for k in range(10):
+            for i in range(10):
+                tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+                print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+
+
+        print("==========step2")
+        print('check query result before restart')
+        tdSql.query('select * from db.t2;')
+        for i in range(insert_rows*2+100):
+            tdSql.checkData(i, 1, 1)
+        # print(tdSql.queryResult)
+        print('restart to commit')
+        self.restart_taosd('db')
+        print('check query result after restart')
+        tdSql.query('select * from db.t2;')
+        for i in range(insert_rows*2+100):
+            tdSql.checkData(i, 1, 1)
+
+
+        print("==================================3 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t3 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(5200):
+            tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
+
+        print("==========step2")
+        print('check query result before restart')
+        tdSql.query('select * from db.t3;')
+        for i in range(5200):
+            tdSql.checkData(i, 1, 2)
+        # print(tdSql.queryResult)
+        print('restart to commit')
+        self.restart_taosd('db')
+        print('check query result after restart')
+        tdSql.query('select * from db.t3;')
+        for i in range(5200):
+            tdSql.checkData(i, 1, 2)
+
+        print("==================================4 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t4 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(100):
+            tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+        for i in range(200, 5000):
+            tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+        for i in range(100):
+            tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t4;')
+        for i in range(100):
+            tdSql.checkData(i, 1, 2)
+        for i in range(100, 200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t4;')
+        for i in range(100):
+            tdSql.checkData(i, 1, 2)
+        for i in range(100, 200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+
+        print("==================================5 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t5 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(100, 200):
+            tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+        for i in range(200, 5000):
+            tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+        for i in range(100, 200):
+            tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t5;')
+        for i in range(100):
+            tdSql.checkData(i, 1, 1)
+        for i in range(100, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5100):
+            tdSql.checkData(i, 1, 1)
+        for i in range(5100, 5200):
+            tdSql.checkData(i, 1, 2)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t5;')
+        for i in range(100):
+            tdSql.checkData(i, 1, 1)
+        for i in range(100, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5100):
+            tdSql.checkData(i, 1, 1)
+        for i in range(5100, 5200):
+            tdSql.checkData(i, 1, 2)
+
+        print("==================================6 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t6 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(-1000, 10000):
+            tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t6;')
+        tdSql.checkRows(11000)
+        for i in range(11000):
+            tdSql.checkData(i, 1, 2)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t6;')
+        tdSql.checkRows(11000)
+        for i in range(11000):
+            tdSql.checkData(i, 1, 2)
+
+
+        print("==================================7 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t7 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(-1000, 10000):
+            tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t7;')
+        tdSql.checkRows(11000)
+        for i in range(11000):
+            tdSql.checkData(i, 1, 2)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t7;')
+        tdSql.checkRows(11000)
+        for i in range(11000):
+            tdSql.checkData(i, 1, 2)
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/merge_commit_data2_update0.py

@@ -0,0 +1,384 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import time
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+
+    def restart_taosd(self,db):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use %s;" % db)
+
+    def date_to_timestamp_microseconds(self, date):
+        datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
+        obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
+        return obj_stamp
+
+    def timestamp_microseconds_to_date(self, timestamp):
+        d = datetime.datetime.fromtimestamp(timestamp/1000)
+        str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
+        return str1
+
+
+
+    def run(self):
+        print("==========step1")
+        print("create table && insert data")
+        sql = 'reset query cache'
+        tdSql.execute(sql)
+        sql = 'drop database if exists db'
+        tdSql.execute(sql)
+        sql = 'create database db update 0 days 30;'
+        tdSql.execute(sql)
+        sql = 'use db;'
+        tdSql.execute(sql)
+        tdSql.execute('create table t1 (ts timestamp, a int)')
+
+
+        print("==================================1 start")
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
+        print("==========step2")
+        print("restart to commit ")
+        self.restart_taosd('db')
+
+        print('check query result after restart')
+        tdSql.query('select * from db.t1;')
+        for i in range(insert_rows):
+            tdSql.checkData(i, 1, 1)
+
+        print("==========step3")
+        print('insert data')
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+        print('check query result before restart')
+        tdSql.query('select * from db.t1;')
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        self.restart_taosd('db')
+        print('check query result after restart')
+        tdSql.query('select * from db.t1;')
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        print("==========step4")
+        print('insert data')
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
+        for i in range(insert_rows):
+            tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t1;')
+        print(tdSql.queryResult)
+        for i in range(insert_rows):
+            tdSql.checkData(i, 1, 1)
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t1;')
+        # print(tdSql.queryResult)
+        for i in range(insert_rows):
+            tdSql.checkData(i, 1, 1)
+        for i in range(insert_rows, insert_rows*2):
+            tdSql.checkData(i, 1, 1)
+
+        print("==================================2 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t2 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+        for i in range(insert_rows):
+            tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for k in range(10):
+            for i in range(10):
+                tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+                # print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
+
+
+        print("==========step2")
+        print('check query result before restart')
+        tdSql.query('select * from db.t2;')
+        for i in range(insert_rows*2+100):
+            tdSql.checkData(i, 1, 1)
+        # print(tdSql.queryResult)
+        print('restart to commit')
+        self.restart_taosd('db')
+        print('check query result after restart')
+        tdSql.query('select * from db.t2;')
+        for i in range(insert_rows*2+100):
+            tdSql.checkData(i, 1, 1)
+
+
+        print("==================================3 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t3 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(5200):
+            tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
+
+        print("==========step2")
+        print('check query result before restart')
+        tdSql.query('select * from db.t3;')
+        for i in range(200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+        # print(tdSql.queryResult)
+        print('restart to commit')
+        self.restart_taosd('db')
+        print('check query result after restart')
+        tdSql.query('select * from db.t3;')
+        for i in range(200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+
+        print("==================================4 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t4 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(100):
+            tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+        for i in range(200, 5000):
+            tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
+
+        for i in range(100):
+            tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t4;')
+        for i in range(200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t4;')
+        for i in range(200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+        #
+        print("==================================5 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t5 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(100, 200):
+            tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+        for i in range(200, 5000):
+            tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
+
+        for i in range(100, 200):
+            tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t5;')
+        for i in range(200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t5;')
+        for i in range(200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(200, 5000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(5000, 5200):
+            tdSql.checkData(i, 1, 1)
+
+        print("==================================6 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t6 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(-1000, 10000):
+            tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t6;')
+        tdSql.checkRows(11000)
+        for i in range(1000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(1000,1200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(1200,6000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(6000,6200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(6200, 11000):
+            tdSql.checkData(i, 1, 2)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t6;')
+        tdSql.checkRows(11000)
+        for i in range(1000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(1000,1200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(1200,6000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(6000,6200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(6200, 11000):
+            tdSql.checkData(i, 1, 2)
+
+
+        print("==================================7 start")
+        print("==========step1")
+        print("create table && insert data")
+        tdSql.execute('create table t7 (ts timestamp, a int)')
+        insert_rows = 200
+        t0 = 1603152000000
+        tdLog.info("insert %d rows" % insert_rows)
+        for i in range(insert_rows):
+            tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
+
+        for i in range(insert_rows):
+            tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
+        print('restart to commit')
+        self.restart_taosd('db')
+
+        for i in range(-1000, 10000):
+            tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
+
+        print('check query result before restart')
+        tdSql.query('select * from db.t7;')
+        tdSql.checkRows(11000)
+        for i in range(1000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(1000,1200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(1200,6000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(6000,6200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(6200, 11000):
+            tdSql.checkData(i, 1, 2)
+
+        print('check query result after restart')
+        self.restart_taosd('db')
+        tdSql.query('select * from db.t7;')
+        tdSql.checkRows(11000)
+        for i in range(1000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(1000,1200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(1200,6000):
+            tdSql.checkData(i, 1, 2)
+        for i in range(6000,6200):
+            tdSql.checkData(i, 1, 1)
+        for i in range(6200, 11000):
+            tdSql.checkData(i, 1, 2)
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/merge_commit_last-0.py

@@ -0,0 +1,309 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.ts = 1603152000000
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use db")
+
+    def run(self):              
+        tdSql.prepare()
+
+        print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")                     
+        tdSql.execute("create table t1 (ts timestamp, a int)")
+
+        for i in range(5):
+            tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
+            self.restartTaosd()
+            tdSql.query("select * from t1")            
+            tdSql.checkRows(1)            
+            tdSql.checkData(0, 1, 0)
+
+        print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
+        tdSql.execute("create table t2 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t2")
+        tdSql.checkData(0, 0, 50)
+
+        for i in range(50):
+            tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
+        tdSql.query("select * from t2")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t2")
+        tdSql.checkData(0, 0, 50)
+
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t2")
+        tdSql.checkData(0, 0, 50)
+
+        print("==============step 3: UPDATE PART OF THE LAST BLOCK")
+        tdSql.execute("create table t3 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t3")
+        tdSql.checkData(0, 0, 50)
+
+        for i in range(25):
+            tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t3")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t3")
+        tdSql.checkData(0, 0, 50)
+
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t3")
+        tdSql.checkData(0, 0, 50)
+        
+        print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
+        tdSql.execute("create table t4 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t4")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t4")
+        tdSql.checkData(0, 0, 50)
+
+        for i in range(25):
+            tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+        
+        for i in range(50, 60):
+            tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t4")        
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t4")
+        tdSql.checkData(0, 0, 70)
+        
+        self.restartTaosd()
+        tdSql.query("select * from t4")        
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t4")
+        tdSql.checkData(0, 0, 70)
+
+        print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
+        tdSql.execute("create table t5 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t5")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-10, 0):
+            tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+        
+        for i in range(25):
+            tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 70)
+
+        self.restartTaosd()
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 70)
+
+        for i in range(-10, 0):
+            tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+        
+        for i in range(25, 50):
+            tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+
+        tdSql.query("select * from t5")        
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 70)
+
+        self.restartTaosd()
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 70)
+        
+
+        print("==============step 6: INSERT AHEAD A LOT OF DATA")
+        tdSql.execute("create table t6 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t6")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-1000, 0):
+            tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t6")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0, 0, 2050)
+
+        self.restartTaosd()
+        tdSql.query("select * from t6")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0, 0, 2050)
+
+        print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
+        tdSql.execute("create table t7 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t7")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-1000, 25):
+            tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t7")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0, 0, 2050)
+
+        self.restartTaosd()
+        tdSql.query("select * from t7")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0, 0, 2050)
+
+        print("==============step 8: INSERT AFTER A LOT AND UPDATE")
+        tdSql.execute("create table t8 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t8")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(25, 6000):
+            tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0, 0, 11950)
+
+        self.restartTaosd()
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0, 0, 11950)
+
+        print("==============step 9: UPDATE ONLY MIDDLE")
+        tdSql.execute("create table t9 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t9")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(20, 30):
+            tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t9")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0, 0, 50)
+
+        self.restartTaosd()
+        tdSql.query("select * from t9")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0, 0, 50)
+        
+        print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
+        tdSql.execute("create table t10 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t10")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-4000, 4000):
+            tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t10")
+        tdSql.checkRows(8000)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0, 0, 15950)
+
+        self.restartTaosd()
+        tdSql.query("select * from t10")
+        tdSql.checkRows(8000)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0, 0, 15950)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/update/merge_commit_last.py

@@ -0,0 +1,321 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.ts = 1603152000000
+    
+    def restartTaosd(self):
+        tdDnodes.stop(1)
+        tdDnodes.startWithoutSleep(1)
+        tdSql.execute("use udb")
+
+    def run(self):              
+        tdSql.prepare()
+
+        tdSql.execute("create database udb update 1 days 30")   
+        tdSql.execute("use udb")
+
+        print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")                     
+        tdSql.execute("create table t1 (ts timestamp, a int)")
+
+        for i in range(5):
+            tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
+            self.restartTaosd()
+            tdSql.query("select * from t1")            
+            tdSql.checkRows(1)            
+            tdSql.checkData(0, 1, i)
+
+        print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
+        tdSql.execute("create table t2 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(50)
+        for i in range(50):
+            tdSql.checkData(i, 1, 1)
+
+        for i in range(50):
+            tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
+        tdSql.query("select * from t2")
+        for i in range(50):
+            tdSql.checkData(i, 1, 2)
+
+        self.restartTaosd()
+        tdSql.query("select * from t2")
+        tdSql.checkRows(50)
+        for i in range(50):
+            tdSql.checkData(i, 1, 2)
+
+        print("==============step 3: UPDATE PART OF THE LAST BLOCK")
+        tdSql.execute("create table t3 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(50)
+        for i in range(50):
+            tdSql.checkData(i, 1, 1)
+
+        for i in range(25):
+            tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t3")
+        for i in range(25):
+            tdSql.checkData(i, 1, 2)
+        for i in range(25, 50):
+            tdSql.checkData(i, 1, 1)
+
+        self.restartTaosd()
+        tdSql.query("select * from t3")
+        tdSql.checkRows(50)
+        for i in range(25):
+            tdSql.checkData(i, 1, 2)
+        for i in range(25, 50):
+            tdSql.checkData(i, 1, 1)
+        
+        print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
+        tdSql.execute("create table t4 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t4")
+        tdSql.checkRows(50)
+        for i in range(50):
+            tdSql.checkData(i, 1, 1)
+
+        for i in range(25):
+            tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+        
+        for i in range(50, 60):
+            tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t4")
+        tdSql.checkRows(60)
+        for i in range(25):
+            tdSql.checkData(i, 1, 2)
+        for i in range(25, 50):
+            tdSql.checkData(i, 1, 1)
+        for i in range(50, 60):
+            tdSql.checkData(i, 1, 2)
+        
+        self.restartTaosd()
+        tdSql.query("select * from t4")
+        tdSql.checkRows(60)
+        for i in range(25):
+            tdSql.checkData(i, 1, 2)
+        for i in range(25, 50):
+            tdSql.checkData(i, 1, 1)
+        for i in range(50, 60):
+            tdSql.checkData(i, 1, 2)
+
+        print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
+        tdSql.execute("create table t5 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
+        
+        self.restartTaosd()
+        tdSql.query("select * from t5")
+        tdSql.checkRows(50)
+        for i in range(50):
+            tdSql.checkData(i, 1, 1)
+        
+        for i in range(-10, 0):
+            tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+        
+        for i in range(25):
+            tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 95)
+
+        self.restartTaosd()
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 95)
+
+        for i in range(-10, 0):
+            tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+        
+        for i in range(25, 50):
+            tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
+
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 155)
+
+        self.restartTaosd()
+        tdSql.query("select * from t5")
+        tdSql.checkRows(60)
+        tdSql.query("select sum(a) from t5")
+        tdSql.checkData(0, 0, 155)
+        
+
+        print("==============step 6: INSERT AHEAD A LOT OF DATA")
+        tdSql.execute("create table t6 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t6")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-1000, 0):
+            tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t6")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0, 0, 2050)
+
+        self.restartTaosd()
+        tdSql.query("select * from t6")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t6")
+        tdSql.checkData(0, 0, 2050)
+
+        print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
+        tdSql.execute("create table t7 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t7")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-1000, 25):
+            tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t7")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0, 0, 2075)
+
+        self.restartTaosd()
+        tdSql.query("select * from t7")
+        tdSql.checkRows(1050)
+        tdSql.query("select sum(a) from t7")
+        tdSql.checkData(0, 0, 2075)
+
+        print("==============step 8: INSERT AFTER A LOT AND UPDATE")
+        tdSql.execute("create table t8 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t8")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(25, 6000):
+            tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0, 0, 11975)
+
+        self.restartTaosd()
+        tdSql.query("select * from t8")
+        tdSql.checkRows(6000)
+        tdSql.query("select sum(a) from t8")
+        tdSql.checkData(0, 0, 11975)
+
+        print("==============step 9: UPDATE ONLY MIDDLE")
+        tdSql.execute("create table t9 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t9")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(20, 30):
+            tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t9")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0, 0, 60)
+
+        self.restartTaosd()
+        tdSql.query("select * from t9")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t9")
+        tdSql.checkData(0, 0, 60)
+        
+        print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
+        tdSql.execute("create table t10 (ts timestamp, a int)")
+
+        for i in range(50):
+            tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
+
+        self.restartTaosd()
+        tdSql.query("select * from t10")
+        tdSql.checkRows(50)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0, 0, 50)
+        
+        for i in range(-4000, 4000):
+            tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
+        
+        tdSql.query("select * from t10")
+        tdSql.checkRows(8000)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0, 0, 16000)
+
+        self.restartTaosd()
+        tdSql.query("select * from t10")
+        tdSql.checkRows(8000)
+        tdSql.query("select sum(a) from t10")
+        tdSql.checkData(0, 0, 16000)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/updatetest.sh

@@ -0,0 +1,10 @@
+# update
+python3 ./test.py -f update/allow_update.py
+python3 ./test.py -f update/allow_update-0.py
+python3 ./test.py -f update/append_commit_data.py
+python3 ./test.py -f update/append_commit_last-0.py
+python3 ./test.py -f update/append_commit_last.py
+python3 ./test.py -f update/merge_commit_data.py
+python3 ./test.py -f update/merge_commit_data2.py
+python3 ./test.py -f update/merge_commit_last-0.py
+python3 ./test.py -f update/merge_commit_last.py

tests/script/general/http/restful_full.sim

@@ -81,7 +81,7 @@ print ===============  step2 - no db
 #11
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'show databases' 127.0.0.1:7111/rest/sql
 print 11-> $system_content
-if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
+if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then
   return -1
 endi