Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

cmake/install.inc

@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.8-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

src/client/src/tscPrepare.c

@@ -258,11 +258,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) {
 
 static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
   if (bind->is_null != NULL && *(bind->is_null)) {
-    if (param->type == TSDB_DATA_TYPE_BINARY || param->type == TSDB_DATA_TYPE_NCHAR) {
+    setNull(data + param->offset, param->type, param->bytes);
-      setVardataNull(data + param->offset, param->type);
-    } else {
-      setNull(data + param->offset, param->type, param->bytes);
-    }
     return TSDB_CODE_SUCCESS;
   }
 
@@ -312,13 +308,13 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
               break;
             }
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_FLOAT:
           case TSDB_DATA_TYPE_DOUBLE:
           case TSDB_DATA_TYPE_TIMESTAMP:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
         }
         memcpy(data + param->offset, &u.v1, sizeof(u.v1));
         return TSDB_CODE_SUCCESS;
@@ -336,40 +332,40 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
             u.v1 = (int8_t)v;
             if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_INT: {
             int32_t v = *(int32_t*)bind->buffer;
             u.v1 = (int8_t)v;
             if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_BIGINT: {
             int64_t v = *(int64_t*)bind->buffer;
             u.v1 = (int8_t)v;
             if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_BINARY:
           case TSDB_DATA_TYPE_NCHAR: {
             int64_t v;
-            int n,r;
+            int     n, r;
             r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n);
-            if (r==1 && n==strlen((const char*)bind->buffer)) {
+            if (r == 1 && n == strlen((const char*)bind->buffer)) {
               u.v1 = (int8_t)v;
               if (v >= SCHAR_MIN && v <= SCHAR_MAX) break;
             }
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_FLOAT:
           case TSDB_DATA_TYPE_DOUBLE:
           case TSDB_DATA_TYPE_TIMESTAMP:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
         }
         memcpy(data + param->offset, &u.v1, sizeof(u.v1));
         return TSDB_CODE_SUCCESS;
-			} break;
+      }
       case TSDB_DATA_TYPE_SMALLINT: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_BOOL:
@@ -383,34 +379,34 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
             u.v2 = (int16_t)v;
             if (v >= SHRT_MIN && v <= SHRT_MAX) break;
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_BIGINT: {
             int64_t v = *(int64_t*)bind->buffer;
             u.v2 = (int16_t)v;
             if (v >= SHRT_MIN && v <= SHRT_MAX) break;
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_BINARY:
           case TSDB_DATA_TYPE_NCHAR: {
             int64_t v;
-            int n,r;
+            int     n, r;
             r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n);
-            if (r==1 && n==strlen((const char*)bind->buffer)) {
+            if (r == 1 && n == strlen((const char*)bind->buffer)) {
               u.v2 = (int16_t)v;
               if (v >= SHRT_MIN && v <= SHRT_MAX) break;
             }
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_FLOAT:
           case TSDB_DATA_TYPE_DOUBLE:
           case TSDB_DATA_TYPE_TIMESTAMP:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
         }
         memcpy(data + param->offset, &u.v2, sizeof(u.v2));
         return TSDB_CODE_SUCCESS;
-			} break;
+      }
       case TSDB_DATA_TYPE_INT: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_BOOL:
@@ -514,17 +510,17 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
               break;
             }
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_FLOAT:
           case TSDB_DATA_TYPE_DOUBLE:
           case TSDB_DATA_TYPE_TIMESTAMP:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
         }
         memcpy(data + param->offset, &u.v8, sizeof(u.v8));
         return TSDB_CODE_SUCCESS;
-			} break;
+      }
       case TSDB_DATA_TYPE_DOUBLE: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_BOOL:
@@ -556,15 +552,15 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
               break;
             }
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
           case TSDB_DATA_TYPE_TIMESTAMP:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
-        } break;
+        }
         memcpy(data + param->offset, &u.f8, sizeof(u.f8));
         return TSDB_CODE_SUCCESS;
-			} break;
+      }
       case TSDB_DATA_TYPE_TIMESTAMP: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_TIMESTAMP: {
@@ -589,10 +585,10 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
           } break;
-        } break;
+        };
         memcpy(data + param->offset, &u.v8, sizeof(u.v8));
         return TSDB_CODE_SUCCESS;
-			} break;
+			}
       case TSDB_DATA_TYPE_BINARY: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_BINARY: {
@@ -602,7 +598,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
             short size = (short)*bind->length;
             STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size);
             return TSDB_CODE_SUCCESS;
-          } break;
+          }
           case TSDB_DATA_TYPE_BOOL:
           case TSDB_DATA_TYPE_TINYINT:
           case TSDB_DATA_TYPE_SMALLINT:
@@ -614,9 +610,9 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
           case TSDB_DATA_TYPE_NCHAR:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
         }
-			} break;
+      }
       case TSDB_DATA_TYPE_NCHAR: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_NCHAR: {
@@ -626,7 +622,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
             }
             varDataSetLen(data + param->offset, output);
             return TSDB_CODE_SUCCESS;
-          } break;
+          }
           case TSDB_DATA_TYPE_BOOL:
           case TSDB_DATA_TYPE_TINYINT:
           case TSDB_DATA_TYPE_SMALLINT:
@@ -638,12 +634,12 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
           case TSDB_DATA_TYPE_BINARY:
           default: {
             return TSDB_CODE_TSC_INVALID_VALUE;
-          } break;
+          }
         }
-			} break;
+      }
       default: {
         return TSDB_CODE_TSC_INVALID_VALUE;
-      } break;
+      }
     }
   }
 

src/client/src/tscSQLParser.c

@@ -80,7 +80,6 @@ static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprLis
 static bool validateIpAddress(const char* ip, size_t size);
 static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
 static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery);
-static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);
 
 static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd);
 
@@ -1768,10 +1767,10 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
   return TSDB_CODE_SUCCESS;
 }
 
-void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) {
+void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) {
   if (pItem->aliasName != NULL) {
     tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
-  } else {
+  } else if (multiCols) {
     char uname[TSDB_COL_NAME_LEN] = {0};
     int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN);
     tstrncpy(uname, pToken->z, len);
@@ -1782,6 +1781,9 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT
     snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname);
 
     tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
+  } else  { // use the user-input result column name
+    int32_t len = MIN(pItem->pNode->token.n + 1, TSDB_COL_NAME_LEN);
+    tstrncpy(name, pItem->pNode->token.z, len);
   }
 }
 
@@ -2056,7 +2058,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
             for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) {
               index.columnIndex = j;
               SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)};
-              setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+              setResultColName(name, pItem, cvtFunc.originFuncId, &t, true);
 
               if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) {
                 return TSDB_CODE_TSC_INVALID_SQL;
@@ -2078,7 +2080,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
             char name[TSDB_COL_NAME_LEN] = {0};
 
             SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
-            setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo);
+
+            bool multiColOutput = pItem->pNode->pParam->nExpr > 1;
+            setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo, multiColOutput);
 
             if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
@@ -2120,7 +2124,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
             char name[TSDB_COL_NAME_LEN] = {0};
             SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)};
-            setResultColName(name, pItem, cvtFunc.originFuncId, &t);
+            setResultColName(name, pItem, cvtFunc.originFuncId, &t, true);
 
             if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
@@ -2951,14 +2955,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd)
   return TSDB_CODE_SUCCESS;
 }
 
-void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) {
-  if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
-    tscFieldInfoUpdateOffset(pQueryInfo);
-  } else {
-    tscFieldInfoUpdateOffset(pQueryInfo);
-  }
-}
-
 static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) {
   if (pColumn == NULL) {
     return NULL;
@@ -3537,7 +3533,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer
 
       if (i == 0) {
         id = p1->uid;
-      } else if (id != p1->uid){
+      } else if (id != p1->uid) {
         return TSDB_CODE_TSC_INVALID_SQL;
       }
     }
@@ -4252,6 +4248,9 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
     tExprTreeDestroy(&p, NULL);
     
     taosArrayDestroy(colList);
+    if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table");
+    }
   }
 
   pCondExpr->pTagCond = NULL;
@@ -6531,7 +6530,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     return code;
   }
 
-  setColumnOffsetValueInResultset(pQueryInfo);
+  tscFieldInfoUpdateOffset(pQueryInfo);
 
   /*
    * fill options are set at the end position, when all columns are set properly

src/client/src/tscServer.c

@@ -62,7 +62,7 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) {
 
   pEpSet->numOfEps = pVgroupInfo->numOfEps;
   for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
-    strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn);
+    tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
     pEpSet->port[i] = pVgroupInfo->epAddr[i].port;
 
     if (!hasFqdn) {
@@ -669,6 +669,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
 
   SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload;
+  tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version));
 
   int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList);
   
@@ -693,8 +694,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->interval.slidingUnit  = pQueryInfo->interval.slidingUnit;
   pQueryMsg->interval.offsetUnit   = pQueryInfo->interval.offsetUnit;
   pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
-  pQueryMsg->numOfTags      = htonl(numOfTags);
   pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
+  pQueryMsg->numOfTags      = htonl(numOfTags);
   pQueryMsg->queryType      = htonl(pQueryInfo->type);
   
   size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
@@ -2030,7 +2031,8 @@ static void createHBObj(STscObj* pObj) {
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0);
   if (pQueryInfo == NULL) {
-    pSql->res.code = terrno;
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    tfree(pSql);
     return;
   }
 
@@ -2038,6 +2040,7 @@ static void createHBObj(STscObj* pObj) {
 
   pSql->cmd.command = pQueryInfo->command;
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     tfree(pSql);
     return;
   }

src/client/src/tscSql.c

@@ -420,7 +420,16 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
     for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) {
       SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i);
       if (pField->visible) {
-        f[j++] = pField->field;
+        f[j] = pField->field;
+
+        // revise the length for binary and nchar fields
+        if (f[j].type == TSDB_DATA_TYPE_BINARY) {
+          f[j].bytes -= VARSTR_HEADER_SIZE;
+        } else if (f[j].type == TSDB_DATA_TYPE_NCHAR) {
+          f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE;
+        }
+
+        j += 1;
       }
     }
 

src/client/src/tscUtil.c

@@ -451,14 +451,16 @@ void tscFreeRegisteredSqlObj(void *pSql) {
   STscObj* pTscObj = (*p)->pTscObj;
 
   assert((*p)->self != 0 && (*p)->self == (p));
+
+  SSqlObj* ptr = *p;
   tscFreeSqlObj(*p);
 
   int32_t ref = T_REF_DEC(pTscObj);
   assert(ref >= 0);
 
-  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
+  tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", ptr, pTscObj, ref);
   if (ref == 0) {
-    tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
+    tscDebug("%p all sqlObj freed, free tscObj:%p", ptr, pTscObj);
     taosRemoveRef(tscRefId, pTscObj->rid);
   }
 }
@@ -644,6 +646,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
   dataBuf->pData = calloc(1, dataBuf->nAllocSize);
   if (dataBuf->pData == NULL) {
     tscError("failed to allocated memory, reason:%s", strerror(errno));
+    tfree(dataBuf);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 

src/client/tests/resultFieldTest.cpp

@@ -0,0 +1,138 @@
+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "taos.h"
+
+namespace {
+static int64_t start_ts = 1433955661000;
+}
+/* test parse time function */
+TEST(testCase, result_field_test) {
+  taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
+  taos_init();
+
+  TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0);
+  if (conn == NULL) {
+    printf("Failed to connect to DB, reason:%s", taos_errstr(conn));
+    exit(-1);
+  }
+
+  TAOS_RES* res = taos_query(conn, "create database if not exists test");
+  ASSERT_EQ(taos_errno(res), 0);
+  taos_free_result(res);
+
+  res = taos_query(conn, "use test");
+  ASSERT_EQ(taos_errno(res), 0);
+  taos_free_result(res);
+
+  res = taos_query(conn, "create table if not exists t1(ts timestamp, k int, a binary(11), b nchar(4))");
+  ASSERT_EQ(taos_errno(res), 0);
+  taos_free_result(res);
+
+  char sql[512] = {0};
+  sprintf(sql, "insert into t1 values(%ld, 99, 'abc', 'test')", start_ts);
+
+  res = taos_query(conn, sql);
+  ASSERT_EQ(taos_errno(res), 0);
+  taos_free_result(res);
+
+  res = taos_query(conn, "select count(*), spread(ts)/(1000 * 3600 * 24), first(a), last(b) from t1");
+  ASSERT_EQ(taos_num_fields(res), 4);
+
+  TAOS_FIELD* fields = taos_fetch_fields(res);
+  ASSERT_EQ(fields[0].bytes, 8);
+  ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_BIGINT);
+  ASSERT_STREQ(fields[0].name, "count(*)");
+
+  ASSERT_EQ(fields[1].bytes, 8);
+  ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_DOUBLE);
+  ASSERT_STREQ(fields[1].name, "spread(ts)/(1000 * 3600 * 24)");
+
+  ASSERT_EQ(fields[2].bytes, 11);
+  ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY);
+  ASSERT_STREQ(fields[2].name, "first(a)");
+
+  ASSERT_EQ(fields[3].bytes, 4);
+  ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR);
+  ASSERT_STREQ(fields[3].name, "last(b)");
+
+  taos_free_result(res);
+
+  res = taos_query(conn, "select last_row(*) from t1");
+  ASSERT_EQ(taos_num_fields(res), 4);
+
+  fields = taos_fetch_fields(res);
+  ASSERT_EQ(fields[0].bytes, 8);
+  ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
+  ASSERT_STREQ(fields[0].name, "last_row(ts)");
+
+  ASSERT_EQ(fields[1].bytes, 4);
+  ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT);
+  ASSERT_STREQ(fields[1].name, "last_row(k)");
+
+  ASSERT_EQ(fields[2].bytes, 11);
+  ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY);
+  ASSERT_STREQ(fields[2].name, "last_row(a)");
+
+  ASSERT_EQ(fields[3].bytes, 4);
+  ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR);
+  ASSERT_STREQ(fields[3].name, "last_row(b)");
+
+  taos_free_result(res);
+  res = taos_query(conn, "select first(*), last(*) from t1");
+  ASSERT_EQ(taos_num_fields(res), 8);
+
+  fields = taos_fetch_fields(res);
+  ASSERT_EQ(fields[0].bytes, 8);
+  ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
+  ASSERT_STREQ(fields[0].name, "first(ts)");
+
+  ASSERT_EQ(fields[1].bytes, 4);
+  ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT);
+  ASSERT_STREQ(fields[1].name, "first(k)");
+
+  ASSERT_EQ(fields[2].bytes, 11);
+  ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY);
+  ASSERT_STREQ(fields[2].name, "first(a)");
+
+  ASSERT_EQ(fields[3].bytes, 4);
+  ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR);
+  ASSERT_STREQ(fields[3].name, "first(b)");
+
+  taos_free_result(res);
+
+  res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1");
+  ASSERT_EQ(taos_num_fields(res), 7);
+
+  fields = taos_fetch_fields(res);
+  ASSERT_EQ(fields[0].bytes, 8);
+  ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP);
+  ASSERT_STREQ(fields[0].name, "first(ts)");
+
+  ASSERT_EQ(fields[1].bytes, 11);
+  ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_BINARY);
+  ASSERT_STREQ(fields[1].name, "first(a)");
+
+  ASSERT_EQ(fields[2].bytes, 4);
+  ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT);
+  ASSERT_STREQ(fields[2].name, "first(k)");
+
+  ASSERT_EQ(fields[3].bytes, 4);
+  ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_INT);
+  ASSERT_STREQ(fields[3].name, "first(k)");
+
+  ASSERT_EQ(fields[4].bytes, 4);
+  ASSERT_EQ(fields[4].type, TSDB_DATA_TYPE_NCHAR);
+  ASSERT_STREQ(fields[4].name, "first(b)");
+
+  ASSERT_EQ(fields[5].bytes, 4);
+  ASSERT_EQ(fields[5].type, TSDB_DATA_TYPE_NCHAR);
+  ASSERT_STREQ(fields[5].name, "first(b)");
+
+  ASSERT_EQ(fields[6].bytes, 8);
+  ASSERT_EQ(fields[6].type, TSDB_DATA_TYPE_TIMESTAMP);
+  ASSERT_STREQ(fields[6].name, "first(ts)");
+
+  taos_free_result(res);
+  taos_close(conn);
+}

src/common/src/tvariant.c

@@ -171,7 +171,9 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
     }
   }
 
-  pDst->nLen = tDataTypeDesc[pDst->nType].nSize;
+  if (pDst->nType != TSDB_DATA_TYPE_ARRAY) {
+    pDst->nLen = tDataTypeDesc[pDst->nType].nSize;
+  }
 }
 
 int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) {

src/connector/jdbc/CMakeLists.txt

@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.8-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.10</version>
+  <version>2.0.13</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>
@@ -49,17 +49,29 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <version>3.5</version>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <version>4.13</version>
       <scope>test</scope>
     </dependency>
+
+    <!-- for restful -->
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>4.5.8</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.9</version>
+    </dependency>
+    <dependency>
+      <groupId>com.alibaba</groupId>
+      <artifactId>fastjson</artifactId>
+      <version>1.2.58</version>
+    </dependency>
   </dependencies>
 
   <build>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.8</version>
+    <version>2.0.13</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
@@ -112,6 +112,13 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.12.4</version>
                 <configuration>
+                    <includes>
+                        <include>**/*Test.java</include>
+                    </includes>
+                    <excludes>
+                        <exclude>**/BatchInsertTest.java</exclude>
+                        <exclude>**/FailOverTest.java</exclude>
+                    </excludes>
                     <testFailureIgnore>true</testFailureIgnore>
                 </configuration>
             </plugin>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java

@@ -15,7 +15,6 @@ public class RestfulConnection implements Connection {
     private final String database;
     private final String url;
 
-
     public RestfulConnection(String host, String port, Properties props, String database, String url) {
         this.host = host;
         this.port = Integer.parseInt(port);
@@ -28,7 +27,7 @@ public class RestfulConnection implements Connection {
     public Statement createStatement() throws SQLException {
         if (isClosed())
             throw new SQLException(TSDBConstants.WrapErrMsg("restful TDengine connection is closed."));
-        return new RestfulStatement(this, this.database);
+        return new RestfulStatement(this, database);
     }
 
     @Override
@@ -104,22 +103,28 @@ public class RestfulConnection implements Connection {
 
     @Override
     public void setTransactionIsolation(int level) throws SQLException {
-
+        //transaction is not supported
+        throw new SQLFeatureNotSupportedException("transactions are not supported");
     }
 
+    /**
+     *
+     */
     @Override
     public int getTransactionIsolation() throws SQLException {
-        return 0;
+        //Connection.TRANSACTION_NONE specifies that transactions are not supported.
+        return Connection.TRANSACTION_NONE;
     }
 
     @Override
     public SQLWarning getWarnings() throws SQLException {
+        //TODO: getWarnings not implemented
         return null;
     }
 
     @Override
     public void clearWarnings() throws SQLException {
-
+        throw new SQLFeatureNotSupportedException("clearWarnings not supported.");
     }
 
     @Override
@@ -209,22 +214,26 @@ public class RestfulConnection implements Connection {
 
     @Override
     public Clob createClob() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public Blob createBlob() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public NClob createNClob() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public SQLXML createSQLXML() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
@@ -254,12 +263,14 @@ public class RestfulConnection implements Connection {
 
     @Override
     public Array createArrayOf(String typeName, Object[] elements) throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public Struct createStruct(String typeName, Object[] attributes) throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
@@ -289,12 +300,16 @@ public class RestfulConnection implements Connection {
 
     @Override
     public <T> T unwrap(Class<T> iface) throws SQLException {
-        return null;
+        try {
+            return iface.cast(this);
+        } catch (ClassCastException cce) {
+            throw new SQLException("Unable to unwrap to " + iface.toString());
+        }
     }
 
     @Override
     public boolean isWrapperFor(Class<?> iface) throws SQLException {
-        return false;
+        return iface.isInstance(this);
     }
 
     public String getHost() {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java

@@ -35,7 +35,7 @@ public class RestfulDriver extends AbstractTaosDriver {
         Properties props = parseURL(url, info);
         String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
         String port = props.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "6041");
-        String database = props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME);
+        String database = props.containsKey(TSDBDriver.PROPERTY_KEY_DBNAME) ? props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME) : null;
 
         String loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":"
                 + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/"
@@ -86,6 +86,7 @@ public class RestfulDriver extends AbstractTaosDriver {
 
     @Override
     public Logger getParentLogger() throws SQLFeatureNotSupportedException {
-        return null;
+        //TODO SQLFeatureNotSupportedException
+        throw new SQLFeatureNotSupportedException();
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.taosdata.jdbc.TSDBConstants;
 import com.taosdata.jdbc.rs.util.HttpClientPoolUtil;
+import com.taosdata.jdbc.utils.SqlSyntaxValidator;
 
 import java.sql.*;
 import java.util.Arrays;
@@ -11,19 +12,23 @@ import java.util.List;
 
 public class RestfulStatement implements Statement {
 
-    private final String catalog;
+    private boolean closed;
+    private String database;
     private final RestfulConnection conn;
 
-    public RestfulStatement(RestfulConnection c, String catalog) {
+    public RestfulStatement(RestfulConnection c, String database) {
         this.conn = c;
-        this.catalog = catalog;
+        this.database = database;
     }
 
     @Override
     public ResultSet executeQuery(String sql) throws SQLException {
+        if (isClosed())
+            throw new SQLException("statement already closed");
+        if (!SqlSyntaxValidator.isSelectSql(sql))
+            throw new SQLException("not a select sql for executeQuery: " + sql);
 
-        final String url = "http://" + conn.getHost() + ":"+conn.getPort()+"/rest/sql";
+        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-
         String result = HttpClientPoolUtil.execute(url, sql);
         String fields = "";
         List<String> words = Arrays.asList(sql.split(" "));
@@ -65,12 +70,29 @@ public class RestfulStatement implements Statement {
 
     @Override
     public int executeUpdate(String sql) throws SQLException {
-        return 0;
+        if (isClosed())
+            throw new SQLException("statement already closed");
+        if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
+            throw new SQLException("not a valid sql for executeUpdate: " + sql);
+
+        if (this.database == null)
+            throw new SQLException("Database not specified or available");
+
+        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+        HttpClientPoolUtil.execute(url, "use " + conn.getDatabase());
+        String result = HttpClientPoolUtil.execute(url, sql);
+        JSONObject jsonObject = JSON.parseObject(result);
+        if (jsonObject.getString("status").equals("error")) {
+            throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " +
+                    jsonObject.getString("desc") + "\n" +
+                    "error code: " + jsonObject.getString("code")));
+        }
+        return Integer.parseInt(jsonObject.getString("rows"));
     }
 
     @Override
     public void close() throws SQLException {
-
+        this.closed = true;
     }
 
     @Override
@@ -115,6 +137,7 @@ public class RestfulStatement implements Statement {
 
     @Override
     public SQLWarning getWarnings() throws SQLException {
+        //TODO: getWarnings not Implemented
         return null;
     }
 
@@ -130,7 +153,29 @@ public class RestfulStatement implements Statement {
 
     @Override
     public boolean execute(String sql) throws SQLException {
-        return false;
+        if (isClosed()) {
+            throw new SQLException("Invalid method call on a closed statement.");
+        }
+        //如果执行了use操作应该将当前Statement的catalog设置为新的database
+        if (SqlSyntaxValidator.isUseSql(sql)) {
+            this.database = sql.trim().replace("use", "").trim();
+        }
+        if (this.database == null)
+            throw new SQLException("Database not specified or available");
+
+        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+        // use database
+        HttpClientPoolUtil.execute(url, "use " + conn.getDatabase());
+        // execute sql
+        String result = HttpClientPoolUtil.execute(url, sql);
+        // parse result
+        JSONObject jsonObject = JSON.parseObject(result);
+        if (jsonObject.getString("status").equals("error")) {
+            throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " +
+                    jsonObject.getString("desc") + "\n" +
+                    "error code: " + jsonObject.getString("code")));
+        }
+        return true;
     }
 
     @Override
@@ -245,7 +290,7 @@ public class RestfulStatement implements Statement {
 
     @Override
     public boolean isClosed() throws SQLException {
-        return false;
+        return closed;
     }
 
     @Override
@@ -270,11 +315,15 @@ public class RestfulStatement implements Statement {
 
     @Override
     public <T> T unwrap(Class<T> iface) throws SQLException {
-        return null;
+        try {
+            return iface.cast(this);
+        } catch (ClassCastException cce) {
+            throw new SQLException("Unable to unwrap to " + iface.toString());
+        }
     }
 
     @Override
     public boolean isWrapperFor(Class<?> iface) throws SQLException {
-        return false;
+        return iface.isInstance(this);
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java

@@ -22,6 +22,9 @@ import java.sql.SQLException;
 
 public class SqlSyntaxValidator {
 
+    private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use"};
+    private static final String[] querySQL = {"select"};
+
     private TSDBConnection tsdbConnection;
 
     public SqlSyntaxValidator(Connection connection) {
@@ -34,7 +37,7 @@ public class SqlSyntaxValidator {
         if (tsdbConnection == null || tsdbConnection.isClosed()) {
             throw new SQLException("invalid connection");
         } else {
-            TSDBJNIConnector jniConnector  = tsdbConnection.getConnection();
+            TSDBJNIConnector jniConnector = tsdbConnection.getConnection();
             if (jniConnector == null) {
                 throw new SQLException("jniConnector is null");
             } else {
@@ -43,4 +46,28 @@ public class SqlSyntaxValidator {
         }
         return res;
     }
+
+    public static boolean isValidForExecuteUpdate(String sql) {
+        for (String prefix : updateSQL) {
+            if (sql.trim().toLowerCase().startsWith(prefix))
+                return true;
+        }
+        return false;
+    }
+
+    public static boolean isUseSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(updateSQL[8]) || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*");
+    }
+
+    public static boolean isUpdateSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(updateSQL[1]);
+    }
+
+    public static boolean isInsertSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(updateSQL[0]);
+    }
+
+    public static boolean isSelectSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(querySQL[0]);
+    }
 }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java

@@ -8,33 +8,43 @@ import java.sql.*;
 public class RestfulDriverTest {
 
     @Test
-    public void testCase001() {
+    public void connect() {
-        try {
+
-            Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
-            Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/?user=root&password=taosdata");
-            Statement statement = connection.createStatement();
-            ResultSet resultSet = statement.executeQuery("select * from log.log");
-            ResultSetMetaData metaData = resultSet.getMetaData();
-            while (resultSet.next()) {
-                for (int i = 1; i <= metaData.getColumnCount(); i++) {
-                    String column = metaData.getColumnLabel(i);
-                    String value = resultSet.getString(i);
-                    System.out.print(column + ":" + value + "\t");
-                }
-                System.out.println();
-            }
-            statement.close();
-            connection.close();
-        } catch (SQLException | ClassNotFoundException e) {
-            e.printStackTrace();
-        }
     }
 
     @Test
-    public void testAcceptUrl() throws SQLException {
+    public void acceptsURL() throws SQLException {
         Driver driver = new RestfulDriver();
         boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041");
         Assert.assertTrue(isAccept);
+        isAccept = driver.acceptsURL("jdbc:TAOS://master:6041");
+        Assert.assertFalse(isAccept);
     }
 
+    @Test
+    public void getPropertyInfo() throws SQLException {
+        Driver driver = new RestfulDriver();
+        final String url = "";
+        DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null);
+    }
+
+    @Test
+    public void getMajorVersion() {
+        Assert.assertEquals(2, new RestfulDriver().getMajorVersion());
+    }
+
+    @Test
+    public void getMinorVersion() {
+        Assert.assertEquals(0, new RestfulDriver().getMinorVersion());
+    }
+
+    @Test
+    public void jdbcCompliant() {
+        Assert.assertFalse(new RestfulDriver().jdbcCompliant());
+    }
+
+    @Test(expected = SQLFeatureNotSupportedException.class)
+    public void getParentLogger() throws SQLFeatureNotSupportedException {
+        new RestfulDriver().getParentLogger();
+    }
 }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java

@@ -0,0 +1,108 @@
+package com.taosdata.jdbc.rs;
+
+import org.junit.*;
+import org.junit.runners.MethodSorters;
+
+import java.sql.*;
+import java.util.Random;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class RestfulJDBCTest {
+
+    private Connection connection;
+
+    @Before
+    public void before() throws ClassNotFoundException, SQLException {
+        Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
+        connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata");
+    }
+
+    @After
+    public void after() throws SQLException {
+        if (connection != null)
+            connection.close();
+    }
+
+
+    /**
+     * 查询所有log.log
+     **/
+    @Test
+    public void testCase001() {
+        try {
+            Statement statement = connection.createStatement();
+            ResultSet resultSet = statement.executeQuery("select * from log.log");
+            ResultSetMetaData metaData = resultSet.getMetaData();
+            while (resultSet.next()) {
+                for (int i = 1; i <= metaData.getColumnCount(); i++) {
+                    String column = metaData.getColumnLabel(i);
+                    String value = resultSet.getString(i);
+                    System.out.print(column + ":" + value + "\t");
+                }
+                System.out.println();
+            }
+            statement.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * create database
+     */
+    @Test
+    public void testCase002() {
+        try (Statement stmt = connection.createStatement()) {
+            stmt.execute("drop database if exists restful_test");
+            stmt.execute("create database if not exists restful_test");
+            stmt.execute("use restful_test");
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * create super table
+     ***/
+    @Test
+    public void testCase003() {
+        try (Statement stmt = connection.createStatement()) {
+            stmt.execute("create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)");
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void testCase004() {
+        try (Statement stmt = connection.createStatement()) {
+            for (int i = 1; i <= 100; i++) {
+                stmt.execute("create table t" + i + " using weather tags('beijing', '" + i + "')");
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private Random random = new Random(System.currentTimeMillis());
+
+    @Test
+    public void testCase005() {
+        try (Statement stmt = connection.createStatement()) {
+            int rows = 0;
+            for (int i = 0; i < 10; i++) {
+                for (int j = 1; j <= 100; j++) {
+                    long currentTimeMillis = System.currentTimeMillis();
+                    int affectRows = stmt.executeUpdate("insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")");
+                    Assert.assertEquals(1, affectRows);
+                    rows += affectRows;
+                }
+            }
+            Assert.assertEquals(1000, rows);
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java

@@ -0,0 +1,28 @@
+package com.taosdata.jdbc.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SqlSyntaxValidatorTest {
+
+    @Test
+    public void validateSqlSyntax() {
+    }
+
+    @Test
+    public void isSelectSQL() {
+        Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather"));
+        Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather"));
+        Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather "));
+        Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)"));
+    }
+
+    @Test
+    public void isUseSQL() {
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test"));
+    }
+}

src/connector/python/linux/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python2/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
-
+    res=[]
-    res = []
-
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
@@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
         except ValueError:
             res.append(None)
 
+    return res 
+
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
     return res
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +158,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -227,8 +270,8 @@ class CTaosInterface(object):
             print('connect to TDengine failed')
             raise ConnectionError("connect to TDengine failed")
             # sys.exit(1)
-        else:
+        #else:
-            print('connect to TDengine success')
+        #    print('connect to TDengine success')
 
         return connection
 
@@ -237,7 +280,7 @@ class CTaosInterface(object):
         '''Close the TDengine handle
         '''
         CTaosInterface.libtaos.taos_close(connection)
-        print('connection is closed')
+        #print('connection is closed')
 
     @staticmethod
     def query(connection, sql):
@@ -310,6 +353,24 @@ class CTaosInterface(object):
 
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 
@@ -393,7 +454,7 @@ class CTaosInterface(object):
     def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(result)
+        return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8')
 
 
 if __name__ == '__main__':
@@ -410,4 +471,4 @@ if __name__ == '__main__':
     print(data)
 
     cinter.freeResult(result)
-    cinter.close(conn)
+    cinter.close(conn)

src/connector/python/linux/python2/taos/cursor.py

@@ -49,7 +49,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(
+            block, self._block_rows = CTaosInterface.fetchRow(
                 self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
@@ -190,6 +190,23 @@ class TDengineCursor(object):
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
 
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0:
+                break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
@@ -203,7 +220,6 @@ class TDengineCursor(object):
             for i in range(len(self._fields)):
                 buffer[i].extend(block[i])
         return list(map(tuple, zip(*buffer)))
-
     def nextset(self):
         """
         """

src/connector/python/linux/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python3/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
-
+    res=[]
-    res = []
-
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
@@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
         except ValueError:
             res.append(None)
 
+    return res 
+
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
     return res
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +158,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -253,7 +296,7 @@ class CTaosInterface(object):
             raise AttributeError("sql is expected as a string")
         # finally:
         #     CTaosInterface.libtaos.close(connection)
-      
+
     @staticmethod
     def affectedRows(result):
         """The affected rows after runing query
@@ -308,29 +351,26 @@ class CTaosInterface(object):
 
         return fields
 
-    # @staticmethod
-    # def fetchBlock(result, fields):
-    #     pblock = ctypes.c_void_p(0)
-    #     num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
-    #         result, ctypes.byref(pblock))
-    #     if num_of_rows == 0:
-    #         return None, 0
-
-    #     isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
-    #     blocks = [None] * len(fields)
-    #     fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
-    #     fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
-    #     for i in range(len(fields)):
-    #         data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
-
-    #         if fields[i]['type'] not in _CONVERT_FUNC:
-    #             raise DatabaseError("Invalid data type returned from database")
-    #         print('====================',fieldLen[i])
-    #         blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
-
-    #     return blocks, abs(num_of_rows)
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 
@@ -350,6 +390,7 @@ class CTaosInterface(object):
         else:
             return None, 0
         return blocks, abs(num_of_rows)
+
     @staticmethod
     def freeResult(result):
         CTaosInterface.libtaos.taos_free_result(result)

src/connector/python/linux/python3/taos/cursor.py

@@ -5,7 +5,6 @@ import threading
 
 # querySeqNum = 0
 
-
 class TDengineCursor(object):
     """Database cursor which is used to manage the context of a fetch operation.
 
@@ -51,7 +50,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(
+            block, self._block_rows = CTaosInterface.fetchRow(
                 self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
@@ -196,18 +195,13 @@ class TDengineCursor(object):
     def fetchall(self):
         """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.
         """
-        # if threading.get_ident() != self._threadId:
-        #     info ="[WARNING] Cursor fetchall:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
-        #     raise OperationalError(info)
-            # print(info)
-            # return None
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
 
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
-            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
             errno = CTaosInterface.libtaos.taos_errno(self._result)
             if errno != 0:
                 raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
@@ -218,6 +212,22 @@ class TDengineCursor(object):
                 buffer[i].extend(block[i])
         return list(map(tuple, zip(*buffer)))
 
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0: break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
     def nextset(self):
         """
         """

src/connector/python/windows/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python2/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
-
+    res=[]
-    res = []
-
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
@@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
         except ValueError:
             res.append(None)
 
+    return res 
+
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
     return res
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +158,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -225,9 +268,10 @@ class CTaosInterface(object):
 
         if connection.value == None:
             print('connect to TDengine failed')
+            raise ConnectionError("connect to TDengine failed")
             # sys.exit(1)
-        else:
+        #else:
-            print('connect to TDengine success')
+        #    print('connect to TDengine success')
 
         return connection
 
@@ -236,7 +280,7 @@ class CTaosInterface(object):
         '''Close the TDengine handle
         '''
         CTaosInterface.libtaos.taos_close(connection)
-        print('connection is closed')
+        #print('connection is closed')
 
     @staticmethod
     def query(connection, sql):
@@ -252,7 +296,7 @@ class CTaosInterface(object):
             raise AttributeError("sql is expected as a string")
         # finally:
         #     CTaosInterface.libtaos.close(connection)
-
+      
     @staticmethod
     def affectedRows(result):
         """The affected rows after runing query
@@ -309,6 +353,24 @@ class CTaosInterface(object):
 
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 
@@ -393,7 +455,7 @@ class CTaosInterface(object):
     def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(result)
+        return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8')
 
 
 if __name__ == '__main__':
@@ -410,4 +472,4 @@ if __name__ == '__main__':
     print(data)
 
     cinter.freeResult(result)
-    cinter.close(conn)
+    cinter.close(conn)

src/connector/python/windows/python2/taos/cursor.py

@@ -50,7 +50,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
             self._block = list(map(tuple, zip(*block)))
@@ -143,7 +143,25 @@ class TDengineCursor(object):
         """
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
+
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0:
+                break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
         
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
@@ -178,7 +196,7 @@ class TDengineCursor(object):
         self._description = None
         self._rowcount = -1
         if self._result is not None:
-            CTaosInterface.freeResult(self._result)        
+            CTaosInterface.freeResult(self._result)
         self._result = None
         self._fields = None
         self._block = None

src/connector/python/windows/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python3/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -104,16 +104,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
             res.append(None)
 
     return res
-    # if num_of_rows > 0:
+    
-    #     for i in range(abs(num_of_rows)):
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
-    #         try:
+    """Function to convert C binary row to python row
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+    """
-    #         except ValueError:
+    res=[]
-    #             res.append(None)
+    if num_of_rows > 0:
-    #     return res
+        for i in range(abs(num_of_rows)):
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]]
+            try:
-    # else:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
+    return res
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +160,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -227,8 +272,8 @@ class CTaosInterface(object):
             print('connect to TDengine failed')
             raise ConnectionError("connect to TDengine failed")
             # sys.exit(1)
-        else:
+        #else:
-            print('connect to TDengine success')
+        #    print('connect to TDengine success')
 
         return connection
 
@@ -237,7 +282,7 @@ class CTaosInterface(object):
         '''Close the TDengine handle
         '''
         CTaosInterface.libtaos.taos_close(connection)
-        print('connection is closed')
+        #print('connection is closed')
 
     @staticmethod
     def query(connection, sql):
@@ -310,6 +355,24 @@ class CTaosInterface(object):
 
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 

src/connector/python/windows/python3/taos/cursor.py

@@ -51,7 +51,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
             self._block = list(map(tuple, zip(*block)))
@@ -144,7 +144,25 @@ class TDengineCursor(object):
         """
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
-        
+
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0:
+                break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
+
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:

src/inc/taoserror.h

@@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "CPU cores
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "Invalid Sync Configuration")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "Sync module not enabled")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION,          0, 0x0902, "Invalid Sync version")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED,          0, 0x0903, "Sync confirm expired")
 
 // wal
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")
@@ -365,7 +366,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG,   0, 0x11A4, "tag value
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL,           0, 0x11A5, "value not find")
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE,           0, 0x11A6, "value type should be boolean, number or string")
 
-
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM,                     0, 0x2100, "out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM,       0, 0x2101, "convertion not a valid literal input")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF,              0, 0x2102, "convertion undefined")

src/inc/taosmsg.h

@@ -464,6 +464,8 @@ typedef struct STimeWindow {
 
 typedef struct {
   SMsgHead    head;
+  char        version[TSDB_VERSION_LEN];
+
   STimeWindow window;
   int32_t     numOfTables;
   int16_t     order;

src/kit/taosdump/taosdump.c

@@ -2146,7 +2146,7 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
   char *    line     = NULL;
   size_t    line_len = 0;
 
-  cmd  = (char *)malloc(COMMAND_SIZE);
+  cmd  = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN);
   if (cmd == NULL) {
     fprintf(stderr, "failed to allocate memory\n");
     return -1;
@@ -2155,7 +2155,7 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
   int lineNo = 0;
   while ((read_len = getline(&line, &line_len, fp)) != -1) {
     ++lineNo;
-    if (read_len >= COMMAND_SIZE) continue;
+    if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
     line[--read_len] = '\0';
 
     //if (read_len == 0 || isCommentLine(line)) {  // line starts with #
@@ -2176,7 +2176,7 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
       fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
     }
 
-    memset(cmd, 0, COMMAND_SIZE);
+    memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN);
     cmd_len = 0;
   }
 

src/mnode/src/mnodeSdb.c

@@ -107,7 +107,7 @@ static taos_queue tsSdbWQueue;
 static SSdbWorkerPool tsSdbPool;
 
 static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused);
-static int32_t sdbWriteWalToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam);
+static int32_t sdbWriteFwdToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam);
 static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action);
 static void    sdbFreeFromQueue(SSdbRow *pRow);
 static void *  sdbWorkerFp(void *pWorker);
@@ -372,7 +372,7 @@ void sdbUpdateSync(void *pMnodes) {
   sprintf(syncInfo.path, "%s", tsMnodeDir);
   syncInfo.getWalInfo = sdbGetWalInfo;
   syncInfo.getFileInfo = sdbGetFileInfo;
-  syncInfo.writeToCache = sdbWriteWalToQueue;
+  syncInfo.writeToCache = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
   syncInfo.notifyRole = sdbNotifyRole;
   tsSdbMgmt.cfg = syncCfg;
@@ -559,7 +559,36 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) {
+static int32_t sdbPerformInsertAction(SWalHead *pHead, SSdbTable *pTable) {
+  SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable};
+  (*pTable->fpDecode)(&row);
+  return sdbInsertHash(pTable, &row);
+}
+
+static int32_t sdbPerformDeleteAction(SWalHead *pHead, SSdbTable *pTable) {
+  void *pObj = sdbGetRowMeta(pTable, pHead->cont);
+  if (pObj == NULL) {
+    sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name,
+             sdbGetKeyStr(pTable, pHead->cont));
+    return TSDB_CODE_SUCCESS;
+  }
+  SSdbRow row = {.pTable = pTable, .pObj = pObj};
+  return sdbDeleteHash(pTable, &row);
+}
+
+static int32_t sdbPerformUpdateAction(SWalHead *pHead, SSdbTable *pTable) {
+  void *pObj = sdbGetRowMeta(pTable, pHead->cont);
+  if (pObj == NULL) {
+    sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name,
+             sdbGetKeyStr(pTable, pHead->cont));
+    return TSDB_CODE_SUCCESS;
+  }
+  SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable};
+  (*pTable->fpDecode)(&row);
+  return sdbUpdateHash(pTable, &row);
+}
+
+static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) {
   SSdbRow *pRow = wparam;
   SWalHead *pHead = hparam;
   int32_t tableId = pHead->msgType / 10;
@@ -568,6 +597,8 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus
   SSdbTable *pTable = sdbGetTableFromId(tableId);
   assert(pTable != NULL);
 
+  if (qtype == TAOS_QTYPE_QUERY) return sdbPerformDeleteAction(pHead, pTable);
+
   pthread_mutex_lock(&tsSdbMgmt.mutex);
   
   if (pHead->version == 0) {
@@ -627,28 +658,17 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus
 
   // from wal or forward msg, row not created, should add into hash
   if (action == SDB_ACTION_INSERT) {
-    SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable};
+    return sdbPerformInsertAction(pHead, pTable);
-    code = (*pTable->fpDecode)(&row);
-    return sdbInsertHash(pTable, &row);
   } else if (action == SDB_ACTION_DELETE) {
-    void *pObj = sdbGetRowMeta(pTable, pHead->cont);
+    if (qtype == TAOS_QTYPE_FWD) {
-    if (pObj == NULL) {
+      // Drop database/stable may take a long time and cause a timeout, so we confirm first then reput it into queue
-      sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name,
+      sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused);
-               sdbGetKeyStr(pTable, pHead->cont));
       return TSDB_CODE_SUCCESS;
+    } else {
+      return sdbPerformDeleteAction(pHead, pTable);
     }
-    SSdbRow row = {.pTable = pTable, .pObj = pObj};
-    return sdbDeleteHash(pTable, &row);
   } else if (action == SDB_ACTION_UPDATE) {
-    void *pObj = sdbGetRowMeta(pTable, pHead->cont);
+    return sdbPerformUpdateAction(pHead, pTable);
-    if (pObj == NULL) {
-      sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name,
-               sdbGetKeyStr(pTable, pHead->cont));
-      return TSDB_CODE_SUCCESS;
-    }
-    SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable};
-    code = (*pTable->fpDecode)(&row);
-    return sdbUpdateHash(pTable, &row);
   } else {
     return TSDB_CODE_MND_INVALID_MSG_TYPE;
   }
@@ -966,7 +986,7 @@ static void sdbFreeFromQueue(SSdbRow *pRow) {
   taosFreeQitem(pRow);
 }
 
-static int32_t sdbWriteWalToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
+static int32_t sdbWriteFwdToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
   SWalHead *pHead = wparam;
 
   int32_t  size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len;

src/query/inc/qUtil.h

@@ -26,7 +26,7 @@
 
 int32_t getOutputInterResultBufSize(SQuery* pQuery);
 
-void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow);
+void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type);
 void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src);
 SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
 
@@ -80,5 +80,8 @@ void* destroyResultRowPool(SResultRowPool* p);
 int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
 int32_t getNumOfUsedResultRows(SResultRowPool* p);
 
+uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv);
+bool isPointInterpoQuery(SQuery *pQuery);
+
 
 #endif  // TDENGINE_QUERYUTIL_H

src/query/inc/sql.y

@@ -556,11 +556,11 @@ having_opt(A) ::= HAVING expr(X).   {A = X;}
 //limit-offset subclause
 %type limit_opt {SLimitVal}
 limit_opt(A) ::= .                     {A.limit = -1; A.offset = 0;}
-limit_opt(A) ::= LIMIT signed(X).      {printf("aa1, %d\n", X); A.limit = X;  A.offset = 0;}
+limit_opt(A) ::= LIMIT signed(X).      {A.limit = X;  A.offset = 0;}
 limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y).
-                                       {printf("aa2\n, %d\n", X); A.limit = X;  A.offset = Y;}
+                                       { A.limit = X;  A.offset = Y;}
 limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y).
-                                       {printf("aa3\n, %d\n", X); A.limit = Y;  A.offset = X;}
+                                       { A.limit = Y;  A.offset = X;}
 
 %type slimit_opt {SLimitVal}
 slimit_opt(A) ::= .                    {A.limit = -1; A.offset = 0;}
@@ -581,7 +581,7 @@ where_opt(A) ::= WHERE expr(X).       {A = X;}
 %type expr {tSQLExpr*}
 %destructor expr {tSQLExprDestroy($$);}
 
-expr(A) ::= LP expr(X) RP.       {A = X; }
+expr(A) ::= LP(X) expr(Y) RP(Z).       {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);}
 
 expr(A) ::= ID(X).               {A = tSQLExprIdValueCreate(&X, TK_ID);}
 expr(A) ::= ID(X) DOT ID(Y).     {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);}

src/query/src/qExecutor.c

@@ -186,7 +186,7 @@ static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData,
 
 static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo);
-static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
+static void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static bool hasMainOutput(SQuery *pQuery);
 static void buildTagQueryResult(SQInfo *pQInfo);
 
@@ -283,8 +283,8 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
   }
 }
 
-static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) {
+static int32_t getMergeResultGroupId(int32_t groupIndex) {
-  int32_t base = 20000000;
+  int32_t base = 50000000;
   return base + (groupIndex * 10000);
 }
 
@@ -947,10 +947,10 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
   if (functionId == TSDB_FUNC_ARITHM) {
     sas->pArithExpr = &pQuery->pExpr1[col];
 
-    sas->offset  = 0;
+    sas->offset    = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1);
-    sas->colList = pQuery->colList;
+    sas->colList   = pQuery->colList;
     sas->numOfCols = pQuery->numOfCols;
-    sas->data    = calloc(pQuery->numOfCols, POINTER_BYTES);
+    sas->data      = calloc(pQuery->numOfCols, POINTER_BYTES);
 
     if (sas->data == NULL) {
       longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -1115,7 +1115,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
 
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
 
-  // not assign result buffer yet, add new result buffer
+  // not assign result buffer yet, add new result buffer, TODO remove it
   char* d = pData;
   int16_t len = bytes;
   if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) {
@@ -1128,7 +1128,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
     longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
   }
 
-  uint64_t uid = groupIndex; // uid is always set to be 0.
+  uint64_t uid = groupIndex;
   SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid);
   if (pResultRow == NULL) {
     return -1;
@@ -1714,7 +1714,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
   // if it is group by normal column, do not set output buffer, the output buffer is pResult
   // fixed output query/multi-output query for normal table
   if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) {
-    resetCtxOutputBuf(pRuntimeEnv);
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
   }
 
   if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) {
@@ -1825,7 +1825,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) {
 }
 
 // todo refactor with isLastRowQuery
-static bool isPointInterpoQuery(SQuery *pQuery) {
+bool isPointInterpoQuery(SQuery *pQuery) {
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     int32_t functionID = pQuery->pExpr1[i].base.functionId;
     if (functionID == TSDB_FUNC_INTERP) {
@@ -2936,10 +2936,24 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   assert(size == pGroupResInfo->numOfDataPages);
 
   bool done = false;
+
+  //TODO add API for release none-dirty pages
+//  SPageInfo* prev = NULL;
+
   for (int32_t j = pGroupResInfo->pageId; j < size; ++j) {
     SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
     tFilePage* pData = getResBufPage(pResultBuf, pi->pageId);
 
+    // release previous buffer pages
+//    if (prev == NULL) {
+//      prev = pi;
+//    } else {
+//      if (prev->pageId != pi->pageId) {
+//        releaseResBufPageInfo(pResultBuf, prev);
+//        prev = pi;
+//      }
+//    }
+
     assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num);
     int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId);
 
@@ -3058,7 +3072,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
   SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool);
   resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
 
-  pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex);
+  pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex);
 
   // todo add windowRes iterator
   int64_t lastTimestamp = -1;
@@ -3339,12 +3353,12 @@ int32_t initResultRow(SResultRow *pResultRow) {
   return TSDB_CODE_SUCCESS;
 }
 
-void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
+void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  int32_t groupIndex = 0;
+  int32_t tid = 0;
-  int32_t uid = 0;
+  int64_t uid = getResultInfoUId(pRuntimeEnv);
-  SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid);
+  SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid);
 
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
@@ -3427,7 +3441,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
     pQuery->limit.offset -= pQuery->rec.rows;
     pQuery->rec.rows = 0;
 
-    resetCtxOutputBuf(pRuntimeEnv);
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
 
     // clear the buffer full flag if exists
     CLEAR_QUERY_STATUS(pQuery, QUERY_RESBUF_FULL);
@@ -3792,7 +3806,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
     return;
   }
 
-  uint64_t uid = 0; // uid is always set to be 0
+  uint64_t uid = getResultInfoUId(pRuntimeEnv);
   SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
       sizeof(groupIndex), true, uid);
   if (pResultRow == NULL) {
@@ -4629,10 +4643,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   int32_t ps = DEFAULT_PAGE_SIZE;
   int32_t rowsize = 0;
   getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
-  int32_t TWOMB = 1024*1024*2;
+  int32_t TENMB = 1024*1024*10;
 
   if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) {
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -4660,7 +4674,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) {
     int32_t numOfResultRows = getInitialPageNum(pQInfo);
     getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -4930,7 +4944,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
   size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
 
   if (isPointInterpoQuery(pQuery)) {
-    resetCtxOutputBuf(pRuntimeEnv);
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
     assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
 
     while (pQInfo->groupIndex < numOfGroups) {
@@ -5096,7 +5110,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       return;
     }
 
-    resetCtxOutputBuf(pRuntimeEnv);
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
     resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo);
 
     SArray *group = GET_TABLEGROUP(pQInfo, 0);
@@ -5456,7 +5470,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
 
   // for ts_comp query, re-initialized is not allowed
   if (!isTSCompQuery(pQuery)) {
-    resetCtxOutputBuf(pRuntimeEnv);
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
   }
 
   // skip blocks without load the actual data block from file if no filter condition present
@@ -5486,7 +5500,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
     qDebug("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64,
            pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey);
 
-    resetCtxOutputBuf(pRuntimeEnv);
+    resetDefaultResInfoOutputBuf(pRuntimeEnv);
   }
 
   limitResults(pRuntimeEnv);
@@ -5811,6 +5825,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
                                char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) {
   int32_t code = TSDB_CODE_SUCCESS;
 
+  if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) {
+    return TSDB_CODE_QRY_INVALID_MSG;
+  }
+
   pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables);
 
   pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey);
@@ -6796,7 +6814,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
 
       qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s);
       if (lseek(fd, 0, SEEK_SET) >= 0) {
-        size_t sz = read(fd, data, (uint32_t)s);
+        size_t sz = read(fd, data, (uint32_t) s);
         if(sz < s) {  // todo handle error
           assert(0);
         }

src/query/src/qParserImpl.c

@@ -187,7 +187,8 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
     pExpr->token.type = pLeft->token.type;
   }
 
-  if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) {
+  if ((pLeft != NULL && pRight != NULL) &&
+      (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) {
     /*
      * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond
      * Otherwise, the time precision is adaptive, determined by the time precision from databases.

src/query/src/qResultbuf.c

@@ -119,8 +119,11 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
     pg->info.offset = allocatePositionInFile(pResultBuf, size);
     pResultBuf->nextPos += size;
 
-    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
-    /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file);
+    assert(ret == 0);
+
+    ret = (int32_t) fwrite(t, 1, size, pResultBuf->file);
+    assert(ret == size);
 
     if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
       pResultBuf->fileSize = pg->info.offset + pg->info.length;
@@ -407,7 +410,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
   }
 
   if (pResultBuf->file != NULL) {
-    qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f",
+    qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb",
         pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
         pResultBuf->fileSize/1024.0);
 

src/query/src/qTsbuf.c

@@ -341,8 +341,10 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
     pBlock->tag.pz = tp;
 
     sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
-  } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
+    UNUSED(sz);
+  } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
     sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
+    UNUSED(sz);
   }
 
   sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);

src/query/src/qUtil.c

@@ -20,6 +20,18 @@
 #include "qExecutor.h"
 #include "qUtil.h"
 
+static int32_t getResultRowKeyInfo(SResultRow* pResult, int16_t type, char** key, int16_t* bytes) {
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+    *key   = varDataVal(pResult->key);
+    *bytes = varDataLen(pResult->key);
+  } else {
+    *key = (char*) &pResult->win.skey;
+    *bytes = tDataTypeDesc[type].nSize;
+  }
+
+  return 0;
+}
+
 int32_t getOutputInterResultBufSize(SQuery* pQuery) {
   int32_t size = 0;
 
@@ -56,6 +68,12 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
     assert(pWindowResInfo->pResult == NULL);
     return;
   }
+
+  if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) {
+    for(int32_t i = 0; i < pWindowResInfo->size; ++i) {
+      tfree(pWindowResInfo->pResult[i]->key);
+    }
+  }
   
   tfree(pWindowResInfo->pResult);
 }
@@ -69,7 +87,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
 
   for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
     SResultRow *pWindowRes = pWindowResInfo->pResult[i];
-    clearResultRow(pRuntimeEnv, pWindowRes);
+    clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
 
     int32_t groupIndex = 0;
     int64_t uid = 0;
@@ -94,12 +112,8 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
   assert(num >= 0 && num <= numOfClosed);
 
-  int16_t  type = pWindowResInfo->type;
+  int16_t type = pWindowResInfo->type;
-  STableId* id  = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0.
+  int64_t uid = getResultInfoUId(pRuntimeEnv);
-  int64_t uid = id->uid;
-  if (pRuntimeEnv->groupbyNormalCol) {
-    uid = 0;
-  }
 
   char    *key  = NULL;
   int16_t  bytes = -1;
@@ -107,16 +121,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
     SResultRow *pResult = pWindowResInfo->pResult[i];
     if (pResult->closed) {  // remove the window slot from hash table
-
+      getResultRowKeyInfo(pResult, type, &key, &bytes);
-      // todo refactor
-      if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-        key   = varDataVal(pResult->key);
-        bytes = varDataLen(pResult->key);
-      } else {
-        key = (char*) &pResult->win.skey;
-        bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
-      }
-
       SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
       taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
     } else {
@@ -134,23 +139,16 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   // move the unclosed window in the front of the window list
   for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
     SResultRow *pWindowRes = pWindowResInfo->pResult[k];
-    clearResultRow(pRuntimeEnv, pWindowRes);
+    clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
   }
   
   pWindowResInfo->size = remain;
 
   for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
     SResultRow *pResult = pWindowResInfo->pResult[k];
-
+    getResultRowKeyInfo(pResult, type, &key, &bytes);
-    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-      key = varDataVal(pResult->key);
-      bytes = varDataLen(pResult->key);
-    } else {
-      key = (char*) &pResult->win.skey;
-      bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
-    }
-
     SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
+
     int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
     assert(p != NULL); 
 
@@ -237,7 +235,7 @@ void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
   getResultRow(pWindowResInfo, slot)->closed = true;
 }
 
-void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
+void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) {
   if (pWindowRes == NULL) {
     return;
   }
@@ -261,7 +259,12 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
   pWindowRes->pageId = -1;
   pWindowRes->rowId = -1;
   pWindowRes->closed = false;
-  pWindowRes->win = TSWINDOW_INITIALIZER;
+
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+    tfree(pWindowRes->key);
+  } else {
+    pWindowRes->win = TSWINDOW_INITIALIZER;
+  }
 }
 
 /**
@@ -379,3 +382,18 @@ void* destroyResultRowPool(SResultRowPool* p) {
   tfree(p);
   return NULL;
 }
+
+uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) {
+  if (!pRuntimeEnv->stableQuery) {
+    return 0;  // for simple table query, the uid is always set to be 0;
+  }
+
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+  if ((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) ||
+      pRuntimeEnv->groupbyNormalCol) {
+    return 0;
+  }
+
+  STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current);
+  return id->uid;
+}

src/query/src/sql.c

@@ -2628,16 +2628,14 @@ static void yy_reduce(
 {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
         break;
       case 177: /* limit_opt ::= LIMIT signed */
+      case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181);
 {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
         break;
       case 178: /* limit_opt ::= LIMIT signed OFFSET signed */
-{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
+{ yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
         break;
       case 179: /* limit_opt ::= LIMIT signed COMMA signed */
-{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
+{ yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
-        break;
-      case 181: /* slimit_opt ::= SLIMIT signed */
-{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
         break;
       case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */
 {yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
@@ -2646,7 +2644,8 @@ static void yy_reduce(
 {yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
         break;
       case 186: /* expr ::= LP expr RP */
-{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; }
+{yylhsminor.yy64 = yymsp[-1].minor.yy64; yylhsminor.yy64->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy64->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);}
+  yymsp[-2].minor.yy64 = yylhsminor.yy64;
         break;
       case 187: /* expr ::= ID */
 {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);}

src/sync/inc/syncInt.h

@@ -28,19 +28,21 @@ extern "C" {
 #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
 
 typedef enum {
-  TAOS_SMSG_SYNC_DATA   = 1,
+  TAOS_SMSG_SYNC_DATA     = 1,
-  TAOS_SMSG_FORWARD     = 2,
+  TAOS_SMSG_FORWARD       = 2,
-  TAOS_SMSG_FORWARD_RSP = 3,
+  TAOS_SMSG_FORWARD_RSP   = 3,
-  TAOS_SMSG_SYNC_REQ    = 4,
+  TAOS_SMSG_SYNC_REQ      = 4,
-  TAOS_SMSG_SYNC_RSP    = 5,
+  TAOS_SMSG_SYNC_RSP      = 5,
-  TAOS_SMSG_SYNC_MUST   = 6,
+  TAOS_SMSG_SYNC_MUST     = 6,
-  TAOS_SMSG_STATUS      = 7
+  TAOS_SMSG_STATUS        = 7,
+  TAOS_SMSG_SYNC_DATA_RSP = 8,
 } ESyncMsgType;
 
 #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
 #define SYNC_RECV_BUFFER_SIZE (5*1024*1024)
 #define SYNC_FWD_TIMER  300
 #define SYNC_ROLE_TIMER 10000
+#define SYNC_WAIT_AFTER_CHOOSE_MASTER 3
 
 #define nodeRole    pNode->peerInfo[pNode->selfIndex]->role
 #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
@@ -64,6 +66,10 @@ typedef struct {
   int32_t   sourceId;  // only for arbitrator
 } SFirstPkt;
 
+typedef struct {
+  int8_t sync;
+} SFirstPktRsp;
+
 typedef struct {
   int8_t    role;
   uint64_t  version;

src/sync/src/syncMain.c

@@ -626,17 +626,9 @@ static void syncChooseMaster(SSyncNode *pNode) {
       sInfo("vgId:%d, start to work as master", pNode->vgId);
       nodeRole = TAOS_SYNC_ROLE_MASTER;
 
-#if 0
+      // Wait for other nodes to receive status to avoid version inconsistency
-      for (int32_t i = 0; i < pNode->replica; ++i) {
+      taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER);
-        if (i == index) continue;
+
-        pPeer = pNode->peerInfo[i];
-        if (pPeer->version == nodeVersion) {
-          pPeer->role = TAOS_SYNC_ROLE_SLAVE;
-          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
-          sInfo("%s, it shall work as slave", pPeer->id);
-        }
-      }
-#endif
       syncResetFlowCtrl(pNode);
       (*pNode->notifyRole)(pNode->vgId, nodeRole);
     } else {
@@ -761,7 +753,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
       sDebug("vgId:%d, choose master", pNode->vgId);
       syncChooseMaster(pNode);
     } else {
-      sDebug("vgId:%d, cannot choose master since roles inconsistent", pNode->vgId);
+      sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId);
     }
   }
 
@@ -1124,7 +1116,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
   }
 
   int32_t     vgId = firstPkt.syncHead.vgId;
-  SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t));
+  SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t));
   if (ppNode == NULL || *ppNode == NULL) {
     sError("vgId:%d, vgId could not be found", vgId);
     taosCloseSocket(connFd);
@@ -1287,7 +1279,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
 
         sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId,
                pFwdInfo->version, time, pFwdInfo->time);
-        syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+        syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_SYN_CONFIRM_EXPIRED);
       }
 
       syncRemoveConfirmedFwdInfo(pNode);
@@ -1321,6 +1313,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
   }
 
   // always update version
+  sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
+         syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
   nodeVersion = pWalHead->version;
 
   if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
@@ -1328,10 +1322,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
   // only pkt from RPC or CQ can be forwarded
   if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
 
-  sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
+    // a hacker way to improve the performance
-         syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
-
-  // a hacker way to improve the performance
   pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead));
   pSyncHead->type = TAOS_SMSG_FORWARD;
   pSyncHead->pversion = 0;
@@ -1352,9 +1343,11 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
 
     int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen);
     if (retLen == fwdLen) {
-      sTrace("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+      sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role],
+             syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len);
     } else {
-      sError("%s, failed to forward, hver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+      sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, syncRole[pPeer->role],
+             syncStatus[pPeer->sstatus], pWalHead->version, retLen);
       syncRestartConnection(pPeer);
     }
   }

src/sync/src/syncRestore.c

@@ -147,12 +147,10 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
 static int32_t syncRestoreWal(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
   int32_t    ret, code = -1;
+  uint64_t   lastVer = 0;
 
-  void *buffer = calloc(SYNC_MAX_SIZE, 1);  // size for one record
+  SWalHead *pHead = calloc(SYNC_MAX_SIZE, 1);  // size for one record
-  if (buffer == NULL) return -1;
+  if (pHead == NULL) return -1;
-
-  SWalHead *pHead = (SWalHead *)buffer;
-  uint64_t lastVer = 0;
 
   while (1) {
     ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
@@ -188,7 +186,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
     sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno));
   }
 
-  free(buffer);
+  free(pHead);
   return code;
 }
 
@@ -233,10 +231,13 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
 int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
   SSyncNode *  pNode = pPeer->pSyncNode;
   SRecvBuffer *pRecv = pNode->pRecv;
-
-  if (pRecv == NULL) return -1;
   int32_t len = pHead->len + sizeof(SWalHead);
 
+  if (pRecv == NULL) {
+    sError("%s, recv buffer is not create yet", pPeer->id);
+    return -1;
+  }
+
   if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
     memcpy(pRecv->offset, pHead, len);
     pRecv->offset += len;
@@ -284,7 +285,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
   nodeSStatus = TAOS_SYNC_STATUS_FILE;
   uint64_t fversion = 0;
 
-  sDebug("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
+  sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
+  SFirstPktRsp firstPktRsp = {.sync = 1};
+  if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) {
+    sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
   int32_t code = syncRestoreFile(pPeer, &fversion);
   if (code < 0) {
     sError("%s, failed to restore file", pPeer->id);
@@ -301,14 +309,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 
   nodeVersion = fversion;
 
-  sDebug("%s, start to restore wal", pPeer->id);
+  sInfo("%s, start to restore wal", pPeer->id);
   if (syncRestoreWal(pPeer) < 0) {
     sError("%s, failed to restore wal", pPeer->id);
     return -1;
   }
 
   nodeSStatus = TAOS_SYNC_STATUS_CACHE;
-  sDebug("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
+  sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
   if (syncProcessBufferedFwd(pPeer) < 0) {
     sError("%s, failed to insert buffered points", pPeer->id);
     return -1;

src/sync/src/syncRetrieve.c

@@ -448,7 +448,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
   return code;
 }
 
-static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
+static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
 
   SFirstPkt firstPkt;
@@ -458,8 +458,24 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
   tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
   firstPkt.port = tsSyncPort;
 
-  if (taosWriteMsg(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) {
+  if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) {
-    sError("%s, failed to send syncCmd", pPeer->id);
+    sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  SFirstPktRsp firstPktRsp;
+  if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) {
+    sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
+  sInfo("%s, start to retrieve, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
+  if (syncRetrieveFirstPkt(pPeer) < 0) {
+    sError("%s, failed to start retrieve", pPeer->id);
     return -1;
   }
 

src/sync/src/taosTcpPool.c

@@ -203,16 +203,19 @@ static void *taosProcessTcpData(void *param) {
       assert(pConn);
 
       if (events[i].events & EPOLLERR) {
+        sDebug("conn is broken since EPOLLERR");
         taosProcessBrokenLink(pConn);
         continue;
       }
 
       if (events[i].events & EPOLLHUP) {
+        sDebug("conn is broken since EPOLLHUP");
         taosProcessBrokenLink(pConn);
         continue;
       }
 
       if (events[i].events & EPOLLRDHUP) {
+        sDebug("conn is broken since EPOLLRDHUP");
         taosProcessBrokenLink(pConn);
         continue;
       }

tests/pytest/fulltest.sh

@@ -158,6 +158,7 @@ python3 ./test.py -f query/bug1471.py
 python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
+python3 ./test.py -f query/bug2218.py 
 
 #stream
 python3 ./test.py -f stream/metric_1.py
@@ -205,7 +206,9 @@ python3 test.py -f query/queryInterval.py
 python3 test.py -f query/queryFillTest.py
 
 # tools
-python3 test.py -f tools/taosdemo.py
+python3 test.py -f tools/taosdemoTest.py
+python3 test.py -f tools/taosdumpTest.py
+python3 test.py -f tools/lowaTest.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py

tests/pytest/insert/restfulInsert.py

@@ -55,9 +55,12 @@ class RestfulInsert:
     def insertUnlimitedData(self, threadID):        
         print("thread %d started" % threadID)
         tablesPerThread = int (self.numOfTables / self.numOfThreads)
+        
+        count = 0
         while True:
             i = 0
-            start = self.ts
+            start = self.ts  + count * self.batchSize          
+            count = count + 1            
             
             for i in range(tablesPerThread):
                 tableID = i + threadID * tablesPerThread
@@ -65,7 +68,7 @@ class RestfulInsert:
                 data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID)
                 values = []
                 for k in range(self.batchSize):
-                    values.append("(%d, %d, %d, %d)" %  (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)))
+                    values.append("(%d, %d, %d, %d)" %  (start + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)))
                 
                 if(self.outOfOrder == False):
                     for k in range(len(values)):            

tests/pytest/query/bug2218.py

@@ -0,0 +1,54 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool)")
+        insertRows = 1000
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(insertRows):
+            ret = tdSql.execute(
+                "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d)" %
+                (t0+i,i%100,i/2,i%100,i%100,i%100,i*1.0,i%2))
+        print("==========step2")
+        print("test col*1*1 desc ")
+        tdSql.query('select c1,c1*1*1,c2*1*1,c3*1*1,c4*1*1,c5*1*1,c6*1*1 from mt0 order by ts  desc limit 2')
+        tdSql.checkData(0,0,99)
+        tdSql.checkData(0,1,0.0)
+        tdSql.checkData(0,2,0.0)
+        tdSql.checkData(0,3,0.0)
+        tdSql.checkData(0,4,0.0)
+        tdSql.checkData(0,5,0.0)
+        tdSql.checkData(0,6,0.0)
+
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/filterOtherTypes.py

@@ -376,11 +376,9 @@ class TDTestCase:
         tdSql.execute("insert into t1 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)")
         tdSql.execute("insert into t2 values(1538548685000, 4) (1538548685001, 5) (1538548685002, 6)")
 
-        tdSql.query("select * from t1 where tag1 like '%g'")
+        tdSql.error("select * from t1 where tag1 like '%g'")        
-        tdSql.checkRows(3)
 
-        tdSql.query("select * from t2 where tag1 like '%g'")
+        tdSql.error("select * from t2 where tag1 like '%g'") 
-        tdSql.checkRows(3)
 
         tdSql.query("select * from meters where tag1 like '%g'")
         tdSql.checkRows(6)
@@ -396,20 +394,16 @@ class TDTestCase:
         tdSql.execute("insert into t5 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)")
         tdSql.execute("insert into t6 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)")
 
-        tdSql.query("select * from t3 where tag1 like '%京'")
+        tdSql.error("select * from t3 where tag1 like '%京'")        
-        tdSql.checkRows(3)
 
-        tdSql.query("select * from t4 where tag1 like '%京'")
+        tdSql.error("select * from t4 where tag1 like '%京'")        
-        tdSql.checkRows(3)
 
         tdSql.query("select * from meters1 where tag1 like '%京'")
         tdSql.checkRows(6)
         
-        tdSql.query("select * from t5 where tag1 like '%g'")
+        tdSql.error("select * from t5 where tag1 like '%g'")        
-        tdSql.checkRows(3)
 
-        tdSql.query("select * from t6 where tag1 like '%g'")
+        tdSql.error("select * from t6 where tag1 like '%g'")        
-        tdSql.checkRows(3)
 
         tdSql.query("select * from meters1 where tag1 like '%g'")
         tdSql.checkRows(6)

tests/pytest/query/queryJoin.py

@@ -175,6 +175,8 @@ class TDTestCase:
         tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts")
         tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3")
         tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;")
+        tdSql.error("select join_mt1.c1,join_mt0.c1 from join_mt1,join_mt0 where join_mt1.ts = join_mt0.ts and join_mt1.t1 = join_mt0.t1 order by t")
+
 
     def stop(self):
         tdSql.close()

tests/pytest/tools/taosdumpTest.py

@@ -0,0 +1,89 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+        self.ts = 1538548685000
+        self.numberOfTables = 10000
+        self.numberOfRecords = 100
+    
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
+        tdSql.execute("create table t1 using st tags(1, 'beijing')")        
+        sql = "insert into t1 values"
+        currts = self.ts
+        for i in range(100):
+            sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
+        tdSql.execute(sql)
+
+        tdSql.execute("create table t2 using st tags(2, 'shanghai')")        
+        sql = "insert into t2 values"
+        currts = self.ts
+        for i in range(100):
+            sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
+        tdSql.execute(sql)
+
+        os.system("taosdump --databases db -o /tmp")
+        
+        tdSql.execute("drop database db")
+        tdSql.query("show databases")
+        tdSql.checkRows(0)
+        
+        os.system("taosdump -i /tmp")
+
+        tdSql.query("show databases")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'db')
+        
+        tdSql.execute("use db")
+        tdSql.query("show stables")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'st')
+
+        tdSql.query("show tables")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, 't2')
+        tdSql.checkData(1, 0, 't1')
+
+        tdSql.query("select * from t1")
+        tdSql.checkRows(100)
+        for i in range(100):
+            tdSql.checkData(i, 1, i)
+            tdSql.checkData(i, 2, "nchar%d" % i)
+
+        tdSql.query("select * from t2")
+        tdSql.checkRows(100)
+        for i in range(100):
+            tdSql.checkData(i, 1, i)
+            tdSql.checkData(i, 2, "nchar%d" % i)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/script/general/parser/col_arithmetic_operation.sim

@@ -5,6 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 500
 sql connect
+#========================================= setup environment ================================
 
 $dbPrefix = ca_db
 $tbPrefix = ca_tb
@@ -28,12 +29,41 @@ sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5
 $i = 0
 $ts = $ts0
 $halfTbNum = $tbNum / 2
-while $i < $halfTbNum
+#while $i < $halfTbNum
-  $tbId = $i + $halfTbNum
+  $t1 = $i + 1
-  $tb = $tbPrefix . $i
+  $t2 = $i + 2
-  $tb1 = $tbPrefix . $tbId
+  $t3 = $i + 3
-  sql create table $tb using $stb tags( $i )
+  $t4 = $i + 4
-  sql create table $tb1 using $stb tags( $tbId )
+
+  $t5 = $i + $halfTbNum
+  $t6 = $t5 + 1
+  $t7 = $t6 + 1
+  $t8 = $t7 + 1
+  $t9 = $t8 + 1
+
+  $tb0 = $tbPrefix . $i
+  $tb1 = $tbPrefix . $t1
+  $tb2 = $tbPrefix . $t2
+  $tb3 = $tbPrefix . $t3
+  $tb4 = $tbPrefix . $t4
+
+  $tb5 = $tbPrefix . $t5
+  $tb6 = $tbPrefix . $t6
+  $tb7 = $tbPrefix . $t7
+  $tb8 = $tbPrefix . $t8
+  $tb9 = $tbPrefix . $t9
+
+  sql create table $tb0 using $stb tags( $i )
+  sql create table $tb1 using $stb tags( $t1 )
+  sql create table $tb2 using $stb tags( $t2 )
+  sql create table $tb3 using $stb tags( $t3 )
+  sql create table $tb4 using $stb tags( $t4 )
+
+  sql create table $tb5 using $stb tags( $t5 )
+  sql create table $tb6 using $stb tags( $t6 )
+  sql create table $tb7 using $stb tags( $t7 )
+  sql create table $tb8 using $stb tags( $t8 )
+  sql create table $tb9 using $stb tags( $t9 )
 
   $x = 0
   while $x < $rowNum
@@ -46,50 +76,61 @@ while $i < $halfTbNum
     $binary = $binary . '
     $nchar = 'nchar . $c
     $nchar = $nchar . '
-    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar )
+    sql insert into $tb0 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb2 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb3 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb4 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
     $x = $x + 1
-  endw 
+  endw
-  $i = $i + 1
+  #$i = $i + 1
-endw 
 
-##### select from table
+  $x = 0
-$tb = $tbPrefix . 0
+  while $x < $rowNum
-## TBASE-344
+    $xs = $x * $delta
-sql select c1*2 from $tb
+    $ts = $ts0 + $xs
-if $rows != $rowNum then
+    $c = $x / 10
-  return -1
+    $c = $c * 10
-endi
+    $c = $x - $c
-if $data00 != 0.000000000 then
+    $binary = 'binary . $c
-  return -1
+    $binary = $binary . '
-endi
+    $nchar = 'nchar . $c
-if $data10 != 2.000000000 then
+    $nchar = $nchar . '
-  return -1
-endi
-if $data20 != 4.000000000 then
-  return -1
-endi
-if $data90 != 18.000000000 then
-  return -1
-endi
 
-sql select c4*1+1/2 from $tb
+    sql insert into $tb5 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb6 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb7 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb8 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb9 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar )
-if $rows != $rowNum then
+    $x = $x + 1
-  return -1
+  endw
-endi
-if $data00 != 0.500000000 then
-  return -1
-endi
-if $data10 != 1.500000000 then
-  return -1
-endi
-if $data90 != 9.500000000 then
-  return -1
-endi
 
-#### illegal operations
+#endw
+
+#=================================== above are setup test environment =============================
+run general/parser/col_arithmetic_query.sim
+
+#======================================= all in files query =======================================
+print ================== restart server to commit data into disk
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 3000
+system sh/exec.sh -n dnode1 -s start
+
+print ================== server restart completed
+sql connect
+sleep 500c
+
+run general/parser/col_arithmetic_query.sim
+
+# ================================================================================================
+
+print ====================> crash
+# sql select spread(ts )/(1000*3600*24) from ca_stb0 interval(1y)
+
+sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y)
+sql_error select first(ts) - last(ts) from stb interval(1y)
+sql_error select top(c1, 2) - last(c1) from stb;
+sql_error select stddev(c1) - last(c1) from stb;
+sql_error select diff(c1) - last(c1) from stb;
+sql_error select first(c7) - last(c7) from stb;
+sql_error select first(c8) - last(c8) from stb;
+sql_error select first(c9) - last(c9) from stb;
 sql_error select max(c2*2) from $tb
 sql_error select max(c1-c2) from $tb
 
+#========================================regression test cases====================================
 print =====================> td-1764
 sql select sum(c1)/count(*), sum(c1) as b, count(*) as b from $stb interval(1y)
 if $rows != 1 then
@@ -108,42 +149,4 @@ if $data02 != 225000 then
   return -1
 endi
 
-sql select first(c1) - last(c1), first(c1) as b, last(c1) as b, min(c1) - max(c1), spread(c1) from ca_stb0 interval(1y)
-if $rows != 1 then
-  return -1
-endi
-
-if $data00 != @18-01-01 00:00:00.000@ then
-  return -1
-endi
-
-if $data01 != -9.000000000 then
-  return -1
-endi
-
-if $data02 != 0 then
-  return -1
-endi
-
-if $data03 != 9 then
-  return -1
-endi
-
-if $data04 != -9.000000000 then
-  return -1
-endi
-
-if $data05 != 9.000000000 then
-  return -1
-endi
-
-sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y)
-sql_error select first(ts) - last(ts) from stb interval(1y)
-sql_error select top(c1, 2) - last(c1) from stb;
-sql_error select stddev(c1) - last(c1) from stb;
-sql_error select diff(c1) - last(c1) from stb;
-sql_error select first(c7) - last(c7) from stb;
-sql_error select first(c8) - last(c8) from stb;
-sql_error select first(c9) - last(c9) from stb;
-
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/general/parser/col_arithmetic_query.sim

@@ -0,0 +1,478 @@
+# ======================================= query test cases ========================================
+# select from table
+
+$dbPrefix = ca_db
+$tbPrefix = ca_tb
+$stbPrefix = ca_stb
+$rowNum = 10000
+
+$i = 0
+$db = $dbPrefix . $i
+sql use $db
+
+$tb = $tbPrefix . 0
+$stb = $stbPrefix . $i
+
+## TBASE-344
+sql select c1*2 from $tb
+if $rows != $rowNum then
+  return -1
+endi
+if $data00 != 0.000000000 then
+  return -1
+endi
+if $data10 != 2.000000000 then
+  return -1
+endi
+if $data20 != 4.000000000 then
+  return -1
+endi
+if $data90 != 18.000000000 then
+  return -1
+endi
+
+# asc/desc order [d.2] ======================================================
+sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc;
+if $rows != 10000 then
+  return -1
+endi
+
+if $data00 != 0.000000000 then
+  return -1
+endi
+
+if $data01 != -nan then
+  print expect -nan, actual: $data01
+  return -1
+endi
+
+if $data10 != 0.666666667 then
+  return -1
+endi
+
+if $data11 != 1.000000000 then
+  return -1
+endi
+
+if $data90 != 6.000000000 then
+  return -1
+endi
+
+if $data91 != 1.000000000 then
+  return -1
+endi
+
+sql select (c1 * 2) % 7.9 from $tb order by ts desc;
+if $rows != 10000 then
+ return -1
+endi
+
+if $data00 != 0.100000000 then
+  print expect 0.100000000, acutal:$data00
+  return -1
+endi
+
+if $data10 != 2.100000000 then
+  return -1
+endi
+
+if $data90 != 6.000000000 then
+ return -1
+endi
+
+# [d.3]
+sql select c1 * c2 /4 from $tb where ts < 1537166000000 and ts > 1537156000000
+if $rows != 17 then
+  return -1
+endi
+
+if $data00 != 12.250000000 then
+  return -1
+endi
+
+if $data10 != 16.000000000 then
+  return -1
+endi
+
+if $data20 != 20.250000000 then
+  print expect 20.250000000, acutal:$data21
+  return -1
+endi
+
+if $data30 != 0.000000000 then
+  return -1
+endi
+
+# no result return [d.3] ==============================================================
+sql select c1 * 91- 7 from $tb where ts < 1537146000000
+if $rows != 0 then
+  return -1
+endi
+
+# no result return [d.3]
+sql select c2 - c2 from $tb where ts > '2018-09-17 12:50:00.000' and ts<'2018-09-17 13:00:00.000'
+if $rows != 0 then
+  return -1
+endi
+
+# single row result aggregation [d.4] =================================================
+# not available
+
+# error cases
+# not available
+
+# multi row result aggregation [d.4]
+sql_error select top(c1, 1) - bottom(c1, 1) from $tb
+sql_error select top(c1, 99) - bottom(c1, 99) from $tb
+sql_error select top(c1,1) - 88 from $tb
+
+# all data types [d.6] ================================================================
+sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb
+if $rows != 10000 then
+ return -1
+endi
+
+if $data00 != 0.000000000 then
+  return -1
+endi
+
+if $data01 != -nan then
+  return -1
+endi
+
+if $data02 != 0.000000000 then
+  return -1
+endi
+
+if $data03 != 0.000000000 then
+  return -1
+endi
+
+if $data04 != 0.000000000 then
+  return -1
+endi
+
+if $data05 != 0.000000000 then
+  return -1
+endi
+
+if $data90 != -0.900000000 then
+  return -1
+endi
+
+if $data91 != 1.000000000 then
+  return -1
+endi
+
+if $data92 != 81.000000000 then
+  return -1
+endi
+
+if $data93 != 0.000000000 then
+  return -1
+endi
+
+if $data94 != 18.000000000 then
+  return -1
+endi
+
+# error case, ts/bool/binary/nchar not support arithmetic expression
+sql_error select ts+ts from $tb
+sql_error select ts+22 from $tb
+sql_error select c7*12 from $tb
+sql_error select c8/55 from $tb
+sql_error select c9+c8 from $tb
+sql_error select c7-c8, c9-c8 from $tb
+sql_error select ts-c9 from $tb
+sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb
+
+# arithmetic expression in join [d.7]==================================================
+
+
+# arithmetic expression in union [d.8]=================================================
+
+
+# arithmetic expression in group by [d.9]==============================================
+# in group by tag, not support for normal table
+sql_error select c5*99 from $tb group by t1
+
+# in group by column
+sql_error select c6-(c6+c3)*12 from $tb group by c3;
+
+
+# limit offset [d.10]==================================================================
+sql select c6 * c1 + 12 from $tb limit 12 offset 99;
+if $rows != 12 then
+ return -1
+endi
+
+if $data00 != 93.000000000 then
+  return -1
+endi
+
+if $data90 != 76.000000000 then
+  return -1
+endi
+
+sql select c4 / 99.123 from $tb limit 10 offset 9999;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 0.090796283 then
+  return -1
+endi
+
+# slimit/soffset not support for normal table query. [d.11]============================
+sql_error select sum(c1) from $tb slimit 1 soffset 19;
+
+# fill [d.12]==========================================================================
+sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12);
+
+# constant column. [d.13]==============================================================
+sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb
+if $rows != 10000 then
+  return -1
+endi
+
+if $data00 != 0 then
+  return -1
+endi
+
+if $data01 != 0.000000000 then
+  return -1
+endi
+
+if $data02 != 12.987654568 then
+  return -1
+endi
+
+if $data03 != 1 then
+  return -1
+endi
+
+if $data04 != 1.100000000 then
+  return -1
+endi
+
+if $data10 != 1 then
+  return -1
+endi
+
+# column value filter [d.14]===========================================================
+sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2
+if $rows != 2000 then
+  return -1
+endi
+
+if $data00 != 0 then
+  return -1
+endi
+
+if $data01 != 0.000000000 then
+  return -1
+endi
+
+if $data02 != 12.987654568 then
+  return -1
+endi
+
+if $data03 != 1 then
+  return -1
+endi
+
+if $data10 != 1 then
+  return -1
+endi
+
+if $data20 != 0 then
+  return -1
+endi
+
+# tag filter(not support for normal table). [d.15]=====================================
+sql_error select c2+99 from $tb where t1=12;
+
+# multi-field output [d.16]============================================================
+sql select c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2 from $tb
+if $rows != $rowNum then
+  return -1
+endi
+if $data00 != 0.500000000 then
+  return -1
+endi
+if $data10 != 1.500000000 then
+  return -1
+endi
+if $data90 != 9.500000000 then
+  return -1
+endi
+
+# interval query [d.17]==================================================================
+sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s)
+sql_error select c7-c9 from $tb interval(2y)
+
+# aggregation query [d.18]===============================================================
+# see test cases below
+
+# first/last query  [d.19]===============================================================
+# see test cases below
+
+# multiple retrieve [d.20]===============================================================
+sql select c2-c2, 911 from $tb
+
+#======================================= aggregation function arithmetic query cases ================
+# asc/desc order [d.2]
+sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 0.000000000 then
+  return -1
+endi
+
+sql select first(c1) * (2/99) from $stb order by ts desc;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 0.000000000 then
+  return -1
+endi
+
+sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb order by ts desc;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1.800000000 then
+  return -1
+endi
+
+if $data01 != 100000 then
+  return -1
+endi
+
+if $data02 != 200000 then
+  return -1
+endi
+
+sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb order by ts asc;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 0.204545455 then
+  return -1
+endi
+
+if $data01 != 9.000000000 then
+  return -1
+endi
+
+if $data02 != 9.000000020 then
+  return -1
+endi
+
+# all possible function in the arithmetic expressioin
+sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2) from $stb where ts <  and ts >
+
+# no result return [d.3]
+sql select first(c1) * 91 - 7, last(c3) from $stb where ts < 1537146000000
+if $rows != 0 then
+  return -1
+endi
+
+# no result return [d.3]
+sql select sum(c2) - avg(c2) from $tb where ts>xxx
+if $rows != 0 then
+  return -1
+endi
+
+# single row result aggregation [d.4]
+sql select
+
+# error cases
+sql_error select first(c1, c2) - last(c1, c2) from $tb
+
+# multi row result aggregation [d.4]
+sql select top(c1, 1) - bottom(c1, 1) from $tb
+sql select top(c1, 99) - bottom(c1, 99) from $tb
+
+# all data types [d.6]
+sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb
+
+# error case, ts/bool/binary/nchar not support arithmetic expression
+sql_error select ts+ts from $tb
+sql_error select ts+22 from $tb
+sql_error select c7*12 from $tb
+sql_error select c8/55 from $tb
+sql_error select c9+c8 from $tb
+
+# arithmetic expression in join [d.7]
+
+
+# arithmetic expression in union [d.8]
+
+
+# arithmetic expression in group by [d.9]
+# in group by tag
+# not support for normal table
+sql_error select c5*99 from $tb group by t1
+
+# in group by column
+sql_error select c6-c6+c3*12 from $tb group by c3;
+
+sql select first(c6) - last(c6) *12 / count(*) from $tb group by c3;
+
+# limit offset [d.10]
+sql select c6-c6+12 from $tb limit 12 offset 99;
+sql select c4/99.123 from $tb limit 1 offset 9999;
+
+# slimit/soffset not suport for normal table query. [d.11]
+sql_error select sum(c1) from $tb slimit 1 soffset 19;
+
+# fill [d.12]
+sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12);
+
+# constant column. [d.13]
+
+
+# column value filter [d.14]
+
+
+# tag filter(not support for normal table). [d.15]
+sql_error select sum(c2)+99 from $tb where t1=12;
+
+# multi-field output [d.16]
+sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb
+
+sql select c4*1+1/2 from $tb
+if $rows != $rowNum then
+  return -1
+endi
+if $data00 != 0.500000000 then
+  return -1
+endi
+if $data10 != 1.500000000 then
+  return -1
+endi
+if $data90 != 9.500000000 then
+  return -1
+endi
+
+# interval query [d.17]
+sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s)
+sql_error select c7-c9 from $tb interval(2y)
+
+# aggregation query [d.18]
+# see test cases below
+
+# first/last query  [d.19]
+# see test cases below
+
+# multiple retrieve [d.20]
+sql select c2-c2 from $tb;
+
+
+sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb

tests/script/tmp/mnodes.sim

@@ -20,6 +20,10 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000
 system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000
 
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
+
 system sh/cfg.sh -n dnode1 -c replica -v 3
 system sh/cfg.sh -n dnode2 -c replica -v 3
 system sh/cfg.sh -n dnode3 -c replica -v 3