From 6e14f9dc8ba8e3a3ef6839f2123973dc33ad38c8 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 9 Nov 2019 14:37:44 +0800 Subject: [PATCH 01/42] Client and server versions may not match --- src/client/src/tscSql.c | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index a2d4abbca3..0e1fead70e 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -138,34 +138,35 @@ TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) { if (taos != NULL) { STscObj* pObj = (STscObj*) taos; - int clientVersionNumber[4] = {0}; - if (!taosGetVersionNumber(version, clientVersionNumber)) { - tscError("taos:%p, invalid client version:%s", taos, version); - //pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; - globalCode = TSDB_CODE_INVALID_CLIENT_VERSION; - taos_close(taos); - return NULL; - } - - char *server_version = taos_get_server_info(taos); - int serverVersionNumber[4] = {0}; - if (!taosGetVersionNumber(server_version, serverVersionNumber)) { - tscError("taos:%p, invalid server version:%s", taos, server_version); - //pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; - globalCode = TSDB_CODE_INVALID_CLIENT_VERSION; - taos_close(taos); - return NULL; - } - // version compare only requires the first 3 segments of the version string int32_t comparedSegments = 3; + char client_version[64] = {0}; + char server_version[64] = {0}; + int clientVersionNumber[4] = {0}; + int serverVersionNumber[4] = {0}; + + strcpy(client_version, version); + strcpy(server_version, taos_get_server_info(taos)); + + if (!taosGetVersionNumber(client_version, clientVersionNumber)) { + tscError("taos:%p, invalid client version:%s", taos, client_version); + pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; + taos_close(taos); + return NULL; + } + + if (!taosGetVersionNumber(server_version, serverVersionNumber)) { + tscError("taos:%p, invalid server version:%s", taos, server_version); + pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; + taos_close(taos); + return NULL; + } for(int32_t i = 0; i < comparedSegments; ++i) { if (clientVersionNumber[i] != serverVersionNumber[i]) { tscError("taos:%p, the %d-th number of server version:%s not matched with client version:%s, close connection", taos, i, server_version, version); - //pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; - globalCode = TSDB_CODE_INVALID_CLIENT_VERSION; + pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; taos_close(taos); return NULL; } From 0d2eb1bf0848bca00377a8ac5515f632c622dfa5 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 15 Nov 2019 18:44:00 +0800 Subject: [PATCH 02/42] [TBASE-1155] --- src/os/darwin/src/tdarwin.c | 4 ---- src/os/linux/src/tsystem.c | 19 ------------------- src/os/windows/src/twindows.c | 4 ---- src/system/src/vnodeImport.c | 3 ++- src/util/src/tutil.c | 19 +++++++++++++++++++ 5 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/os/darwin/src/tdarwin.c b/src/os/darwin/src/tdarwin.c index 2f97c7c376..71e8a56466 100644 --- a/src/os/darwin/src/tdarwin.c +++ b/src/os/darwin/src/tdarwin.c @@ -243,10 +243,6 @@ int taosInitTimer(void (*callback)(int), int ms) { return setitimer(ITIMER_REAL, &tv, NULL); } -char *taosCharsetReplace(char *charsetstr) { - return charsetstr; -} - void taosGetSystemTimezone() { // get and set default timezone SGlobalConfig *cfg_timezone = tsGetConfigOption("timezone"); diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index 0989f007b3..e1f771759a 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -210,25 +210,6 @@ void taosGetSystemTimezone() { pPrint("timezone not configured, set to system default:%s", tsTimezone); } -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - {"utf8", "UTF-8"}, {"936", "CP936"}, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} - /* * POSIX format locale string: * (Language Strings)_(Country/Region Strings).(code_page) diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index 75bc8c2839..53563341c4 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -96,10 +96,6 @@ void __sync_val_restore_32(int32_t *ptr, int32_t newval) { void tsPrintOsInfo() {} -char *taosCharsetReplace(char *charsetstr) { - return charsetstr; -} - void taosGetSystemTimezone() { // get and set default timezone SGlobalConfig *cfg_timezone = tsGetConfigOption("timezone"); diff --git a/src/system/src/vnodeImport.c b/src/system/src/vnodeImport.c index 9691206bf9..bfffc66aa5 100644 --- a/src/system/src/vnodeImport.c +++ b/src/system/src/vnodeImport.c @@ -546,6 +546,7 @@ int vnodeImportToCache(SImportInfo *pImport, char *payload, int rows) { return code; } + assert(rows); dTrace("vid:%d sid:%d id:%s, %d rows data will be imported to cache, firstKey:%ld lastKey:%ld", pObj->vnode, pObj->sid, pObj->meterId, rows, firstKey, lastKey); @@ -781,7 +782,7 @@ int vnodeImportStartToCache(SImportInfo *pImport, char *payload, int rows) { pImport->importedRows = rows; code = vnodeImportToCache(pImport, payload, rows); } else { - dTrace("vid:%d sid:%d id:%s, data is already imported to cache", pObj->vnode, pObj->sid, pObj->meterId); + dTrace("vid:%d sid:%d id:%s, data is already imported to cache, firstKey:%lld", pObj->vnode, pObj->sid, pObj->meterId, pImport->firstKey); } return code; diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cb93319900..0b407a848b 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -508,3 +508,22 @@ char *taosIpStr(int ipInt) { sprintf(ipStr, "0x%x:%d.%d.%d.%d", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, ipInt >> 24); return ipStr; } + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + { "utf8", "UTF-8" }, { "936", "CP936" }, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} \ No newline at end of file From 6bc4ab252bde7a29b384e7b6786928647d019d7f Mon Sep 17 00:00:00 2001 From: huili Date: Sat, 30 Nov 2019 18:08:57 +0800 Subject: [PATCH 03/42] [modify the name of the client install package] --- packaging/tools/makeclient.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 0e52e0970a..7e22116ac5 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -20,7 +20,7 @@ release_dir="${top_dir}/release" community_dir="${script_dir}/../../../community/src" package_name='linux' -install_dir="${release_dir}/TDengine-client-enterprise-${version}-${package_name}-$(echo ${build_time}| tr ': ' -)" +install_dir="${release_dir}/taos-client-${version}-${package_name}-$(echo ${build_time}| tr ': ' -)" # Directories and files. bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${script_dir}/remove_client.sh" From 27f52b89759cfc1c25c2b4a25410803b1fbecc3e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 30 Nov 2019 22:37:22 +0800 Subject: [PATCH 04/42] fix #808 --- src/rpc/src/trpc.c | 49 ++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index d84746817d..9f006ab05a 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -1059,6 +1059,16 @@ int taosBuildErrorMsgToPeer(char *pMsg, int code, char *pReply) { return msgLen; } +void taosReportDisconnection(SRpcChann *pChann, SRpcConn *pConn) +{ + SSchedMsg schedMsg; + schedMsg.fp = taosProcessSchedMsg; + schedMsg.msg = NULL; + schedMsg.ahandle = pConn->ahandle; + schedMsg.thandle = pConn; + taosScheduleTask(pChann->qhandle, &schedMsg); +} + void taosProcessIdleTimer(void *param, void *tmrId) { SRpcConn *pConn = (SRpcConn *)param; if (pConn->signature != param) { @@ -1074,22 +1084,20 @@ void taosProcessIdleTimer(void *param, void *tmrId) { return; } + int reportDisc = 0; + pthread_mutex_lock(&pChann->mutex); tTrace("%s cid:%d sid:%d id:%s, close the connection since no activity pConn:%p", pServer->label, pConn->chann, pConn->sid, pConn->meterId, pConn); if (pConn->rspReceived == 0) { pConn->rspReceived = 1; - - SSchedMsg schedMsg; - schedMsg.fp = taosProcessSchedMsg; - schedMsg.msg = NULL; - schedMsg.ahandle = pConn->ahandle; - schedMsg.thandle = pConn; - taosScheduleTask(pChann->qhandle, &schedMsg); + reportDisc = 1; } pthread_mutex_unlock(&pChann->mutex); + + if (reportDisc) taosReportDisconnection(pChann, pConn); } void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, uint16_t port, void *shandle, void *thandle, @@ -1114,11 +1122,7 @@ void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, uint16_t por pConn->meterId, pConn); pConn->rspReceived = 1; pConn->chandle = NULL; - schedMsg.fp = taosProcessSchedMsg; - schedMsg.msg = NULL; - schedMsg.ahandle = pConn->ahandle; - schedMsg.thandle = pConn; - taosScheduleTask(pChann->qhandle, &schedMsg); + taosReportDisconnection(pChann, pConn); } tfree(data); return NULL; @@ -1330,6 +1334,7 @@ void taosProcessTaosTimer(void *param, void *tmrId) { STaosHeader *pHeader = NULL; SRpcConn * pConn = (SRpcConn *)param; int msgLen; + int reportDisc = 0; if (pConn->signature != param) { tError("pConn Signature:0x%x, pConn:0x%x not matched", pConn->signature, param); @@ -1379,13 +1384,7 @@ void taosProcessTaosTimer(void *param, void *tmrId) { pConn->sid, pConn->meterId, taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort, pConn); if (pConn->rspReceived == 0) { pConn->rspReceived = 1; - - SSchedMsg schedMsg; - schedMsg.fp = taosProcessSchedMsg; - schedMsg.msg = NULL; - schedMsg.ahandle = pConn->ahandle; - schedMsg.thandle = pConn; - taosScheduleTask(pChann->qhandle, &schedMsg); + reportDisc = 1; } } } @@ -1397,6 +1396,7 @@ void taosProcessTaosTimer(void *param, void *tmrId) { pthread_mutex_unlock(&pChann->mutex); + if (reportDisc) taosReportDisconnection(pChann, pConn); } void taosGetRpcConnInfo(void *thandle, uint32_t *peerId, uint32_t *peerIp, uint16_t *peerPort, int *cid, int *sid) { @@ -1443,22 +1443,19 @@ void taosStopRpcConn(void *thandle) { tTrace("%s cid:%d sid:%d id:%s, stop the connection pConn:%p", pServer->label, pConn->chann, pConn->sid, pConn->meterId, pConn); + int reportDisc = 0; pthread_mutex_lock(&pChann->mutex); if (pConn->outType) { pConn->rspReceived = 1; - SSchedMsg schedMsg; - schedMsg.fp = taosProcessSchedMsg; - schedMsg.msg = NULL; - schedMsg.ahandle = pConn->ahandle; - schedMsg.thandle = pConn; + reportDisc = 1; pthread_mutex_unlock(&pChann->mutex); - - taosScheduleTask(pChann->qhandle, &schedMsg); } else { pthread_mutex_unlock(&pChann->mutex); taosCloseRpcConn(pConn); } + + if (reportDisc) taosReportDisconnection(pChann, pConn); } int taosAuthenticateMsg(uint8_t *pMsg, int msgLen, uint8_t *pAuth, uint8_t *pKey) { From 0224972e63befb26827cf85bfd258229f26be7dd Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 30 Nov 2019 23:17:58 +0800 Subject: [PATCH 05/42] fix some release scripts --- packaging/release.sh | 6 +++--- packaging/tools/install.sh | 4 ++-- packaging/tools/release_note | 11 +++++++++++ src/inc/tsdb.h | 5 ++++- src/util/src/version.c | 4 ++-- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index 6a00f0b79d..58f69589d8 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -123,11 +123,11 @@ cd ${compile_dir} # arm only support lite ver if [ -z "$armver" ]; then - cmake ${top_dir}/../ + cmake ../ elif [ "$armver" == "arm64" ]; then - cmake ${top_dir}/../ -DVERSION=lite -DARMVER=arm64 + cmake ../ -DARMVER=arm64 elif [ "$armver" == "arm32" ]; then - cmake ${top_dir}/../ -DVERSION=lite -DARMVER=arm32 + cmake ../ -DARMVER=arm32 else echo "input parameter error!!!" return diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 2911452fb4..019e614e0b 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -313,9 +313,9 @@ vercomp () { function is_version_compatible() { - curr_version=$(${bin_dir}/taosd -V | cut -d ' ' -f 2) + curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 2) - min_compatible_version=$(${script_dir}/bin/taosd -V | cut -d ' ' -f 4) + min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 4) vercomp $curr_version $min_compatible_version case $? in diff --git a/packaging/tools/release_note b/packaging/tools/release_note index 3a3cd81ca9..4578a4523c 100644 --- a/packaging/tools/release_note +++ b/packaging/tools/release_note @@ -1,3 +1,14 @@ +taos-1.6.4.0 (Release on 2019-12-01) +Bug fixed: + 1.Look for possible causes of file corruption and fix them + 2.Encapsulate memory allocation functions to reduce the possibility of crashes + 3.Increase Arm64 compilation options + 4.Remove most of the warnings in the code + 5.Provide a variety of connector usage documents + 6.Network connection can be selected in udp and tcp + 7.Allow the maximum number of Tags to be 32 + 8.Bugs reported by the user + taos-1.5.2.6 (Release on 2019-05-13) Bug fixed: - Nchar strings sometimes were wrongly truncated on Window diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index ea356a257a..8b057654c6 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -147,7 +147,10 @@ extern "C" { #define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) #define TSDB_REPLICA_MIN_NUM 1 -#define TSDB_REPLICA_MAX_NUM 3 +/* + * this is defined in CMakeList.txt + */ +//#define TSDB_REPLICA_MAX_NUM 3 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta diff --git a/src/util/src/version.c b/src/util/src/version.c index 35cd40a942..87426123ec 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,4 @@ char version[64] = "1.6.4.0"; char compatible_version[64] = "1.6.1.0"; -char gitinfo[128] = "b6e308866e315483915f4c42a2717547ed0b9d36"; -char buildinfo[512] = "Built by ubuntu at 2019-11-26 21:56"; +char gitinfo[128] = "e86237890ee9daa8f6ab64ed225a027ae63ebdfc"; +char buildinfo[512] = "Built by ubuntu at 2019-11-30 23:16"; From 79fc94f019f82f54ed137277aa9c79d6365994b1 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 1 Dec 2019 09:29:58 +0800 Subject: [PATCH 06/42] [TBASE-1280] --- src/rpc/src/tudp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/src/tudp.c b/src/rpc/src/tudp.c index bfc95373fc..fb0b37d93b 100644 --- a/src/rpc/src/tudp.c +++ b/src/rpc/src/tudp.c @@ -377,6 +377,7 @@ void *taosTransferDataViaTcp(void *argv) { pThead->tcp = 1; pThead->msgType = (char)(pHeader->msgType - 1); pThead->msgLen = (int32_t)htonl(sizeof(STaosHeader)); + uint32_t id = pThead->sourceId; pThead->sourceId = pThead->destId; pThead->destId = id; pMonitor->ip = pTransfer->ip; pMonitor->port = pTransfer->port; pMonitor->pSet = pSet; From a7f79b508f77f849bd69a8905cd0cc8bd2df02f8 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 1 Dec 2019 12:27:29 +0800 Subject: [PATCH 07/42] public v1.6.4.0 --- src/util/src/version.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/version.c b/src/util/src/version.c index 87426123ec..ed83066f80 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,4 @@ char version[64] = "1.6.4.0"; char compatible_version[64] = "1.6.1.0"; -char gitinfo[128] = "e86237890ee9daa8f6ab64ed225a027ae63ebdfc"; -char buildinfo[512] = "Built by ubuntu at 2019-11-30 23:16"; +char gitinfo[128] = "6d27c11e3b23ae69366df366a6517853648c41f7"; +char buildinfo[512] = "Built by ubuntu at 2019-12-01 12:27"; From faca8084e11dd83c07ba4c18dced20c54175f5a4 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 1 Dec 2019 13:24:12 +0800 Subject: [PATCH 08/42] resolve conflicts --- src/os/windows/src/twindows.c | 3 - src/system/src/vnodeImport.c | 985 ---------------------------------- src/util/src/tutil.c | 22 - 3 files changed, 1010 deletions(-) delete mode 100644 src/system/src/vnodeImport.c diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index 49c2466669..98be6b60ba 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -132,8 +132,6 @@ short interlocked_or_fetch_16(short volatile* ptr, short val) { return _InterlockedOr16(ptr, val) | val; } -<<<<<<< HEAD -======= long interlocked_or_fetch_32(long volatile* ptr, long val) { return _InterlockedOr(ptr, val) | val; } @@ -207,7 +205,6 @@ __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { void tsPrintOsInfo() {} ->>>>>>> release/v1.6.4.0 void taosGetSystemTimezone() { // get and set default timezone SGlobalConfig *cfg_timezone = tsGetConfigOption("timezone"); diff --git a/src/system/src/vnodeImport.c b/src/system/src/vnodeImport.c deleted file mode 100644 index bfffc66aa5..0000000000 --- a/src/system/src/vnodeImport.c +++ /dev/null @@ -1,985 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include - -#include "trpc.h" -#include "ttimer.h" -#include "vnode.h" -#include "vnodeMgmt.h" -#include "vnodeShell.h" -#include "vnodeShell.h" -#include "vnodeUtil.h" -#pragma GCC diagnostic ignored "-Wpointer-sign" -#pragma GCC diagnostic ignored "-Wint-conversion" - -typedef struct { - SCompHeader *headList; - SCompInfo compInfo; - int last; // 0:last block in data file, 1:not the last block - int newBlocks; - int oldNumOfBlocks; - int64_t compInfoOffset; // offset for compInfo in head file - int64_t leftOffset; // copy from this offset to end of head file - int64_t hfdSize; // old head file size -} SHeadInfo; - -typedef struct { - void *signature; - SShellObj *pShell; - SMeterObj *pObj; - int retry; - TSKEY firstKey; - TSKEY lastKey; - int importedRows; - int commit; // start to commit if it is set to 1 - - int slot; // slot/block to start writing the import data - int pos; // pos to start writing the import data in the slot/block - TSKEY key; - - // only for file - int numOfPoints; - int fileId; - int64_t offset; // offset in data file - SData *sdata[TSDB_MAX_COLUMNS]; - char *buffer; - char *payload; - char *opayload; // allocated space for payload from client - int rows; -} SImportInfo; - -int vnodeImportData(SMeterObj *pObj, SImportInfo *pImport); - -int vnodeGetImportStartPart(SMeterObj *pObj, char *payload, int rows, TSKEY key1) { - int i; - - for (i = 0; i < rows; ++i) { - TSKEY key = *((TSKEY *)(payload + i * pObj->bytesPerPoint)); - if (key >= key1) break; - } - - return i; -} - -int vnodeGetImportEndPart(SMeterObj *pObj, char *payload, int rows, char **pStart, TSKEY key0) { - int i; - - for (i = 0; i < rows; ++i) { - TSKEY key = *((TSKEY *)(payload + i * pObj->bytesPerPoint)); - if (key > key0) break; - } - - *pStart = payload + i * pObj->bytesPerPoint; - return rows - i; -} - -int vnodeCloseFileForImport(SMeterObj *pObj, SHeadInfo *pHinfo) { - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SVnodeCfg *pCfg = &pVnode->cfg; - TSCKSUM chksum = 0; - - if (pHinfo->newBlocks == 0 || pHinfo->compInfoOffset == 0) return 0; - - if (pHinfo->oldNumOfBlocks == 0) twrite(pVnode->nfd, &chksum, sizeof(TSCKSUM)); - - int leftSize = pHinfo->hfdSize - pHinfo->leftOffset; - if (leftSize > 0) { - lseek(pVnode->hfd, pHinfo->leftOffset, SEEK_SET); - tsendfile(pVnode->nfd, pVnode->hfd, NULL, leftSize); - } - - pHinfo->compInfo.numOfBlocks += pHinfo->newBlocks; - int offset = (pHinfo->compInfo.numOfBlocks - pHinfo->oldNumOfBlocks) * sizeof(SCompBlock); - if (pHinfo->oldNumOfBlocks == 0) offset += sizeof(SCompInfo) + sizeof(TSCKSUM); - - pHinfo->headList[pObj->sid].compInfoOffset = pHinfo->compInfoOffset; - for (int sid = pObj->sid + 1; sid < pCfg->maxSessions; ++sid) { - if (pHinfo->headList[sid].compInfoOffset) pHinfo->headList[sid].compInfoOffset += offset; - } - - lseek(pVnode->nfd, TSDB_FILE_HEADER_LEN, SEEK_SET); - int tmsize = sizeof(SCompHeader) * pCfg->maxSessions + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pHinfo->headList, tmsize); - twrite(pVnode->nfd, pHinfo->headList, tmsize); - - int size = pHinfo->compInfo.numOfBlocks * sizeof(SCompBlock); - char *buffer = malloc(size); - lseek(pVnode->nfd, pHinfo->compInfoOffset + sizeof(SCompInfo), SEEK_SET); - read(pVnode->nfd, buffer, size); - SCompBlock *pBlock = (SCompBlock *)(buffer + (pHinfo->compInfo.numOfBlocks - 1) * sizeof(SCompBlock)); - - pHinfo->compInfo.uid = pObj->uid; - pHinfo->compInfo.delimiter = TSDB_VNODE_DELIMITER; - pHinfo->compInfo.last = pBlock->last; - - taosCalcChecksumAppend(0, (uint8_t *)(&pHinfo->compInfo), sizeof(SCompInfo)); - lseek(pVnode->nfd, pHinfo->compInfoOffset, SEEK_SET); - twrite(pVnode->nfd, &pHinfo->compInfo, sizeof(SCompInfo)); - - chksum = taosCalcChecksum(0, (uint8_t *)buffer, size); - lseek(pVnode->nfd, pHinfo->compInfoOffset + sizeof(SCompInfo) + size, SEEK_SET); - twrite(pVnode->nfd, &chksum, sizeof(TSCKSUM)); - free(buffer); - - vnodeCloseCommitFiles(pVnode); - - return 0; -} - -int vnodeProcessLastBlock(SImportInfo *pImport, SHeadInfo *pHinfo, SData *data[]) { - SMeterObj *pObj = pImport->pObj; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SCompBlock lastBlock; - int code = 0; - - if (pHinfo->compInfo.last == 0) return 0; - - // read into memory - uint64_t offset = - pHinfo->compInfoOffset + (pHinfo->compInfo.numOfBlocks - 1) * sizeof(SCompBlock) + sizeof(SCompInfo); - lseek(pVnode->hfd, offset, SEEK_SET); - read(pVnode->hfd, &lastBlock, sizeof(SCompBlock)); - assert(lastBlock.last); - - if (lastBlock.sversion != pObj->sversion) { - lseek(pVnode->lfd, lastBlock.offset, SEEK_SET); - lastBlock.offset = lseek(pVnode->dfd, 0, SEEK_END); - tsendfile(pVnode->dfd, pVnode->lfd, NULL, lastBlock.len); - - lastBlock.last = 0; - lseek(pVnode->hfd, offset, SEEK_SET); - twrite(pVnode->hfd, &lastBlock, sizeof(SCompBlock)); - } else { - vnodeReadLastBlockToMem(pObj, &lastBlock, data); - pHinfo->compInfo.numOfBlocks--; - code = lastBlock.numOfPoints; - } - - return code; -} - -int vnodeOpenFileForImport(SImportInfo *pImport, char *payload, SHeadInfo *pHinfo, SData *data[]) { - SMeterObj *pObj = pImport->pObj; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SVnodeCfg *pCfg = &pVnode->cfg; - TSKEY firstKey = *((TSKEY *)payload); - struct stat filestat; - int sid, rowsBefore = 0; - - if (pVnode->nfd <= 0 || firstKey > pVnode->commitLastKey) { - if (pVnode->nfd > 0) vnodeCloseFileForImport(pObj, pHinfo); - - pVnode->commitFirstKey = firstKey; - if (vnodeOpenCommitFiles(pVnode, pObj->sid) < 0) return -1; - - fstat(pVnode->hfd, &filestat); - pHinfo->hfdSize = filestat.st_size; - pHinfo->newBlocks = 0; - pHinfo->last = 1; // by default, new blockes are at the end of block list - - lseek(pVnode->hfd, TSDB_FILE_HEADER_LEN, SEEK_SET); - read(pVnode->hfd, pHinfo->headList, sizeof(SCompHeader) * pCfg->maxSessions); - - if (pHinfo->headList[pObj->sid].compInfoOffset > 0) { - lseek(pVnode->hfd, pHinfo->headList[pObj->sid].compInfoOffset, SEEK_SET); - if (read(pVnode->hfd, &pHinfo->compInfo, sizeof(SCompInfo)) != sizeof(SCompInfo)) { - dError("vid:%d sid:%d, failed to read compInfo from file:%s", pObj->vnode, pObj->sid, pVnode->cfn); - return -1; - } - - if (pHinfo->compInfo.uid == pObj->uid) { - pHinfo->compInfoOffset = pHinfo->headList[pObj->sid].compInfoOffset; - pHinfo->leftOffset = pHinfo->headList[pObj->sid].compInfoOffset + sizeof(SCompInfo); - } else { - pHinfo->headList[pObj->sid].compInfoOffset = 0; - } - } - - if ( pHinfo->headList[pObj->sid].compInfoOffset == 0 ) { - memset(&pHinfo->compInfo, 0, sizeof(SCompInfo)); - pHinfo->compInfo.uid = pObj->uid; - - for (sid = pObj->sid + 1; sid < pCfg->maxSessions; ++sid) - if (pHinfo->headList[sid].compInfoOffset > 0) break; - - pHinfo->compInfoOffset = (sid == pCfg->maxSessions) ? pHinfo->hfdSize : pHinfo->headList[sid].compInfoOffset; - pHinfo->leftOffset = pHinfo->compInfoOffset; - } - - pHinfo->oldNumOfBlocks = pHinfo->compInfo.numOfBlocks; - lseek(pVnode->hfd, 0, SEEK_SET); - lseek(pVnode->nfd, 0, SEEK_SET); - tsendfile(pVnode->nfd, pVnode->hfd, NULL, pHinfo->compInfoOffset); - twrite(pVnode->nfd, &pHinfo->compInfo, sizeof(SCompInfo)); - if (pHinfo->headList[pObj->sid].compInfoOffset > 0) lseek(pVnode->hfd, sizeof(SCompInfo), SEEK_CUR); - - if (pVnode->commitFileId < pImport->fileId) { - if (pHinfo->compInfo.numOfBlocks > 0) - pHinfo->leftOffset += pHinfo->compInfo.numOfBlocks * sizeof(SCompBlock); - - rowsBefore = vnodeProcessLastBlock(pImport, pHinfo, data); - - // copy all existing compBlockInfo - lseek(pVnode->hfd, pHinfo->compInfoOffset + sizeof(SCompInfo), SEEK_SET); - if (pHinfo->compInfo.numOfBlocks > 0) - tsendfile(pVnode->nfd, pVnode->hfd, NULL, pHinfo->compInfo.numOfBlocks * sizeof(SCompBlock)); - - } else if (pVnode->commitFileId == pImport->fileId) { - int slots = pImport->pos ? pImport->slot + 1 : pImport->slot; - assert(slots >= 0); - pHinfo->leftOffset += slots * sizeof(SCompBlock); - - // check if last block is at last file, if it is, read into memory - if (pImport->pos == 0 && pHinfo->compInfo.numOfBlocks > 0 && pImport->slot == pHinfo->compInfo.numOfBlocks && - pHinfo->compInfo.last) { - rowsBefore = vnodeProcessLastBlock(pImport, pHinfo, data); - if ( rowsBefore > 0 ) pImport->slot--; - } - - // this block will be replaced by new blocks - if (pImport->pos > 0) pHinfo->compInfo.numOfBlocks--; - - if (pImport->slot > 0) { - lseek(pVnode->hfd, pHinfo->compInfoOffset + sizeof(SCompInfo), SEEK_SET); - tsendfile(pVnode->nfd, pVnode->hfd, NULL, pImport->slot * sizeof(SCompBlock)); - } - - if (pImport->slot < pHinfo->compInfo.numOfBlocks) - pHinfo->last = 0; // new blocks are not at the end of block list - - } else { - // nothing - - pHinfo->last = 0; // new blocks are not at the end of block list - } - } - - return rowsBefore; -} - -extern int vnodeSendShellSubmitRspMsg(SShellObj *pObj, int code, int numOfPoints); -int vnodeImportToFile(SImportInfo *pImport); - -void vnodeProcessImportTimer(void *param, void *tmrId) { - SImportInfo *pImport = (SImportInfo *)param; - if (pImport == NULL || pImport->signature != param) { - dError("import timer is messed up, signature:%p", pImport); - return; - } - - SMeterObj *pObj = pImport->pObj; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SCachePool *pPool = (SCachePool *)pVnode->pCachePool; - SShellObj *pShell = pImport->pShell; - - pImport->retry++; - - //slow query will block the import operation - int32_t state = vnodeSetMeterState(pObj, TSDB_METER_STATE_IMPORTING); - if (state >= TSDB_METER_STATE_DELETING) { - dError("vid:%d sid:%d id:%s, meter is deleted, failed to import, state:%d", - pObj->vnode, pObj->sid, pObj->meterId, state); - return; - } - - int32_t num = 0; - pthread_mutex_lock(&pVnode->vmutex); - num = pObj->numOfQueries; - pthread_mutex_unlock(&pVnode->vmutex); - - //if the num == 0, it will never be increased before state is set to TSDB_METER_STATE_READY - int32_t commitInProcess = 0; - pthread_mutex_lock(&pPool->vmutex); - if (((commitInProcess = pPool->commitInProcess) == 1) || num > 0 || state != TSDB_METER_STATE_READY) { - pthread_mutex_unlock(&pPool->vmutex); - vnodeClearMeterState(pObj, TSDB_METER_STATE_IMPORTING); - - if (pImport->retry < 1000) { - dTrace("vid:%d sid:%d id:%s, import failed, retry later. commit in process or queries on it, or not ready." - "commitInProcess:%d, numOfQueries:%d, state:%d", pObj->vnode, pObj->sid, pObj->meterId, - commitInProcess, num, state); - - taosTmrStart(vnodeProcessImportTimer, 10, pImport, vnodeTmrCtrl); - return; - } else { - pShell->code = TSDB_CODE_TOO_SLOW; - } - } else { - pPool->commitInProcess = 1; - pthread_mutex_unlock(&pPool->vmutex); - int code = vnodeImportData(pObj, pImport); - if (pShell) { - pShell->code = code; - pShell->numOfTotalPoints += pImport->importedRows; - } - } - - vnodeClearMeterState(pObj, TSDB_METER_STATE_IMPORTING); - - pVnode->version++; - - // send response back to shell - if (pShell) { - pShell->count--; - if (pShell->count <= 0) vnodeSendShellSubmitRspMsg(pImport->pShell, pShell->code, pShell->numOfTotalPoints); - } - - pImport->signature = NULL; - free(pImport->opayload); - free(pImport); -} - -int vnodeImportToFile(SImportInfo *pImport) { - SMeterObj *pObj = pImport->pObj; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SVnodeCfg *pCfg = &pVnode->cfg; - SHeadInfo headInfo; - int code = 0, col; - SCompBlock compBlock; - char *payload = pImport->payload; - int rows = pImport->rows; - SCachePool *pPool = (SCachePool *)pVnode->pCachePool; - - TSKEY lastKey = *((TSKEY *)(payload + pObj->bytesPerPoint * (rows - 1))); - TSKEY firstKey = *((TSKEY *)payload); - memset(&headInfo, 0, sizeof(headInfo)); - headInfo.headList = malloc(sizeof(SCompHeader) * pCfg->maxSessions + sizeof(TSCKSUM)); - - SData *cdata[TSDB_MAX_COLUMNS]; - char * buffer1 = - malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES) * pObj->numOfColumns); - cdata[0] = (SData *)buffer1; - - SData *data[TSDB_MAX_COLUMNS]; - char * buffer2 = - malloc(pObj->bytesPerPoint * pCfg->rowsInFileBlock + (sizeof(SData) + EXTRA_BYTES) * pObj->numOfColumns); - data[0] = (SData *)buffer2; - - for (col = 1; col < pObj->numOfColumns; ++col) { - cdata[col] = (SData *)(((char *)cdata[col - 1]) + sizeof(SData) + EXTRA_BYTES + - pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes); - data[col] = (SData *)(((char *)data[col - 1]) + sizeof(SData) + EXTRA_BYTES + - pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes); - } - - int rowsBefore = 0; - int rowsRead = 0; - int rowsUnread = 0; - int leftRows = rows; // left number of rows of imported data - int row, rowsToWrite; - int64_t offset[TSDB_MAX_COLUMNS]; - - if (pImport->pos > 0) { - for (col = 0; col < pObj->numOfColumns; ++col) - memcpy(data[col]->data, pImport->sdata[col]->data, pImport->pos * pObj->schema[col].bytes); - - rowsBefore = pImport->pos; - rowsRead = pImport->pos; - rowsUnread = pImport->numOfPoints - pImport->pos; - } - - dTrace("vid:%d sid:%d id:%s, %d rows data will be imported to file, firstKey:%ld lastKey:%ld", - pObj->vnode, pObj->sid, pObj->meterId, rows, firstKey, lastKey); - do { - if (leftRows > 0) { - code = vnodeOpenFileForImport(pImport, payload, &headInfo, data); - if (code < 0) goto _exit; - if (code > 0) { - rowsBefore = code; - code = 0; - }; - } else { - // if payload is already imported, rows unread shall still be processed - rowsBefore = 0; - } - - int rowsToProcess = pObj->pointsPerFileBlock - rowsBefore; - if (rowsToProcess > leftRows) rowsToProcess = leftRows; - - for (col = 0; col < pObj->numOfColumns; ++col) { - offset[col] = data[col]->data + rowsBefore * pObj->schema[col].bytes; - } - - row = 0; - if (leftRows > 0) { - for (row = 0; row < rowsToProcess; ++row) { - if (*((TSKEY *)payload) > pVnode->commitLastKey) break; - - for (col = 0; col < pObj->numOfColumns; ++col) { - memcpy((void *)offset[col], payload, pObj->schema[col].bytes); - payload += pObj->schema[col].bytes; - offset[col] += pObj->schema[col].bytes; - } - } - } - - leftRows -= row; - rowsToWrite = rowsBefore + row; - rowsBefore = 0; - - if (leftRows == 0 && rowsUnread > 0) { - // copy the unread - int rowsToCopy = pObj->pointsPerFileBlock - rowsToWrite; - if (rowsToCopy > rowsUnread) rowsToCopy = rowsUnread; - - for (col = 0; col < pObj->numOfColumns; ++col) { - int bytes = pObj->schema[col].bytes; - memcpy(data[col]->data + rowsToWrite * bytes, pImport->sdata[col]->data + rowsRead * bytes, rowsToCopy * bytes); - } - - rowsRead += rowsToCopy; - rowsUnread -= rowsToCopy; - rowsToWrite += rowsToCopy; - } - - for (col = 0; col < pObj->numOfColumns; ++col) { - data[col]->len = rowsToWrite * pObj->schema[col].bytes; - } - - compBlock.last = headInfo.last; - vnodeWriteBlockToFile(pObj, &compBlock, data, cdata, rowsToWrite); - twrite(pVnode->nfd, &compBlock, sizeof(SCompBlock)); - - rowsToWrite = 0; - headInfo.newBlocks++; - - } while (leftRows > 0 || rowsUnread > 0); - - if (compBlock.keyLast > pObj->lastKeyOnFile) - pObj->lastKeyOnFile = compBlock.keyLast; - - vnodeCloseFileForImport(pObj, &headInfo); - dTrace("vid:%d sid:%d id:%s, %d rows data are imported to file", pObj->vnode, pObj->sid, pObj->meterId, rows); - - SCacheInfo *pInfo = (SCacheInfo *)pObj->pCache; - pthread_mutex_lock(&pPool->vmutex); - - if (pInfo->numOfBlocks > 0) { - int slot = (pInfo->currentSlot - pInfo->numOfBlocks + 1 + pInfo->maxBlocks) % pInfo->maxBlocks; - TSKEY firstKeyInCache = *((TSKEY *)(pInfo->cacheBlocks[slot]->offset[0])); - - // data may be in commited cache, cache shall be released - if (lastKey > firstKeyInCache) { - while (slot != pInfo->commitSlot) { - SCacheBlock *pCacheBlock = pInfo->cacheBlocks[slot]; - vnodeFreeCacheBlock(pCacheBlock); - slot = (slot + 1 + pInfo->maxBlocks) % pInfo->maxBlocks; - } - - // last slot, the uncommitted slots shall be shifted - SCacheBlock *pCacheBlock = pInfo->cacheBlocks[slot]; - int points = pCacheBlock->numOfPoints - pInfo->commitPoint; - if (points > 0) { - for (int col = 0; col < pObj->numOfColumns; ++col) { - int size = points * pObj->schema[col].bytes; - memmove(pCacheBlock->offset[col], pCacheBlock->offset[col] + pObj->schema[col].bytes * pInfo->commitPoint, size); - } - } - - if (pInfo->commitPoint != pObj->pointsPerBlock) { - // commit point shall be set to 0 if last block is not full - pInfo->commitPoint = 0; - pCacheBlock->numOfPoints = points; - if (slot == pInfo->currentSlot) { - __sync_fetch_and_add(&pObj->freePoints, pInfo->commitPoint); - } - } else { - // if last block is full and committed - SCacheBlock *pCacheBlock = pInfo->cacheBlocks[slot]; - if (pCacheBlock->pMeterObj == pObj) { - vnodeFreeCacheBlock(pCacheBlock); - } - } - } - } - - if (lastKey > pObj->lastKeyOnFile) pObj->lastKeyOnFile = lastKey; - - pthread_mutex_unlock(&pPool->vmutex); - -_exit: - tfree(headInfo.headList); - tfree(buffer1); - tfree(buffer2); - tfree(pImport->buffer); - - return code; -} - -int vnodeImportToCache(SImportInfo *pImport, char *payload, int rows) { - SMeterObj *pObj = pImport->pObj; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SVnodeCfg *pCfg = &pVnode->cfg; - int code = -1; - SCacheInfo *pInfo = (SCacheInfo *)pObj->pCache; - int slot, pos, row, col, points, tpoints; - - char *data[TSDB_MAX_COLUMNS], *current[TSDB_MAX_COLUMNS]; - int slots = pInfo->unCommittedBlocks + 1; - int trows = slots * pObj->pointsPerBlock + rows; // max rows in buffer - int tsize = (trows / pObj->pointsPerBlock + 1) * pCfg->cacheBlockSize; - TSKEY firstKey = *((TSKEY *)payload); - TSKEY lastKey = *((TSKEY *)(payload + pObj->bytesPerPoint * (rows - 1))); - - if (pObj->freePoints < rows || pObj->freePoints < (pObj->pointsPerBlock << 1)) { - dError("vid:%d sid:%d id:%s, import failed, cache is full, freePoints:%d", pObj->vnode, pObj->sid, pObj->meterId, - pObj->freePoints); - pImport->importedRows = 0; - pImport->commit = 1; - code = TSDB_CODE_ACTION_IN_PROGRESS; - return code; - } - - assert(rows); - dTrace("vid:%d sid:%d id:%s, %d rows data will be imported to cache, firstKey:%ld lastKey:%ld", - pObj->vnode, pObj->sid, pObj->meterId, rows, firstKey, lastKey); - - pthread_mutex_lock(&(pVnode->vmutex)); - if (firstKey < pVnode->firstKey) pVnode->firstKey = firstKey; - pthread_mutex_unlock(&(pVnode->vmutex)); - - char *buffer = malloc(tsize); // buffer to hold unCommitted data plus import data - data[0] = buffer; - current[0] = data[0]; - for (col = 1; col < pObj->numOfColumns; ++col) { - data[col] = data[col - 1] + trows * pObj->schema[col - 1].bytes; - current[col] = data[col]; - } - - // write import data into buffer first - for (row = 0; row < rows; ++row) { - for (col = 0; col < pObj->numOfColumns; ++col) { - memcpy(current[col], payload, pObj->schema[col].bytes); - payload += pObj->schema[col].bytes; - current[col] += pObj->schema[col].bytes; - } - } - - // copy the overwritten data into buffer - tpoints = rows; - pos = pImport->pos; - slot = pImport->slot; - while (1) { - points = pInfo->cacheBlocks[slot]->numOfPoints - pos; - for (col = 0; col < pObj->numOfColumns; ++col) { - int size = points * pObj->schema[col].bytes; - memcpy(current[col], pInfo->cacheBlocks[slot]->offset[col] + pos * pObj->schema[col].bytes, size); - current[col] += size; - } - pos = 0; - tpoints += points; - - if (slot == pInfo->currentSlot) break; - slot = (slot + 1) % pInfo->maxBlocks; - } - - for (col = 0; col < pObj->numOfColumns; ++col) current[col] = data[col]; - pos = pImport->pos; - - // write back to existing slots first - slot = pImport->slot; - while (1) { - points = (tpoints > pObj->pointsPerBlock - pos) ? pObj->pointsPerBlock - pos : tpoints; - SCacheBlock *pCacheBlock = pInfo->cacheBlocks[slot]; - for (col = 0; col < pObj->numOfColumns; ++col) { - int size = points * pObj->schema[col].bytes; - memcpy(pCacheBlock->offset[col] + pos * pObj->schema[col].bytes, current[col], size); - current[col] += size; - } - pCacheBlock->numOfPoints = points + pos; - pos = 0; - tpoints -= points; - - if (slot == pInfo->currentSlot) break; - slot = (slot + 1) % pInfo->maxBlocks; - } - - // allocate new cache block if there are still data left - while (tpoints > 0) { - pImport->commit = vnodeAllocateCacheBlock(pObj); - if (pImport->commit < 0) goto _exit; - points = (tpoints > pObj->pointsPerBlock) ? pObj->pointsPerBlock : tpoints; - SCacheBlock *pCacheBlock = pInfo->cacheBlocks[pInfo->currentSlot]; - for (col = 0; col < pObj->numOfColumns; ++col) { - int size = points * pObj->schema[col].bytes; - memcpy(pCacheBlock->offset[col] + pos * pObj->schema[col].bytes, current[col], size); - current[col] += size; - } - tpoints -= points; - pCacheBlock->numOfPoints = points; - } - - code = 0; - __sync_fetch_and_sub(&pObj->freePoints, rows); - dTrace("vid:%d sid:%d id:%s, %d rows data are imported to cache", pObj->vnode, pObj->sid, pObj->meterId, rows); - -_exit: - free(buffer); - return code; -} - -int vnodeFindKeyInFile(SImportInfo *pImport, int order) { - SMeterObj *pObj = pImport->pObj; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - int code = -1; - SQuery query; - SColumnFilter colList[TSDB_MAX_COLUMNS] = {0}; - - TSKEY key = order ? pImport->firstKey : pImport->lastKey; - memset(&query, 0, sizeof(query)); - query.order.order = order; - query.skey = key; - query.ekey = order ? INT64_MAX : 0; - query.colList = colList; - query.numOfCols = pObj->numOfColumns; - - for (int16_t i = 0; i < pObj->numOfColumns; ++i) { - colList[i].data.colId = pObj->schema[i].colId; - colList[i].data.bytes = pObj->schema[i].bytes; - colList[i].data.type = pObj->schema[i].type; - - colList[i].colIdx = i; - colList[i].colIdxInBuf = i; - } - - int ret = vnodeSearchPointInFile(pObj, &query); - - if (ret >= 0) { - if (query.slot < 0) { - pImport->slot = 0; - pImport->pos = 0; - pImport->key = 0; - pImport->fileId = pVnode->fileId - pVnode->numOfFiles + 1; - dTrace("vid:%d sid:%d id:%s, import to head of file", pObj->vnode, pObj->sid, pObj->meterId); - code = 0; - } else if (query.slot >= 0) { - code = 0; - pImport->slot = query.slot; - pImport->pos = query.pos; - pImport->key = query.key; - pImport->fileId = query.fileId; - SCompBlock *pBlock = &query.pBlock[query.slot]; - pImport->numOfPoints = pBlock->numOfPoints; - - if (pImport->key != key) { - if (order == 0) { - pImport->pos++; - - if (pImport->pos >= pBlock->numOfPoints) { - pImport->slot++; - pImport->pos = 0; - } - } else { - if (pImport->pos < 0) pImport->pos = 0; - } - } - - if (pImport->key != key && pImport->pos > 0) { - if ( pObj->sversion != pBlock->sversion ) { - dError("vid:%d sid:%d id:%s, import sversion not matached, expected:%d received:%d", pObj->vnode, pObj->sid, - pBlock->sversion, pObj->sversion); - code = TSDB_CODE_OTHERS; - } else { - pImport->offset = pBlock->offset; - - pImport->buffer = - malloc(pObj->bytesPerPoint * pVnode->cfg.rowsInFileBlock + sizeof(SData) * pObj->numOfColumns); - pImport->sdata[0] = (SData *)pImport->buffer; - for (int col = 1; col < pObj->numOfColumns; ++col) - pImport->sdata[col] = (SData *)(((char *)pImport->sdata[col - 1]) + sizeof(SData) + - pObj->pointsPerFileBlock * pObj->schema[col - 1].bytes); - - code = vnodeReadCompBlockToMem(pObj, &query, pImport->sdata); - if (code < 0) { - code = -code; - tfree(pImport->buffer); - } - } - } - } - } else { - dError("vid:%d sid:%d id:%s, file is corrupted, import failed", pObj->vnode, pObj->sid, pObj->meterId); - code = -ret; - } - - tclose(query.hfd); - tclose(query.dfd); - tclose(query.lfd); - vnodeFreeFields(&query); - tfree(query.pBlock); - - return code; -} - -int vnodeFindKeyInCache(SImportInfo *pImport, int order) { - SMeterObj *pObj = pImport->pObj; - int code = 0; - SQuery query; - SCacheInfo *pInfo = (SCacheInfo *)pObj->pCache; - - TSKEY key = order ? pImport->firstKey : pImport->lastKey; - memset(&query, 0, sizeof(query)); - query.order.order = order; - query.skey = key; - query.ekey = order ? pImport->lastKey : pImport->firstKey; - vnodeSearchPointInCache(pObj, &query); - - if (query.slot < 0) { - pImport->slot = pInfo->commitSlot; - if (pInfo->commitPoint >= pObj->pointsPerBlock) pImport->slot = (pImport->slot + 1) % pInfo->maxBlocks; - pImport->pos = 0; - pImport->key = 0; - dTrace("vid:%d sid:%d id:%s, key:%ld, import to head of cache", pObj->vnode, pObj->sid, pObj->meterId, key); - code = 0; - } else { - pImport->slot = query.slot; - pImport->pos = query.pos; - pImport->key = query.key; - - if (key != query.key) { - if (order == 0) { - // since pos is the position which has smaller key, data shall be imported after it - pImport->pos++; - if (pImport->pos >= pObj->pointsPerBlock) { - pImport->slot = (pImport->slot + 1) % pInfo->maxBlocks; - pImport->pos = 0; - } - } else { - if (pImport->pos < 0) pImport->pos = 0; - } - } - code = 0; - } - - return code; -} - -int vnodeImportStartToCache(SImportInfo *pImport, char *payload, int rows) { - int code = 0; - SMeterObj *pObj = pImport->pObj; - - code = vnodeFindKeyInCache(pImport, 1); - if (code != 0) return code; - - if (pImport->key != pImport->firstKey) { - rows = vnodeGetImportStartPart(pObj, payload, rows, pImport->key); - pImport->importedRows = rows; - code = vnodeImportToCache(pImport, payload, rows); - } else { - dTrace("vid:%d sid:%d id:%s, data is already imported to cache, firstKey:%lld", pObj->vnode, pObj->sid, pObj->meterId, pImport->firstKey); - } - - return code; -} - -int vnodeImportStartToFile(SImportInfo *pImport, char *payload, int rows) { - int code = 0; - SMeterObj *pObj = pImport->pObj; - - code = vnodeFindKeyInFile(pImport, 1); - if (code != 0) return code; - - assert(pImport->slot >= 0); - - if (pImport->key != pImport->firstKey) { - pImport->payload = payload; - pImport->rows = vnodeGetImportStartPart(pObj, payload, rows, pImport->key); - pImport->importedRows = pImport->rows; - code = vnodeImportToFile(pImport); - } else { - dTrace("vid:%d sid:%d id:%s, data is already imported to file", pObj->vnode, pObj->sid, pObj->meterId); - } - - return code; -} - -int vnodeImportWholeToFile(SImportInfo *pImport, char *payload, int rows) { - int code = 0; - SMeterObj *pObj = pImport->pObj; - - code = vnodeFindKeyInFile(pImport, 0); - if (code != 0) return code; - - if (pImport->key != pImport->lastKey) { - pImport->payload = payload; - pImport->rows = vnodeGetImportEndPart(pObj, payload, rows, &pImport->payload, pImport->key); - pImport->importedRows = pImport->rows; - code = vnodeImportToFile(pImport); - } else { - code = vnodeImportStartToFile(pImport, payload, rows); - } - - return code; -} - -int vnodeImportWholeToCache(SImportInfo *pImport, char *payload, int rows) { - int code = 0; - SMeterObj *pObj = pImport->pObj; - - code = vnodeFindKeyInCache(pImport, 0); - if (code != 0) return code; - - if (pImport->key != pImport->lastKey) { - char *pStart; - if ( pImport->key < pObj->lastKeyOnFile ) pImport->key = pObj->lastKeyOnFile; - rows = vnodeGetImportEndPart(pObj, payload, rows, &pStart, pImport->key); - pImport->importedRows = rows; - code = vnodeImportToCache(pImport, pStart, rows); - } else { - if (pImport->firstKey > pObj->lastKeyOnFile) { - code = vnodeImportStartToCache(pImport, payload, rows); - } else if (pImport->firstKey < pObj->lastKeyOnFile) { - code = vnodeImportStartToFile(pImport, payload, rows); - } else { // firstKey == pObj->lastKeyOnFile - dTrace("vid:%d sid:%d id:%s, data is already there", pObj->vnode, pObj->sid, pObj->meterId); - } - } - - return code; -} - -int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, void *param, int sversion, - int *pNumOfPoints, TSKEY now) { - SSubmitMsg *pSubmit = (SSubmitMsg *)cont; - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - int rows; - char *payload; - int code = TSDB_CODE_ACTION_IN_PROGRESS; - SCachePool *pPool = (SCachePool *)pVnode->pCachePool; - SShellObj *pShell = (SShellObj *)param; - int pointsImported = 0; - - rows = htons(pSubmit->numOfRows); - int expectedLen = rows * pObj->bytesPerPoint + sizeof(pSubmit->numOfRows); - if (expectedLen != contLen) { - dError("vid:%d sid:%d id:%s, invalid import, expected:%d, contLen:%d", pObj->vnode, pObj->sid, pObj->meterId, - expectedLen, contLen); - return TSDB_CODE_WRONG_MSG_SIZE; - } - - if (sversion != pObj->sversion) { - dError("vid:%d sid:%d id:%s, invalid sversion, expected:%d received:%d", pObj->vnode, pObj->sid, pObj->meterId, - pObj->sversion, sversion); - return TSDB_CODE_OTHERS; - } - - payload = pSubmit->payLoad; - int firstId = (*(TSKEY *)payload)/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - int lastId = (*(TSKEY *)(payload+pObj->bytesPerPoint*(rows-1)))/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - int cfile = now/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision]; - if ((firstId <= cfile - pVnode->maxFiles) || (firstId > cfile + 1) || (lastId <= cfile - pVnode->maxFiles) || (lastId > cfile + 1)) { - dError("vid:%d sid:%d id:%s, invalid timestamp to import, rows:%d firstKey: %ld lastKey: %ld", - pObj->vnode, pObj->sid, pObj->meterId, rows, *(TSKEY *)(payload), *(TSKEY *)(payload+pObj->bytesPerPoint*(rows-1))); - return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE; - } - - if ( pVnode->cfg.commitLog && source != TSDB_DATA_SOURCE_LOG) { - if (pVnode->logFd < 0) return TSDB_CODE_INVALID_COMMIT_LOG; - code = vnodeWriteToCommitLog(pObj, TSDB_ACTION_IMPORT, cont, contLen, sversion); - if (code != 0) return code; - } - - if (*((TSKEY *)(pSubmit->payLoad + (rows - 1) * pObj->bytesPerPoint)) > pObj->lastKey) { - vnodeClearMeterState(pObj, TSDB_METER_STATE_IMPORTING); - vnodeSetMeterState(pObj, TSDB_METER_STATE_INSERT); - code = vnodeInsertPoints(pObj, cont, contLen, TSDB_DATA_SOURCE_LOG, NULL, pObj->sversion, &pointsImported, now); - - if (pShell) { - pShell->code = code; - pShell->numOfTotalPoints += pointsImported; - } - - vnodeClearMeterState(pObj, TSDB_METER_STATE_INSERT); - } else { - SImportInfo *pNew, import; - - dTrace("vid:%d sid:%d id:%s, import %d rows data", pObj->vnode, pObj->sid, pObj->meterId, rows); - memset(&import, 0, sizeof(import)); - import.firstKey = *((TSKEY *)(payload)); - import.lastKey = *((TSKEY *)(pSubmit->payLoad + (rows - 1) * pObj->bytesPerPoint)); - import.pObj = pObj; - import.pShell = pShell; - import.payload = payload; - import.rows = rows; - - int32_t num = 0; - pthread_mutex_lock(&pVnode->vmutex); - num = pObj->numOfQueries; - pthread_mutex_unlock(&pVnode->vmutex); - - int32_t commitInProcess = 0; - - pthread_mutex_lock(&pPool->vmutex); - if (((commitInProcess = pPool->commitInProcess) == 1) || num > 0) { - pthread_mutex_unlock(&pPool->vmutex); - - pNew = (SImportInfo *)malloc(sizeof(SImportInfo)); - memcpy(pNew, &import, sizeof(SImportInfo)); - pNew->signature = pNew; - int payloadLen = contLen - sizeof(SSubmitMsg); - pNew->payload = malloc(payloadLen); - pNew->opayload = pNew->payload; - memcpy(pNew->payload, payload, payloadLen); - - dTrace("vid:%d sid:%d id:%s, import later, commit in process:%d, numOfQueries:%d", pObj->vnode, pObj->sid, - pObj->meterId, commitInProcess, pObj->numOfQueries); - - taosTmrStart(vnodeProcessImportTimer, 10, pNew, vnodeTmrCtrl); - return 0; - } else { - pPool->commitInProcess = 1; - pthread_mutex_unlock(&pPool->vmutex); - int code = vnodeImportData(pObj, &import); - if (pShell) { - pShell->code = code; - pShell->numOfTotalPoints += import.importedRows; - } - } - } - - pVnode->version++; - - if (pShell) { - pShell->count--; - if (pShell->count <= 0) vnodeSendShellSubmitRspMsg(pShell, pShell->code, pShell->numOfTotalPoints); - } - - return 0; -} - -//todo abort from the procedure if the meter is going to be dropped -int vnodeImportData(SMeterObj *pObj, SImportInfo *pImport) { - int code = 0; - - if (pImport->lastKey > pObj->lastKeyOnFile) { - code = vnodeImportWholeToCache(pImport, pImport->payload, pImport->rows); - } else if (pImport->lastKey < pObj->lastKeyOnFile) { - code = vnodeImportWholeToFile(pImport, pImport->payload, pImport->rows); - } else { // lastKey == pObj->lastkeyOnFile - code = vnodeImportStartToFile(pImport, pImport->payload, pImport->rows); - } - - SVnodeObj *pVnode = &vnodeList[pObj->vnode]; - SCachePool *pPool = (SCachePool *)pVnode->pCachePool; - pPool->commitInProcess = 0; - - if (pImport->commit) vnodeProcessCommitTimer(pVnode, NULL); - - return code; -} diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index bd52ac0bc4..e0dcb0aa3f 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -493,27 +493,6 @@ char *taosIpStr(uint32_t ipInt) { sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); return ipStr; } -<<<<<<< HEAD - -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} -======= #ifndef CLUSTER void taosCleanupTier() {} @@ -549,4 +528,3 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } ->>>>>>> release/v1.6.4.0 From 2e60dfb3a054772a60966ce3055c4a4ff62cfa6b Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Mon, 2 Dec 2019 11:13:33 +0800 Subject: [PATCH 09/42] [TBASE 1270]revise the go connector description --- .../webdocs/markdowndocs/connector-ch.md | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index a8ceaa848f..03801fac89 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -391,15 +391,64 @@ curl -u username:password -d '' :/rest/sql ## Go Connector -TDengine提供了GO驱动程序“taosSql”包。taosSql驱动包是基于GO的“database/sql/driver”接口的实现。用户可在安装后的/usr/local/taos/connector/go目录获得GO的客户端驱动程序。用户需将驱动包/usr/local/taos/connector/go/src/taosSql目录拷贝到应用程序工程的src目录下。然后在应用程序中导入驱动包，就可以使用“database/sql”中定义的接口访问TDengine： +#### 安装TDengine + +Go的链接器使用了到了 libtaos.so 和taos.h，因此，在使用Go连接器之前，需要在程序运行的机器上安装TDengine以获得相关的驱动文件。 + +#### Go语言引入package +TDengine提供了GO驱动程序“taosSql”包。taosSql驱动包是基于GO的“database/sql/driver”接口的实现。用户可以通过`go get`命令来获取驱动包。 +```sh +go get github.com/taosdata/TDengine/src/connector/go/src/taosSql +``` +然后在应用程序中导入驱动包，就可以使用“database/sql”中定义的接口访问TDengine： ```Go import ( "database/sql" - _ "taosSql" + _ "github.com/taosdata/TDengine/src/connector/go/src/taosSql" ) ``` taosSql驱动包内采用cgo模式，调用了TDengine的C/C++同步接口，与TDengine进行交互，因此，在数据库操作执行完成之前，客户端应用将处于阻塞状态。单个数据库连接，在同一时刻只能有一个线程调用API。客户应用可以建立多个连接，进行多线程的数据写入或查询处理。 -更多使用的细节，请参考下载目录中的示例源码。 +#### Go语言使用参考 +在Go程序中使用TDengine写入方法大致可以分为以下几步 +1. 打开TDengine数据库链接 + +首先需要调用sql包中的Open方法，打开数据库，并获得db对象 +```go + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname) + if err != nil { + log.Fatalf("Open database error: %s\n", err) + } + defer db.Close() +``` +其中参数为 +- taosDataname: 涛思数据库的名称，其值为字符串"taosSql" +- dbuser和dbpassword: 链接TDengine的用户名和密码，缺省为root和taosdata，类型为字符串 +- daemonUrl: 为TDengine的地址，其形式为`ip address:port`形式，port填写缺省值0即可。例如："116.118.24.71:0" +- dbname：TDengine中的database名称，通过`create database`创建的数据库。如果为空则在后续的写入和查询操作必须通过”数据库名.超级表名或表名“的方式指定数据库名 + +2. 创建数据库 + +打开TDengine数据库连接后，首选需要创建数据库。基本用法和直接在TDengine客户端shell下一样，通过create database + 数据库名的方法来创建。 +```go + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/") + if err != nil { + log.Fatalf("Open database error: %s\n", err) + } + defer db.Close() + + //准备创建数据库语句 + sqlcmd := fmt.Sprintf("create database if not exists %s", dbname) + + //执行语句并检查错误 + _, err = db.Exec(sqlcmd) + if err != nil { + log.Fatalf("Create database error: %s\n", err) + } +``` + +3. 创建表、写入和查询数据 + +在创建好了数据库后，就可以开始创建表和写入查询数据了。这些操作的基本思路都是首先组装SQL语句，然后调用db.Exec执行，并检查错误信息和执行相应的处理。可以参考上面的样例代码 From 167d6cb70edc8a92d32f804eaa5f51d30d28c95e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 Dec 2019 15:47:18 +0800 Subject: [PATCH 10/42] fix #813 --- src/system/detail/src/vnodeImport.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/system/detail/src/vnodeImport.c b/src/system/detail/src/vnodeImport.c index 2a0fda29f9..f0019a92ee 100644 --- a/src/system/detail/src/vnodeImport.c +++ b/src/system/detail/src/vnodeImport.c @@ -466,8 +466,6 @@ static int vnodeLoadNeededBlockData(SMeterObj *pObj, SImportHandle *pHandle, int SCompBlock *pBlock = pHandle->pBlocks + blockId; *code = TSDB_CODE_SUCCESS; - assert(pBlock->sversion == pObj->sversion); - SVnodeObj *pVnode = vnodeList + pObj->vnode; int dfd = pBlock->last ? pVnode->lfd : pVnode->dfd; @@ -989,6 +987,13 @@ static int vnodeMergeDataIntoFile(SImportInfo *pImport, const char *payload, int } } + int aslot = MIN(blockIter.slot, importHandle.compInfo.numOfBlocks - 1); + int64_t sversion = importHandle.pBlocks[aslot].sversion; + if (sversion != pObj->sversion) { + code = TSDB_CODE_OTHERS; + goto _error_merge; + } + // Open the new .t file if not opened yet. if (pVnode->nfd <= 0) { if (vnodeOpenTempFilesForImport(&importHandle, pObj, fid) < 0) { From e593c12cef6b7247d5d7ed24d372182fdbc31ee4 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 16:18:32 +0800 Subject: [PATCH 11/42] Instructions for using the RESTful interface --- .../webdocs/markdowndocs/connector-ch.md | 126 ++++++++++++++++-- 1 file changed, 116 insertions(+), 10 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 9261b9eef6..e629a44bf1 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -316,11 +316,23 @@ import taos 如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. -HTTP请求的Header里需带有身份认证信息，TDengine单机版仅支持Basic认证机制。 +HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 + +- 自定义身份认证信息如下所示（稍后介绍） + +``` +Authorization: Taosd +``` + +- Basic身份认证信息如下所示 + +``` +Authorization: Basic +``` HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应提供数据库前缀，例如\.\。如果表名不带数据库前缀，系统会返回错误。因为HTTP模块只是一个简单的转发，没有当前DB的概念。 -使用curl来发起一个HTTP Request, 语法如下： +使用curl通过自定义身份认证方式来发起一个HTTP Request, 语法如下： ``` curl -H 'Authorization: Basic ' -d '' :/rest/sql @@ -332,7 +344,7 @@ curl -H 'Authorization: Basic ' -d '' :/rest/sql curl -u username:password -d '' :/rest/sql ``` -其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串，例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==` +其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串, 例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==` ### HTTP返回格式 @@ -356,21 +368,55 @@ curl -u username:password -d '' :/rest/sql - 第三行是具体返回的数据，一排一排的呈现。如果不返回结果集，仅[[affected_rows]] - 第四行”rows”表明总共多少行数据 +### 自定义授权码 + +HTTP请求中需要带有授权码``, 用于身份识别。授权码通常由管理员提供, 可简单的通过发送`HTTP GET`请求来获取授权码, 操作如下： + +``` +curl http://:6020/rest/login// +``` + +其中, `ip`是TDengine数据库的IP地址, `username`为数据库用户名, `password`为数据库密码, 返回值为`JSON`格式, 各字段含义如下： + +- status：请求结果的标志位 + +- code：返回值代码 + +- desc: 授权码 + +获取授权码示例： + +``` +curl http://192.168.0.1:6020/rest/login/root/taosdata +``` + +返回值： + +``` +{ + "status": "succ", + "code": 0, + "desc": +"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" +} +``` + ### 使用示例 - 在demo库里查询表t1的所有记录, curl如下： - `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` - - 返回值： +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` +``` +返回值： ``` { "status": "succ", "head": ["column1","column2","column3"], "data": [ - ["2017-12-12 23:44:25.730", 1, 2.3], - ["2017-12-12 22:44:25.728", 4, 5.6] + ["2017-12-12 22:44:25.728",4,5.60000], + ["2017-12-12 23:44:25.730",1,2.30000] ], "rows": 2 } @@ -378,9 +424,11 @@ curl -u username:password -d '' :/rest/sql - 创建库demo： - `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql` +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql` +``` - 返回值： +返回值： ``` { "status": "succ", @@ -390,6 +438,64 @@ curl -u username:password -d '' :/rest/sql } ``` +### 其他用法 + +#### 结果集采用Unix时间戳 + +HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 + +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt` +``` + +返回值： + +``` +{ + "status": "succ", + "head": ["column1","column2","column3"], + "data": [ + [1513089865728,4,5.60000], + [1513093465730,1,2.30000] + ], + "rows": 2 +} +``` + +#### 结果集采用UTC时间字符串 + +HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 +``` + curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc` +``` + +返回值： + +``` +{ + "status": "succ", + "head": ["column1","column2","column3"], + "data": [ + ["2017-12-12T22:44:25.728+0800",4,5.60000], + ["2017-12-12T23:44:25.730+0800",1,2.30000] + ], + "rows": 2 +} +``` + +### 重要配置项 + +下面仅列出一些与RESTFul接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 + +- httpIp: 对外提供RESTFul服务的IP地址，默认绑定到0.0.0.0 +- httpPort: 对外提供RESTFul服务的端口号，默认绑定到6020 +- httpMaxThreads: 启动的线程数量，默认为2 +- httpCacheSessions: 缓存连接的数量，并发请求数目需小于此数值的10倍，默认值为100 +- restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240 +- httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 +- httpDebugFlag: 日志开关，131：仅错误和报警信息，135：所有，默认131 + + ## Go Connector #### 安装TDengine From 0517feb1fd1c52733ca8683b78c913a7eb594a43 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 16:31:38 +0800 Subject: [PATCH 12/42] Instructions for using the RESTful interface --- .../webdocs/markdowndocs/connector-ch.md | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index e629a44bf1..85596f2053 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -306,15 +306,16 @@ import taos ### HTTP请求格式 -​ `http://:/rest/sql` +``` +http://:/rest/sql +``` -​ 参数说明： +​参数说明： -​ IP: 集群中的任一台主机 +- IP: 集群中的任一台主机 +- PORT: 配置文件中httpPort配置项，缺省为6020 -​ PORT: 配置文件中httpPort配置项，缺省为6020 - -如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. +例如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 @@ -348,7 +349,8 @@ curl -u username:password -d '' :/rest/sql ### HTTP返回格式 -返回值为JSON格式，如下： +返回值为JSON格式，如下: + ``` { "status": "succ", @@ -363,10 +365,10 @@ curl -u username:password -d '' :/rest/sql 说明： -- 第一行”status”告知操作结果是成功还是失败; -- 第二行”head”是表的定义，如果不返回结果集，仅有一列“affected_rows”; -- 第三行是具体返回的数据，一排一排的呈现。如果不返回结果集，仅[[affected_rows]] -- 第四行”rows”表明总共多少行数据 +- status: 告知操作结果是成功还是失败 +- head: 表的定义，如果不返回结果集，仅有一列“affected_rows” +- data: 具体返回的数据，一排一排的呈现,如果不返回结果集，仅[[affected_rows]] +- rows: 表明总共多少行数据 ### 自定义授权码 @@ -403,7 +405,7 @@ curl http://192.168.0.1:6020/rest/login/root/taosdata ### 使用示例 -- 在demo库里查询表t1的所有记录, curl如下： +- 在demo库里查询表t1的所有记录： ``` curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` @@ -445,7 +447,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 ``` -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt ``` 返回值： @@ -466,7 +468,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 1 HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 ``` - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc` + curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc ``` 返回值： From e5418dd3e13370be8a5d5b7518e5069472389e58 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Mon, 2 Dec 2019 18:37:51 +0800 Subject: [PATCH 13/42] Update TAOS SQL-ch.md Add some detailed description with respect to the time unit conversion for interval clause. --- documentation/webdocs/markdowndocs/TAOS SQL-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md index 6a8549bbd2..347ac4f21f 100644 --- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md @@ -18,6 +18,7 @@ TDengine提供类似SQL语法，用户可以在TDengine Shell中使用SQL语句 - 插入记录时，如果时间戳为0，插入数据时使用服务器当前时间 - Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数 - 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。数字后面的时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据 +- TDengine暂不支持时间窗口按照自然年和自然月切分。Where条件中的时间窗口单位的换算关系如下：interval(1y) 等效于 interval(365d), interval(1n) 等效于 interval(30d), interval(1w) 等效于 interval(7d) TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。 From d468cc5eb2d5187c06982b4bcec11fb195101a54 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 2 Dec 2019 19:01:20 +0800 Subject: [PATCH 14/42] [TBASE-1271] --- packaging/deb/makedeb.sh | 2 +- packaging/rpm/makerpm.sh | 27 ++++++++++++++++++++++++--- packaging/rpm/tdengine.spec | 2 +- packaging/tools/makeclient.sh | 4 ++-- packaging/tools/makepkg.sh | 4 ++-- src/kit/shell/src/shellEngine.c | 14 ++++++++++++-- src/kit/shell/src/shellLinux.c | 9 +++++++++ src/system/detail/src/dnodeService.c | 7 ++++++- 8 files changed, 57 insertions(+), 12 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 5766bd6836..501a06eddb 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -63,7 +63,7 @@ debver="Version: "$tdengine_ver sed -i "2c$debver" ${pkg_dir}/DEBIAN/control #get taos version, then set deb name -debname="tdengine-"${tdengine_ver}".deb" +debname="TDengine-"${tdengine_ver}".deb" # make deb package dpkg -b ${pkg_dir} $debname diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh index 3b0f1d098e..e301f5ece9 100755 --- a/packaging/rpm/makerpm.sh +++ b/packaging/rpm/makerpm.sh @@ -2,6 +2,9 @@ # # Generate rpm package for centos +#set -e +#set -x + #curr_dir=$(pwd) compile_dir=$1 output_dir=$2 @@ -24,8 +27,25 @@ if command -v sudo > /dev/null; then csudo="sudo" fi +function cp_rpm_package() { +local cur_dir +cd $1 +cur_dir=$(pwd) + +for dirlist in $(ls ${cur_dir}); do + if test -d ${dirlist}; then + cd ${dirlist} + cp_rpm_package ${cur_dir}/${dirlist} + cd .. + fi + if test -e ${dirlist}; then + cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm + fi +done +} + if [ -d ${pkg_dir} ]; then - ${csudo} rm -rf ${pkg_dir} + ${csudo} rm -rf ${pkg_dir} fi ${csudo} mkdir -p ${pkg_dir} cd ${pkg_dir} @@ -35,7 +55,8 @@ ${csudo} mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS ${csudo} rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file} # copy rpm package to output_dir, then clean temp dir -#echo "rmpbuild end, cur_dir: $(pwd) " -${csudo} cp -rf RPMS/* ${output_dir} +#${csudo} cp -rf RPMS/* ${output_dir} +cp_rpm_package ${pkg_dir}/RPMS + cd .. ${csudo} rm -rf ${pkg_dir} diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index c0c0eacfae..ef02fb90fc 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -2,7 +2,7 @@ %define cfg_install_dir /etc/taos %define __strip /bin/true -Name: tdengine +Name: TDengine Version: %{_version} Release: 3%{?dist} Summary: tdengine from taosdata diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 7e22116ac5..b4948bb3a7 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -19,8 +19,8 @@ code_dir="${top_dir}/src" release_dir="${top_dir}/release" community_dir="${script_dir}/../../../community/src" -package_name='linux' -install_dir="${release_dir}/taos-client-${version}-${package_name}-$(echo ${build_time}| tr ': ' -)" +#package_name='linux' +install_dir="${release_dir}/TDengine-client-${version}" # Directories and files. bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${script_dir}/remove_client.sh" diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 24f3a0b8d1..714b74dbe6 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -16,8 +16,8 @@ build_dir="${compile_dir}/build" code_dir="${top_dir}/src" release_dir="${top_dir}/release" -package_name='linux' -install_dir="${release_dir}/taos-${version}-${package_name}-$(echo ${build_time}| tr ': ' -)" +#package_name='linux' +install_dir="${release_dir}/TDengine-${version}" # Directories and files. bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${script_dir}/remove.sh" diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 2eb9893556..ecda6912c3 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -29,9 +29,19 @@ #elif defined(DARWIN) char CLIENT_VERSION[] = "Welcome to the TDengine shell from mac, client version:%s "; #else - char CLIENT_VERSION[] = "Welcome to the TDengine shell from linux, client version:%s "; + #ifdef CLUSTER + char CLIENT_VERSION[] = "Welcome to the TDengine shell from linux, enterprise client version:%s "; + #else + char CLIENT_VERSION[] = "Welcome to the TDengine shell from linux, community client version:%s "; + #endif #endif -char SERVER_VERSION[] = "server version:%s\nCopyright (c) 2017 by TAOS Data, Inc. All rights reserved.\n\n"; + +#ifdef CLUSTER + char SERVER_VERSION[] = "enterprise server version:%s\nCopyright (c) 2017 by TAOS Data, Inc. All rights reserved.\n\n"; +#else + char SERVER_VERSION[] = "community server version:%s\nCopyright (c) 2017 by TAOS Data, Inc. All rights reserved.\n\n"; +#endif + char PROMPT_HEADER[] = "taos> "; char CONTINUE_PROMPT[] = " -> "; int prompt_size = 6; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 0ece4efbb2..ad8bf6c5c3 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -105,6 +105,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { static struct argp argp = {options, parse_opt, args_doc, doc}; void shellParseArgument(int argc, char *argv[], struct arguments *arguments) { + char verType[32] = {0}; + #ifdef CLUSTER + sprintf(verType, "enterprise version: %s\n", version); + #else + sprintf(verType, "community version: %s\n", version); + #endif + + argp_program_version = verType; + argp_parse(&argp, argc, argv, 0, 0, arguments); if (arguments->abort) { error(10, 0, "ABORTED"); diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index 9764afc593..f03bd5f3bb 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -55,7 +55,12 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } } else if (strcmp(argv[i], "-V") == 0) { - printf("version: %s compatible_version: %s\n", version, compatible_version); + #ifdef CLUSTER + printf("enterprise version: %s compatible_version: %s\n", version, compatible_version); + #else + printf("community version: %s compatible_version: %s\n", version, compatible_version); + #endif + printf("gitinfo: %s\n", gitinfo); printf("buildinfo: %s\n", buildinfo); return 0; From 2002c8e20b0774be018e4d58d7f0e92a1c2919ac Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 20:34:32 +0800 Subject: [PATCH 15/42] rror may occur when reading http chunked body --- src/modules/http/src/httpHandle.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/modules/http/src/httpHandle.c b/src/modules/http/src/httpHandle.c index 16e8378fb8..c736825b37 100644 --- a/src/modules/http/src/httpHandle.c +++ b/src/modules/http/src/httpHandle.c @@ -293,19 +293,14 @@ bool httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) { int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) { int dataReadLen = pParser->bufsize - (int)(pParser->data.pos - pParser->buffer); if (dataReadLen > pParser->data.len) { - httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, dataReadLen:%d > pContext->data.len:%d", - pContext, pContext->fd, pContext->ipstr, dataReadLen, pParser->data.len); + httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, read size:%d dataReadLen:%d > pContext->data.len:%d", + pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR); return HTTP_CHECK_BODY_ERROR; } else if (dataReadLen < pParser->data.len) { - httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, dataReadLen:%d < pContext->data.len:%d, continue read", - pContext, pContext->fd, pContext->ipstr, dataReadLen, pParser->data.len); - if (!httpReadDataImp(pContext)) { - httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr); - return HTTP_CHECK_BODY_ERROR; - } else { - return HTTP_CHECK_BODY_CONTINUE; - } + httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read", + pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len); + return HTTP_CHECK_BODY_CONTINUE; } else { return HTTP_CHECK_BODY_SUCCESS; } From 5d25f776a4bcb49b432e9809301eb623ec98cc2f Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 2 Dec 2019 22:02:11 +0800 Subject: [PATCH 16/42] python connector user guide --- .../tdenginedocs-cn/connector/index.html | 90 +++++++++++++++---- .../tdenginedocs-en/connector/index.html | 66 +++++++++++++- 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/documentation/tdenginedocs-cn/connector/index.html b/documentation/tdenginedocs-cn/connector/index.html index fca343e977..d808e42dda 100644 --- a/documentation/tdenginedocs-cn/connector/index.html +++ b/documentation/tdenginedocs-cn/connector/index.html @@ -114,23 +114,83 @@ public Connection getConn() throws Exception{

对于TDengine操作的报错信息，用户可使用JDBCDriver包里提供的枚举类TSDBError.java来获取error message和error code的列表。对于更多的具体操作的相关代码，请参考TDengine提供的使用示范项目JDBCDemo。

Python Connector

-

Python客户端安装

-

用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包。用户可以通过pip命令安装：

-

​ pip install src/connector/python/python2/

-

或

-

​ pip install src/connector/python/python3/

-

如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。

-

Python客户端接口

+

安装准备

+
  • 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端
    • +
  • 已安装python 2.7 or >= 3.4
    • +
  • 已安装pip
    • +

    安装

    +

    Linux

    +

    用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包， 然后通过pip命令安装

    +
    pip install src/connector/python/linux/python2/
    +

    或者

    +
    pip install src/connector/python/linux/python3/
    +

    Windows

    +

    在已安装Windows 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python2\
    +

    或者

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python3\
    +

    * 如果机器上没有pip命令，用户可将src/connector/python/windows/python3或src/connector/python/windows/python2下的taos文件夹拷贝到应用程序的目录使用。

    +

    Python client interfaces

    在使用TDengine的python接口时，需导入TDengine客户端模块：

    -
    import taos
    -

    用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法：

    +
    import taos
    +

    代码示例

    +
  • 获取连接
    • +
    
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+
    +
  • 写入数据
    • +
    
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+
    +
  • 写入数据
    • +
    +c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+
     +
  • 关闭连接
    • +
    +c1.close()
+conn.close()
+
     +

    帮助信息

    +

    用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法:

      -

    • TaosConnection类

      -

      参考python中help(taos.TaosConnection)。

      • -

    • TaosCursor类

      -

      参考python中help(taos.TaosCursor)。

      • -

    • connect方法

      -

      用于生成taos.TaosConnection的实例。

      • +

    • TaosConnection 类

      +

      参考python中help(taos.TDengineConnection)

      • +

    • TaosCursor 类

      +

      参考python中help(taos.TDengineCursor)

      • +

    • connect 方法

      +

      用于生成taos.TDengineConnection的实例。

    RESTful Connector

    为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。

    diff --git a/documentation/tdenginedocs-en/connector/index.html b/documentation/tdenginedocs-en/connector/index.html index ce32c062ff..ba1cbcaf85 100644 --- a/documentation/tdenginedocs-en/connector/index.html +++ b/documentation/tdenginedocs-en/connector/index.html @@ -122,15 +122,75 @@ public Connection getConn() throws Exception{

    All the error codes and error messages can be found in TSDBError.java . For a more detailed coding example, please refer to the demo project JDBCDemo in TDengine's code examples.

    Python Connector

    +

    Pre-requirement

    +
  • TDengine installed, TDengine-client installed if on Windows
    • +
  • python 2.7 or >= 3.4
    • +
  • pip installed

    • Install TDengine Python client

    +

    Linux

    Users can find python client packages in our source code directory src/connector/python. There are two directories corresponding two python versions. Please choose the correct package to install. Users can use pip command to install:

    -
    pip install src/connector/python/python2/
    +
    pip install src/connector/python/linux/python2/

    or

    -
    pip install src/connector/python/python3/
    -

    If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.

    +
    pip install src/connector/python/linux/python3/
    +

    Windows

    +

    Assume you have installed the Windows client , copy the file "C:\TDengine\driver\taos.dll" to the folder "C:\windows\system32", and then enter the cmd command interface

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python2\
    +

    or

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python3\
    +

    * If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.

    Python client interfaces

    To use TDengine Python client, import TDengine module at first:

    import taos
    +

    Examples

    +
  • get the connection
    • +
    
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+
    +
  • insert records into the database
    • +
    
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+
    +
  • query the database
    • +
    +c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+
     +
  • close the connection
    • +
    +c1.close()
+conn.close()
+
     +

    Help information

    Users can get module information from Python help interface or refer to our [python code example](). We list the main classes and methods below:

    • TaosConnection class

      From 387a049e388b9e9a83d7033431832db59730765c Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 22:27:38 +0800 Subject: [PATCH 17/42] version.c --- src/util/src/version.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/src/version.c b/src/util/src/version.c index ed83066f80..4eab3fb7e2 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,4 @@ -char version[64] = "1.6.4.0"; +char version[64] = "1.6.4.1"; char compatible_version[64] = "1.6.1.0"; -char gitinfo[128] = "6d27c11e3b23ae69366df366a6517853648c41f7"; -char buildinfo[512] = "Built by ubuntu at 2019-12-01 12:27"; +char gitinfo[128] = "893fac9da79ef9b88355fcd18d29057adf909bbd"; +char buildinfo[512] = "Built by ubuntu at 2019-12-02 22:21"; From bb370933c7eae96004682bb039ccdcf076e20902 Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 3 Dec 2019 08:48:13 +0800 Subject: [PATCH 18/42] python connector user guide --- .../tdenginedocs-cn/connector/index.html | 23 ++++++++++--------- .../tdenginedocs-en/connector/index.html | 13 ++++++----- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/documentation/tdenginedocs-cn/connector/index.html b/documentation/tdenginedocs-cn/connector/index.html index d808e42dda..8b4e989e8a 100644 --- a/documentation/tdenginedocs-cn/connector/index.html +++ b/documentation/tdenginedocs-cn/connector/index.html @@ -125,35 +125,36 @@ public Connection getConn() throws Exception{

      或者

      pip install src/connector/python/linux/python3/

      Windows

      -

      在已安装Windows 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面

      +

      在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面

      cd C:\TDengine\connector\python\windows
      pip install python2\

      或者

      cd C:\TDengine\connector\python\windows
      pip install python3\

      * 如果机器上没有pip命令，用户可将src/connector/python/windows/python3或src/connector/python/windows/python2下的taos文件夹拷贝到应用程序的目录使用。

      -

      Python client interfaces

      -

      在使用TDengine的python接口时，需导入TDengine客户端模块：

      -
      import taos
      +

      使用

      代码示例

      +
    • 导入TDengine客户端模块：
      • +
      import taos
    • 获取连接
      • 
 conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
 c1 = conn.cursor()
      +

      * host 是TDengine 服务端所有IP, config 为客户端配置文件所在目录

    • 写入数据
      • 
 import datetime
  
-# create a database
+# 创建数据库
 c1.execute('create database db')
 c1.execute('use db')
-# create a table
+# 建表
 c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
-# insert a record
+# 插入数据
 start_time = datetime.datetime(2019, 11, 1)
 affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
-# insert multiple records in a batch
+# 批量插入数据
 time_interval = datetime.timedelta(seconds=60)
 sqlcmd = ['insert into tb values']
 for irow in range(1,11):
@@ -164,15 +165,15 @@ affected_rows = c1.execute(' '.join(sqlcmd))
 
    • 写入数据
      • 
 
       c1.execute('select * from tb')
-# fetch all returned results
+# 拉取查询结果
 data = c1.fetchall()
-# data is a list of returned rows with each row being a tuple
+# 返回的结果是一个列表，每一行构成列表的一个元素
 numOfRows = c1.rowcount
 numOfCols = c1.descriptions
 for irow in range(numOfRows):
   print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
   
-# use the cursor as an iterator to retrieve all returned results
+# 直接使用cursor 循环拉取查询结果
 c1.execute('select * from tb')
 for data in c1:
   print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
diff --git a/documentation/tdenginedocs-en/connector/index.html b/documentation/tdenginedocs-en/connector/index.html
index ba1cbcaf85..0f9e6b4717 100644
--- a/documentation/tdenginedocs-en/connector/index.html
+++ b/documentation/tdenginedocs-en/connector/index.html
@@ -126,29 +126,30 @@ public Connection getConn() throws Exception{
 
    • TDengine installed, TDengine-client installed if on Windows
      • 
 
    • python 2.7 or >= 3.4
      • 
 
    • pip installed 
      • 
-
      Install TDengine Python client
      
+
      Installation
      
 
      Linux
      
-
      Users can find python client packages in our source code directory src/connector/python. There are two directories corresponding two python versions. Please choose the correct package to install. Users can use pip command to install:
      
+
      Users can find python client packages in our source code directory src/connector/python. There are two directories corresponding to two python versions. Please choose the correct package to install. Users can use pip command to install:
      
 
      pip install src/connector/python/linux/python2/
      
 
      or
      
 
      pip install src/connector/python/linux/python3/
      
 
      Windows
      
-
      Assume you have installed the Windows client , copy the file "C:\TDengine\driver\taos.dll" to the folder "C:\windows\system32", and then enter the cmd command interface
      
+
      Assumed the Windows TDengine client has been installed , copy the file "C:\TDengine\driver\taos.dll" to the folder "C:\windows\system32", and then enter the cmd Windows command interface
      
 
      cd C:\TDengine\connector\python\windows
      
 
      pip install python2\
      
 
      or
      
 
      cd C:\TDengine\connector\python\windows
      
 
      pip install python3\
      
 
      * If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.
      
-
      Python client interfaces
      
-
      To use TDengine Python client, import TDengine module at first:
      
-
      import taos 
      
+
      Usage
      
 
      Examples
      
+
    • import TDengine module at first:
      • 
+
      import taos 
      
 
    • get the connection
      • 
 
      
 conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
 c1 = conn.cursor()
 
      
+
      * host is the IP of TDengine server, and config is the directory where exists the TDengine client configure file
      
 
    • insert records into the database
      • 
 
      
 import datetime

From 61f1506d3c92f59cad68d0e05daa3fc7f4c5c7c5 Mon Sep 17 00:00:00 2001
From: Allenscript <990479310@qq.com>
Date: Tue, 3 Dec 2019 13:07:42 +0800
Subject: [PATCH 19/42] fix compile for raspbian on RaspberryPI 4B atomic not
 found error

---
 src/os/linux/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/os/linux/CMakeLists.txt b/src/os/linux/CMakeLists.txt
index a702cac759..0f725947cc 100644
--- a/src/os/linux/CMakeLists.txt
+++ b/src/os/linux/CMakeLists.txt
@@ -6,5 +6,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(os ${SRC})
-  TARGET_LINK_LIBRARIES(os m rt)
+  TARGET_LINK_LIBRARIES(os m rt atomic)
 ENDIF ()

From 13298fe5d6b5c7d48735139a42a375e2f6c301e6 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Tue, 3 Dec 2019 13:54:33 +0800
Subject: [PATCH 20/42] [TBASE-816]

---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a1b2c16f46..71cb0bfd43 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -146,7 +146,7 @@ IF (NOT DEFINED TD_CLUSTER)
         SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -Wno-char-subscripts -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
       ENDIF ()
     ELSE ()
-      SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -Wno-char-subscripts -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -Wno-char-subscripts -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
     ENDIF ()
     ADD_DEFINITIONS(-DLINUX)
     ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
@@ -156,7 +156,7 @@ IF (NOT DEFINED TD_CLUSTER)
     ENDIF ()
     SET(DEBUG_FLAGS "-O0 -DDEBUG")
     SET(RELEASE_FLAGS "-O0")
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -Wno-char-subscripts -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -Wno-char-subscripts -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
     ADD_DEFINITIONS(-DLINUX)
     ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
     ADD_DEFINITIONS(-DUSE_LIBICONV)

From c933b3ea68c049c9c6fc784f589b603cfe59fb59 Mon Sep 17 00:00:00 2001
From: fang 
Date: Tue, 3 Dec 2019 14:12:31 +0800
Subject: [PATCH 21/42] python connector user guide

---
 .../tdenginedocs-cn/connector/index.html      |  2 +-
 .../webdocs/markdowndocs/Connector.md         | 82 +++++++++++++++--
 .../webdocs/markdowndocs/connector-ch.md      | 89 +++++++++++++++++--
 3 files changed, 154 insertions(+), 19 deletions(-)

diff --git a/documentation/tdenginedocs-cn/connector/index.html b/documentation/tdenginedocs-cn/connector/index.html
index 8b4e989e8a..3167c1521f 100644
--- a/documentation/tdenginedocs-cn/connector/index.html
+++ b/documentation/tdenginedocs-cn/connector/index.html
@@ -162,7 +162,7 @@ for irow in range(1,11):
   sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
 affected_rows = c1.execute(' '.join(sqlcmd))
 
      
-
    • 写入数据
      • 
+
    • 查询数据
      • 
 
       c1.execute('select * from tb')
 # 拉取查询结果
diff --git a/documentation/webdocs/markdowndocs/Connector.md b/documentation/webdocs/markdowndocs/Connector.md
index a8cea06cc8..2ecb14c710 100644
--- a/documentation/webdocs/markdowndocs/Connector.md
+++ b/documentation/webdocs/markdowndocs/Connector.md
@@ -273,29 +273,93 @@ All the error codes and error messages can be found in `TSDBError.java` . For a
 
 ## Python Connector
 
-### Install TDengine Python client
+### Pre-requirement
+* TDengine installed, TDengine-client installed if on Windows
+* python 2.7 or >= 3.4
+* pip installed
 
-Users can find python client packages in our source code directory _src/connector/python_. There are two directories corresponding two python versions. Please choose the correct package to install. Users can use _pip_ command to install:
+### Installation
+#### Linux
+
+Users can find python client packages in our source code directory _src/connector/python_. There are two directories corresponding to two python versions. Please choose the correct package to install. Users can use _pip_ command to install:
 
 ```cmd
-pip install src/connector/python/[linux|Windows]/python2/
+pip install src/connector/python/linux/python3/
 ```
 
 or
 
 ```
-pip install src/connector/python/[linux|Windows]/python3/
+pip install src/connector/python/linux/python2/
 ```
+#### Windows
+Assumed the Windows TDengine client has been installed , copy the file "C:\TDengine\driver\taos.dll" to the folder "C:\windows\system32", and then enter the _cmd_ Windows command interface
+```
+cd C:\TDengine\connector\python\windows
+pip install python3\
+```
+or
+```
+cd C:\TDengine\connector\python\windows
+pip install python2\
+```
+*If _pip_ command is not installed on the system, users can choose to install pip or just copy the _taos_ directory in the python client directory to the application directory to use.
 
-If _pip_ command is not installed on the system, users can choose to install pip or just copy the _taos_ directory in the python client directory to the application directory to use.
-
-### Python client interfaces
-
-To use TDengine Python client, import TDengine module at first:
+### Usage
+#### Examples
+* import TDengine module
 
 ```python
 import taos 
 ```
+* get the connection
+```python
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+```
+*host is the IP of TDengine server, and config is the directory where exists the TDengine client configure file
+* insert records into the database
+```python
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+```
+* query the database
+```python
+c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+```
+* close the connection
+```python
+c1.close()
+conn.close()
+```
+#### Help information
 
 Users can get module information from Python help interface or refer to our [python code example](). We list the main classes and methods below:
 
diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md
index 9261b9eef6..4266ddffc4 100644
--- a/documentation/webdocs/markdowndocs/connector-ch.md
+++ b/documentation/webdocs/markdowndocs/connector-ch.md
@@ -265,26 +265,97 @@ public Connection getConn() throws Exception{
 
 ## Python Connector
 
+### 安装准备
+* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端
+* 已安装python 2.7 or >= 3.4
+* 已安装pip
+
 ### Python客户端安装
 
+#### Linux
+
 用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包。用户可以通过pip命令安装： 
 
-​		`pip install src/connector/python/[linux|windows]/python2/`
+​		`pip install src/connector/python/linux/python2/`
 
 或
 
-​		`pip install src/connector/python/[linux|windows]/python3/`
-
-如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。
-对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。所有TDengine的连接器，均需依赖taos.dll。
-
-### Python客户端接口
-
-在使用TDengine的python接口时，需导入TDengine客户端模块：
+​		`pip install src/connector/python/linux/python3/`
 
+#### Windows
+在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面
+```cmd
+cd C:\TDengine\connector\python\windows
+pip install python2\
 ```
+或
+```cmd
+cd C:\TDengine\connector\python\windows
+pip install python3\
+```
+
+*如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。
+对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。
+
+### 使用
+
+#### 代码示例
+
+* 导入TDengine客户端模块
+
+```python
 import taos 
 ```
+* 获取连接
+```python
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+```
+*host 是TDengine 服务端所有IP, config 为客户端配置文件所在目录
+
+* 写入数据
+```python
+import datetime
+ 
+# 创建数据库
+c1.execute('create database db')
+c1.execute('use db')
+# 建表
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# 插入数据
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# 批量插入数据
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+```
+
+* 查询数据
+```python
+c1.execute('select * from tb')
+# 拉取查询结果
+data = c1.fetchall()
+# 返回的结果是一个列表，每一行构成列表的一个元素
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# 直接使用cursor 循环拉取查询结果
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+```
+* 关闭连接
+```python
+c1.close()
+conn.close()
+```
+#### 帮助信息
 
 用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法：
 

From 9a659e3b4d9ea6db7b03e38f50287acbc28bd109 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Tue, 3 Dec 2019 15:11:04 +0800
Subject: [PATCH 22/42] [TBASE-816]

---
 packaging/deb/makedeb.sh | 12 +++++++++++-
 packaging/release.sh     |  4 ++--
 packaging/rpm/makerpm.sh |  7 +++++++
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh
index 501a06eddb..17bb5aabae 100755
--- a/packaging/deb/makedeb.sh
+++ b/packaging/deb/makedeb.sh
@@ -7,6 +7,7 @@
 compile_dir=$1
 output_dir=$2
 tdengine_ver=$3
+armver=$4
 
 script_dir="$(dirname $(readlink -f $0))"
 top_dir="$(readlink -m ${script_dir}/../..)"
@@ -63,7 +64,16 @@ debver="Version: "$tdengine_ver
 sed -i "2c$debver" ${pkg_dir}/DEBIAN/control
  
 #get taos version, then set deb name
-debname="TDengine-"${tdengine_ver}".deb"
+if [ -z "$armver" ]; then
+  debname="TDengine-"${tdengine_ver}".deb"
+elif [ "$armver" == "arm64" ]; then
+  debname="TDengine-"${tdengine_ver}"-arm64.deb"
+elif [ "$armver" == "arm32" ]; then
+  debname="TDengine-"${tdengine_ver}-arm32".deb"
+else
+  echo "input parameter error!!!"
+  return
+fi
 
 # make deb package
 dpkg -b ${pkg_dir} $debname
diff --git a/packaging/release.sh b/packaging/release.sh
index 58f69589d8..0c806a159f 100755
--- a/packaging/release.sh
+++ b/packaging/release.sh
@@ -149,7 +149,7 @@ if [ -d ${output_dir} ]; then
 fi  
 ${csudo} mkdir -p ${output_dir} 
 cd ${script_dir}/deb
-${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version}
+${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version} ${armver}
   
 echo "do rpm package for the centos system"
 output_dir="${top_dir}/rpms"
@@ -158,7 +158,7 @@ if [ -d ${output_dir} ]; then
 fi
 ${csudo} mkdir -p ${output_dir}  
 cd ${script_dir}/rpm
-${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version}
+${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version} ${armver}
 
 echo "do tar.gz package for all systems"  
 cd ${script_dir}/tools
diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh
index e301f5ece9..aef01875cb 100755
--- a/packaging/rpm/makerpm.sh
+++ b/packaging/rpm/makerpm.sh
@@ -9,6 +9,7 @@
 compile_dir=$1
 output_dir=$2
 tdengine_ver=$3
+armver=$4
 
 script_dir="$(dirname $(readlink -f $0))"
 top_dir="$(readlink -m ${script_dir}/../..)"
@@ -58,5 +59,11 @@ ${csudo} rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_di
 #${csudo} cp -rf RPMS/* ${output_dir}
 cp_rpm_package ${pkg_dir}/RPMS 
 
+if [ "$armver" == "arm64" ]; then
+  mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/TDengine-${tdengine_ver}-arm64.rpm
+elif [ "$armver" == "arm32" ]; then
+  mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/TDengine-${tdengine_ver}-arm32.rpm
+fi
+
 cd ..
 ${csudo} rm -rf ${pkg_dir}

From 8fb2d0c03eb435865e50c4432c27b5b0aae3f6b6 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Tue, 3 Dec 2019 15:22:35 +0800
Subject: [PATCH 23/42] [TBASE-816]

---
 packaging/deb/makedeb.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh
index 17bb5aabae..5c2df734fa 100755
--- a/packaging/deb/makedeb.sh
+++ b/packaging/deb/makedeb.sh
@@ -69,7 +69,7 @@ if [ -z "$armver" ]; then
 elif [ "$armver" == "arm64" ]; then
   debname="TDengine-"${tdengine_ver}"-arm64.deb"
 elif [ "$armver" == "arm32" ]; then
-  debname="TDengine-"${tdengine_ver}-arm32".deb"
+  debname="TDengine-"${tdengine_ver}"-arm32.deb"
 else
   echo "input parameter error!!!"
   return

From bff25a36a47cabdb85dd3ece84c900c3318c3c6d Mon Sep 17 00:00:00 2001
From: slguan 
Date: Tue, 3 Dec 2019 17:37:10 +0800
Subject: [PATCH 24/42] C # usage

---
 .../webdocs/markdowndocs/connector-ch.md      | 36 ++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md
index 85596f2053..e96f6f21d2 100644
--- a/documentation/webdocs/markdowndocs/connector-ch.md
+++ b/documentation/webdocs/markdowndocs/connector-ch.md
@@ -502,7 +502,7 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间
 
 #### 安装TDengine
 
-Go的链接器使用了到了 libtaos.so 和taos.h，因此，在使用Go连接器之前，需要在程序运行的机器上安装TDengine以获得相关的驱动文件。
+Go的连接器使用到了 libtaos.so 和taos.h，因此，在使用Go连接器之前，需要在程序运行的机器上安装TDengine以获得相关的驱动文件。
 
 #### Go语言引入package
 TDengine提供了GO驱动程序“taosSql”包。taosSql驱动包是基于GO的“database/sql/driver”接口的实现。用户可以通过`go get`命令来获取驱动包。
@@ -561,3 +561,37 @@ taosSql驱动包内采用cgo模式，调用了TDengine的C/C++同步接口，与
 3. 创建表、写入和查询数据
 
 在创建好了数据库后，就可以开始创建表和写入查询数据了。这些操作的基本思路都是首先组装SQL语句，然后调用db.Exec执行，并检查错误信息和执行相应的处理。可以参考上面的样例代码
+
+## CSharp Connector
+
+在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。
+
+#### 安装TDengine客户端
+
+C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。
+
+安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件
+
+- TDengineDriver.cs 调用taos.dll文件的Native C方法
+- TDengineTest.cs 参考程序示例
+
+在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件
+
+#### 使用方法
+
+- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中
+- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法
+- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中
+
+#### 注意事项
+
+- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。
+- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。
+
+#### 第三方驱动
+
+Maikebing.Data.Taos是一个基于TDengine的RESTful Connector构建的ADO.Net提供器，该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考
+
+```
+https://gitee.com/maikebing/Maikebing.EntityFrameworkCore.Taos
+```

From ee33d78494db839cd5c457af5f7926158feabedc Mon Sep 17 00:00:00 2001
From: slguan 
Date: Tue, 3 Dec 2019 19:00:24 +0800
Subject: [PATCH 25/42] windows client user manual

---
 .../webdocs/markdowndocs/connector-ch.md      | 82 ++++++++++++++++++-
 1 file changed, 81 insertions(+), 1 deletion(-)

diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md
index e96f6f21d2..ce996f4876 100644
--- a/documentation/webdocs/markdowndocs/connector-ch.md
+++ b/documentation/webdocs/markdowndocs/connector-ch.md
@@ -310,7 +310,7 @@ import taos
 http://:/rest/sql
 ```
 
-​参数说明：
+参数说明：
 
 - IP: 集群中的任一台主机
 - PORT: 配置文件中httpPort配置项，缺省为6020
@@ -595,3 +595,83 @@ Maikebing.Data.Taos是一个基于TDengine的RESTful Connector构建的ADO.Net
 ```
 https://gitee.com/maikebing/Maikebing.EntityFrameworkCore.Taos
 ```
+
+## Windows客户端及程序接口
+
+### 客户端安装
+
+在Windows操作系统下，TDengine提供64位的Windows客户端，客户端安装程序为.exe文件，运行该文件即可安装，安装路径为C:\TDengine。Windows的客户端可运行在主流的64位Windows平台之上，客户端目录结构如下：
+
+```
+├── cfg
+│   └── taos.cfg
+├── connector
+│   ├── go
+│   ├── grafana
+│   ├── jdbc
+│   └── python
+├── driver
+│   ├── taos.dll
+│   ├── taos.exp
+│   └── taos.lib
+├── examples
+│   ├── bash
+│   ├── c
+│   ├── C#
+│   ├── go
+│   ├── JDBC
+│   ├── lua
+│   ├── matlab
+│   ├── nodejs
+│   ├── python
+│   ├── R
+│   └── rust
+├── include
+│   └── taos.h
+└── taos.exe
+```
+
+其中，最常用的文件列出如下：
+
++ Client可执行文件: C:/TDengine/taos.exe 
++ 配置文件: C:/TDengine/cfg/taos.cfg
++ C驱动程序目录: C:/TDengine/driver
++ C驱动程序头文件: C:/TDengine/include
++ JDBC驱动程序目录: C:/TDengine/connector/jdbc
++ GO驱动程序目录：C:/TDengine/connector/go
++ Python驱动程序目录：C:/TDengine/connector/python
++ C#驱动程序及示例代码: C:/TDengine/examples/C#
++ 日志目录（第一次运行程序时生成）：C:/TDengine/log
+
+### 注意事项
+
+#### Shell工具注意事项
+
+在开始菜单中搜索cmd程序，通过命令行方式执行taos.exe即可打开TDengine的Client程序，如下所示，其中ServerIP为TDengine所在Linux服务器的IP地址
+
+```
+taos -h 
+```
+
+在cmd中对taos的使用与Linux平台没有差别，但需要注意以下几点：
+
++ 确保Windows防火墙或者其他杀毒软件处于关闭状态，TDengine的服务端与客户端通信的端口请参考`服务端配置`章节
++ 确认客户端连接时指定了正确的服务器IP地址
++ ping服务器IP，如果没有反应，请检查你的网络
+
+#### C++接口注意事项
+
+TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序使用时，需要包含TDengine头文件taos.h，连接时需要链接TDengine库taos.lib，运行时将taos.dll放到可执行文件目录下。
+
+#### JDBC接口注意事项
+
+在Windows系统上，应用程序可以使用JDBC接口来操纵数据库，使用JDBC接口的注意事项如下：
+
++ 将JDBC驱动程序(JDBCDriver-1.0.0-dist.jar)放置到当前的CLASS_PATH中;
+
++ 将Windows开发包(taos.dll)放置到system32目录下。
+
+
+
+
+

From c2d353f5432cfeb200ee4b189ae4651958a52dcc Mon Sep 17 00:00:00 2001
From: haojun Liao 
Date: Tue, 3 Dec 2019 22:43:20 +0800
Subject: [PATCH 26/42] Update administrator-ch.md

add more information about client configuration.
---
 .../webdocs/markdowndocs/administrator-ch.md  | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index ed822fb8c9..b2e66b7568 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -66,12 +66,25 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 
 ## 客户端配置 
 
-TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见[Shell命令行程序](#_TDengine_Shell命令行程序)。本节主要讲解taos客户端应用在配置文件taos.cfg文件中使用到的参数。
+TDengine系统的前台交互客户端应用程序为taos（Windows平台上为taos.exe），可以使用taos.cfg来配置启动和运行配置项。可以使用 `taos -?` 来获取全部的可选项。
+启动的时候如果不指定taos加载配置文件路径，默认读取`/etc/taos/`路径下的`taos.cfg`文件来设置启动选项，如果在默认路径下找不到配置文件，则使用默认配置的设置来启动程序，并且会在启动的时候打印一行告警信息。指定配置文件来启动`taos`的命令如下：
+```
+taos -c /home/cfg/
+```
+**注意：启动设置的是配置文件所在目录，而不是配置文件本身**
 
-客户端配置参数列表及解释
+如果`/home/cfg/`目录下没有配置文件，程序会继续启动并打印如下告警信息：
+```
+Welcome to the TDengine shell from linux, client version:1.6.4.0
+option file:/home/cfg/taos.cfg not found, all options are set to system default
+```
+更多taos的使用方法请见[Shell命令行程序](#_TDengine_Shell命令行程序)。本节主要讲解taos客户端应用在配置文件taos.cfg文件中使用到的参数。
+
+客户端配置参数如下
 
 - masterIP：客户端默认发起请求的服务器的IP地址
-- charset：指明客户端所使用的字符集，默认值为UTF-8。TDengine存储nchar类型数据时使用的是unicode存储，因此客户端需要告知服务自己所使用的字符集，也即客户端所在系统的字符集。
+- charset：指明客户端所使用的字符集，默认值为UTF-8。TDengine为了存储中文、日文等非ASCII编码的宽字符，提供一种专门的字段类型`nchar`。如果数据写入`nchar`字段，将统一采用`UCS4-LE`格式对字段中数据进行编码存储。需要注意的是，**编码正确性**是客户端来保证。因此，如果用户想要正常使用`nchar`字段来存储诸如中文、日文等非ASCII字符，需要正确设置客户端的编码格式。客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为`UTF-8`，也有部分中文编码，例如`GB18030`或`GBK`等。在docker环境中默认的编码是`POSIX`。因此，客户端需要设置自己所使用的字符集，即所在系统的当前编码字符集才能正确地确保数据转换为`UCS4-LE`编码格式。
+如果配置文件中不设置`charset`，taos将自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则中断启动，要求用户在配置文件中指定charset，才能正常启动。
 - locale：设置系统语言环境。Linux上客户端与服务端共享
 - defaultUser：默认登录用户，默认值root
 - defaultPass：默认登录密码，默认值taosdata
@@ -193,4 +206,4 @@ KILL STREAM 
 
 TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
 
-这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
\ No newline at end of file
+这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。

From 90b3558e97745a3e1cb6d0cdd64d997501e50e87 Mon Sep 17 00:00:00 2001
From: haojun Liao 
Date: Tue, 3 Dec 2019 22:43:22 +0800
Subject: [PATCH 27/42] Update administrator-ch.md

add more information about client configuration.
---
 .../webdocs/markdowndocs/administrator-ch.md  | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index ed822fb8c9..b2e66b7568 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -66,12 +66,25 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 
 ## 客户端配置 
 
-TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见[Shell命令行程序](#_TDengine_Shell命令行程序)。本节主要讲解taos客户端应用在配置文件taos.cfg文件中使用到的参数。
+TDengine系统的前台交互客户端应用程序为taos（Windows平台上为taos.exe），可以使用taos.cfg来配置启动和运行配置项。可以使用 `taos -?` 来获取全部的可选项。
+启动的时候如果不指定taos加载配置文件路径，默认读取`/etc/taos/`路径下的`taos.cfg`文件来设置启动选项，如果在默认路径下找不到配置文件，则使用默认配置的设置来启动程序，并且会在启动的时候打印一行告警信息。指定配置文件来启动`taos`的命令如下：
+```
+taos -c /home/cfg/
+```
+**注意：启动设置的是配置文件所在目录，而不是配置文件本身**
 
-客户端配置参数列表及解释
+如果`/home/cfg/`目录下没有配置文件，程序会继续启动并打印如下告警信息：
+```
+Welcome to the TDengine shell from linux, client version:1.6.4.0
+option file:/home/cfg/taos.cfg not found, all options are set to system default
+```
+更多taos的使用方法请见[Shell命令行程序](#_TDengine_Shell命令行程序)。本节主要讲解taos客户端应用在配置文件taos.cfg文件中使用到的参数。
+
+客户端配置参数如下
 
 - masterIP：客户端默认发起请求的服务器的IP地址
-- charset：指明客户端所使用的字符集，默认值为UTF-8。TDengine存储nchar类型数据时使用的是unicode存储，因此客户端需要告知服务自己所使用的字符集，也即客户端所在系统的字符集。
+- charset：指明客户端所使用的字符集，默认值为UTF-8。TDengine为了存储中文、日文等非ASCII编码的宽字符，提供一种专门的字段类型`nchar`。如果数据写入`nchar`字段，将统一采用`UCS4-LE`格式对字段中数据进行编码存储。需要注意的是，**编码正确性**是客户端来保证。因此，如果用户想要正常使用`nchar`字段来存储诸如中文、日文等非ASCII字符，需要正确设置客户端的编码格式。客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为`UTF-8`，也有部分中文编码，例如`GB18030`或`GBK`等。在docker环境中默认的编码是`POSIX`。因此，客户端需要设置自己所使用的字符集，即所在系统的当前编码字符集才能正确地确保数据转换为`UCS4-LE`编码格式。
+如果配置文件中不设置`charset`，taos将自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则中断启动，要求用户在配置文件中指定charset，才能正常启动。
 - locale：设置系统语言环境。Linux上客户端与服务端共享
 - defaultUser：默认登录用户，默认值root
 - defaultPass：默认登录密码，默认值taosdata
@@ -193,4 +206,4 @@ KILL STREAM 
 
 TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
 
-这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
\ No newline at end of file
+这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。

From 5a00db8e427de6c4dff3e1fa4cd0bd10324e1149 Mon Sep 17 00:00:00 2001
From: fang 
Date: Wed, 4 Dec 2019 09:04:57 +0800
Subject: [PATCH 28/42] Chinese version for node.js connector

---
 .../webdocs/markdowndocs/Connector.md         |   8 +-
 .../webdocs/markdowndocs/connector-ch.md      | 136 ++++++++++++++++++
 2 files changed, 140 insertions(+), 4 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/Connector.md b/documentation/webdocs/markdowndocs/Connector.md
index 2ecb14c710..a979c0158e 100644
--- a/documentation/webdocs/markdowndocs/Connector.md
+++ b/documentation/webdocs/markdowndocs/Connector.md
@@ -533,15 +533,15 @@ To interact with TDengine, we make use of the [node-gyp](https://github.com/node
 
      in your Terminal) 
 
-    - This step will install `gcc` and the related toolchain containing `make`
+    - 安装`gcc` 和`make`
 
-### On Windows
+### Windows
 
-#### Option 1
+#### 安装方法1
 
 Install all the required tools and configurations using Microsoft's [windows-build-tools](https://github.com/felixrieseberg/windows-build-tools) using `npm install --global --production windows-build-tools` from an elevated PowerShell or CMD.exe (run as Administrator).
 
-#### Option 2
+#### 安装方法2
 
 Install tools and configuration manually:
 
diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md
index 4266ddffc4..1cc03703be 100644
--- a/documentation/webdocs/markdowndocs/connector-ch.md
+++ b/documentation/webdocs/markdowndocs/connector-ch.md
@@ -524,3 +524,139 @@ taosSql驱动包内采用cgo模式，调用了TDengine的C/C++同步接口，与
 3. 创建表、写入和查询数据
 
 在创建好了数据库后，就可以开始创建表和写入查询数据了。这些操作的基本思路都是首先组装SQL语句，然后调用db.Exec执行，并检查错误信息和执行相应的处理。可以参考上面的样例代码
+
+## Node.js Connector
+
+TDengine 同时也提供了node.js 的连接器。用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。[具体安装步骤如下](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)：
+
+首先，通过[npm](https://www.npmjs.com/)安装node.js 连接器.
+
+```cmd
+npm install td-connector
+```
+我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下
+
+To interact with TDengine, we make use of the [node-gyp](https://github.com/nodejs/node-gyp) library. To install, you will need to install the following depending on platform (the following instructions are quoted from node-gyp)我们使用[node-gyp](https://github.com/nodejs/node-gyp)和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件：
+
+### Unix
+
+- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
+- `make`
+- c语言编译器比如[GCC](https://gcc.gnu.org)
+
+### macOS
+
+- `python` (建议`v2.7` , `v3.x.x` 目前还不支持)
+
+- Xcode
+
+  - 然后通过Xcode安装
+
+    ```
+    Command Line Tools
+    ```
+
+    在
+    ```
+    Xcode -> Preferences -> Locations
+    ```
+
+    目录下可以找到这个工具。或者在终端里执行
+
+    ```
+    xcode-select --install
+    ```
+
+
+    - 该步执行后 `gcc` 和 `make`就被安装上了
+
+### Windows
+
+#### 安装方法1
+
+使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具
+
+#### 安装方法2
+
+手动安装以下工具:
+
+- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) 
+- 安装 [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` 
+- 进入`cmd`命令行界面, `npm config set msvs_version 2017`
+
+如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules)
+
+如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual  C++ ATL for ARM64".
+
+### 使用方法
+
+(http://docs.taosdata.com/node)
+以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[该文档](http://docs.taosdata.com/node)
+
+#### 连接
+
+使用node.js连接器时，必须先require```td-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信 
+
+```javascript
+const taos = require('td-connector');
+var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0})
+var cursor = conn.cursor(); // Initializing a new cursor
+```
+
+关闭连接可执行
+
+```javascript
+conn.close();
+```
+
+#### 查询
+
+可通过 ```cursor.query``` 函数来查询数据库。
+
+```javascript
+var query = cursor.query('show databases;')
+```
+
+查询的结果可以通过 ```query.execute()``` 函数获取并打印出来
+
+```javascript
+var promise = query.execute();
+promise.then(function(result) {
+  result.pretty(); 
+});
+```
+格式化查询语句还可以使用```query```的```bind```方法。如下面的示例：```query```会自动将提供的数值填入查询语句的```?```里。
+
+```javascript
+var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5);
+query.execute().then(function(result) {
+  result.pretty();
+})
+```
+如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下：
+
+
+```javascript
+var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true)
+promise.then(function(result) {
+  result.pretty();
+})
+```
+#### 异步函数
+异步查询数据库的操作和上面类似，只需要在`cursor.execute`, `TaosQuery.execute`等函数后面加上`_a`。
+```javascript
+var promise1 = cursor.query('select count(*), avg(v1), avg(v2) from meter1;').execute_a()
+var promise2 = cursor.query('select count(*), avg(v1), avg(v2) from meter2;').execute_a();
+promise1.then(function(result) {
+  result.pretty();
+})
+promise2.then(function(result) {
+  result.pretty();
+})
+```
+
+
+### 示例
+[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例
+
+[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`.
\ No newline at end of file

From ccb02180f41a7746f21c95ecbda04cf327570515 Mon Sep 17 00:00:00 2001
From: fang 
Date: Wed, 4 Dec 2019 09:10:49 +0800
Subject: [PATCH 29/42] Chinese version for node.js connector

---
 documentation/webdocs/markdowndocs/Connector.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/Connector.md b/documentation/webdocs/markdowndocs/Connector.md
index a979c0158e..2ecb14c710 100644
--- a/documentation/webdocs/markdowndocs/Connector.md
+++ b/documentation/webdocs/markdowndocs/Connector.md
@@ -533,15 +533,15 @@ To interact with TDengine, we make use of the [node-gyp](https://github.com/node
 
      in your Terminal) 
 
-    - 安装`gcc` 和`make`
+    - This step will install `gcc` and the related toolchain containing `make`
 
-### Windows
+### On Windows
 
-#### 安装方法1
+#### Option 1
 
 Install all the required tools and configurations using Microsoft's [windows-build-tools](https://github.com/felixrieseberg/windows-build-tools) using `npm install --global --production windows-build-tools` from an elevated PowerShell or CMD.exe (run as Administrator).
 
-#### 安装方法2
+#### Option 2
 
 Install tools and configuration manually:
 

From fccf3c2d7dae56454fd89bd5a9c9b342f0110e35 Mon Sep 17 00:00:00 2001
From: haojun Liao 
Date: Wed, 4 Dec 2019 13:45:42 +0800
Subject: [PATCH 30/42] Update administrator-ch.md

---
 .../webdocs/markdowndocs/administrator-ch.md  | 107 ++++++++++++++++--
 1 file changed, 99 insertions(+), 8 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index b2e66b7568..03e3b85b95 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -80,19 +80,110 @@ option file:/home/cfg/taos.cfg not found, all options are set to system default
 ```
 更多taos的使用方法请见[Shell命令行程序](#_TDengine_Shell命令行程序)。本节主要讲解taos客户端应用在配置文件taos.cfg文件中使用到的参数。
 
-客户端配置参数如下
+客户端配置参数说明
 
-- masterIP：客户端默认发起请求的服务器的IP地址
-- charset：指明客户端所使用的字符集，默认值为UTF-8。TDengine为了存储中文、日文等非ASCII编码的宽字符，提供一种专门的字段类型`nchar`。如果数据写入`nchar`字段，将统一采用`UCS4-LE`格式对字段中数据进行编码存储。需要注意的是，**编码正确性**是客户端来保证。因此，如果用户想要正常使用`nchar`字段来存储诸如中文、日文等非ASCII字符，需要正确设置客户端的编码格式。客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为`UTF-8`，也有部分中文编码，例如`GB18030`或`GBK`等。在docker环境中默认的编码是`POSIX`。因此，客户端需要设置自己所使用的字符集，即所在系统的当前编码字符集才能正确地确保数据转换为`UCS4-LE`编码格式。
-如果配置文件中不设置`charset`，taos将自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则中断启动，要求用户在配置文件中指定charset，才能正常启动。
-- locale：设置系统语言环境。Linux上客户端与服务端共享
-- defaultUser：默认登录用户，默认值root
-- defaultPass：默认登录密码，默认值taosdata
+**masterIP**
+客户端默认发起请求的服务器的IP地址
+
+**locale**
+- 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
+- 是否必须设置：否
+
+TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型`nchar`。写入`nchar`字段的数据将统一采用`UCS4-LE`格式进行编码并发送到服务器。需要注意的是，**编码正确性**是客户端来保证。因此，如果用户想要正常使用`nchar`字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。
+
+客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为`UTF-8`，部分中文系统编码则可能是`GB18030`或`GBK`等。在docker环境中默认的编码是`POSIX`。在中文版Windows系统中，编码则是`CP936`。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证`nchar`中的数据正确转换为`UCS4-LE`编码格式。
+
+在 Linux 中 locale 的命名规则为：
+`<语言>_<地区>.<字符集编码>`
+如：`zh_CN.UTF-8`，zh代表中文，CN代表大陆地区，UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与Mac OSX系统可以通过设置locale来确定系统的字符编码，由于Windows使用的locale中不是POSIX标准的locale格式，因此在Windows下需要采用另一个配置参数`charset`来指定字符编码。在Linux系统中也可以使用charset来指定字符编码。
+
+**charset**
+- 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
+- 是否必须设置：否
+
+如果配置文件中不设置`charset`，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，**则中断启动过程**。
+
+在Linux系统中，locale信息包含了字符编码信息，因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如：
+```
+locale zh_CN.UTF-8
+```
+在Windows系统中，无法从locale获取系统当前编码。如果无法从配置文件中读取字符串编码信息，`taos`默认设置为字符编码为`CP936`。其等效在配置文件中添加如下配置：
+```
+charset CP936
+```
+如果需要调整字符编码，请查阅当前操作系统使用的编码，并在配置文件中正确设置。
+
+在Linux系统中，如果用户同时设置了locale和字符集编码charset，并且locale和charset的不一致，后设置的值将覆盖前面设置的值。
+```
+locale zh_CN.UTF-8
+charset GBK
+```
+则`charset`的有效值是`GBK`。
+```
+charset GBK
+locale zh_CN.UTF-8
+```
+`charset`的有效值是`UTF-8`。
+
+**sockettype**
+- 默认值：UDP
+- 是否必须设置：否
+
+客户端连接服务端的套接字的方式，可以使用`UDP`和`TCP`两种配置。
+在客户端和服务端之间的通讯需要经过恶劣的网络环境下（如公共网络、互联网）、客户端与数据库服务端连接不稳定（由于MTU的问题导致UDP丢包）的情况下，可以将连接的套接字类型调整为`TCP`
+
+>注意：客户端套接字的类型需要和服务端的套接字类型相同，否则无法连接数据库。
+
+**compressMsgSize**
+- 默认值：-1
+- 是否必须设置：否
+
+客户端与服务器之间进行消息通讯过程中，对通讯的消息进行压缩的阈值，默认值为-1（不压缩）。如果要压缩消息，建议设置为64330字节，即大于64330字节的消息体才进行压缩。在配置文件中增加如下配置项即可：
+```
+compressMsgSize 64330
+```
+如果配置项设置为0，`compressMsgSize 0`表示对所有的消息均进行压缩。
+
+**timezone**
+- 默认值：从系统中动态获取当前的时区设置
+- 是否必须设置：否
+
+客户端运行系统所在的时区。为应对多时区的数据写入和查询问题，TDengine采用Unix时间戳([Unix Timestamp](https://en.wikipedia.org/wiki/Unix_time))来记录和存储时间戳。Unix时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间（）转换为正确的Unix时间戳，需要设置正确的时区。
+
+在Linux系统中，客户端会自动读取系统设置的时区信息。用户也可以采用多种方式在配置文件设置时区。例如：
+```
+timezone UTC-8
+timezone GMT-8
+timezone Asia/Shanghai
+```
+均是合法的设置东八区时区的格式。
+
+
+时区的设置对于查询和写入SQL语句中非Unix时间戳的内容（时间戳字符串、关键词`now`的解析）产生影响。例如：
+```
+SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08';
+```
+在东八区转换的SQL语句等效于
+```
+SELECT count(*) FROM table_name WHERE TS<1554955268000;
+```
+在UTC时区，SQL语句等效于
+```
+SELECT count(*) FROM table_name WHERE TS<1554984068000;
+```
+为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，`2013-04-12T15:52:01.123+08:00`或者ISO-8601格式时间戳字符串`2013-04-12T15:52:01.123+0800`。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
+
+**defaultUser**
+默认登录用户，默认值root
+
+**defaultPass**
+默认登录密码，默认值taosdata
 
 TCP/UDP端口，以及日志的配置参数，与server的配置参数完全一样。
 
 启动taos时，你也可以从命令行指定IP地址、端口号，用户名和密码，否则就从taos.cfg读取。
 
+
 ## 用户管理
 
 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下：
@@ -206,4 +297,4 @@ KILL STREAM 
 
 TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
 
-这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
+这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项`monitor`将其关闭或打开。

From 3c283b3d2ba1966cb92b84fbad8f5f3f8c4c517b Mon Sep 17 00:00:00 2001
From: haojun Liao 
Date: Wed, 4 Dec 2019 14:36:01 +0800
Subject: [PATCH 31/42] Update administrator-ch.md

---
 .../webdocs/markdowndocs/administrator-ch.md  | 27 +++++++++++++++----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index 03e3b85b95..96e0562aac 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -163,7 +163,7 @@ timezone Asia/Shanghai
 ```
 SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08';
 ```
-在东八区转换的SQL语句等效于
+在东八区，SQL语句等效于
 ```
 SELECT count(*) FROM table_name WHERE TS<1554955268000;
 ```
@@ -174,15 +174,32 @@ SELECT count(*) FROM table_name WHERE TS<1554984068000;
 为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，`2013-04-12T15:52:01.123+08:00`或者ISO-8601格式时间戳字符串`2013-04-12T15:52:01.123+0800`。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。
 
 **defaultUser**
-默认登录用户，默认值root
+- 默认值：root
+- 是否必须设置：否
+
+登录用户名，客户端登录的时候，如果不指定用户名，则自动使用该用户名登录。默认情况下，以下的两个命令等效
+```
+taos
+taos -u root
+```
+用户名为从配置中读取的`defaultUser`配置项。如果更改`defaultUser abc`，则以下两个命令等效：
+```
+taos
+taos -u abc
+```
 
 **defaultPass**
-默认登录密码，默认值taosdata
+- 默认值：taosdata
+- 是否必须设置：否
+
+登录用户名，客户端登录的时候，如果不指定密码，则自动使用该密码登录。默认情况下，以下的两个命令等效
+```
+taos
+taos -ptaosdata
+```
 
 TCP/UDP端口，以及日志的配置参数，与server的配置参数完全一样。
 
-启动taos时，你也可以从命令行指定IP地址、端口号，用户名和密码，否则就从taos.cfg读取。
-
 
 ## 用户管理
 

From 24a8e3a65b79314ad6eddbddb6e9ab79181235bb Mon Sep 17 00:00:00 2001
From: haojun Liao 
Date: Wed, 4 Dec 2019 15:56:59 +0800
Subject: [PATCH 32/42] Update administrator-ch.md

---
 .../webdocs/markdowndocs/administrator-ch.md  | 122 +++++++++++++-----
 1 file changed, 88 insertions(+), 34 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md
index 96e0562aac..6eb066b4c6 100644
--- a/documentation/webdocs/markdowndocs/administrator-ch.md
+++ b/documentation/webdocs/markdowndocs/administrator-ch.md
@@ -2,15 +2,15 @@
 
 ## 文件目录结构
 
-安装TDengine后，默认会在操作系统中生成下列目录或文件：
+安装TDengine的过程中，安装程序将在操作系统中创建以下目录或文件：
 
 | 目录/文件               | 说明                                              |
 | ---------------------- | :------------------------------------------------|
-| /etc/taos/taos.cfg     | TDengine默认[配置文件]                            |
-| /usr/local/taos/driver | TDengine动态链接库目录                            |
-| /var/lib/taos          | TDengine默认数据文件目录,可通过[配置文件]修改位置.    |
-| /var/log/taos          | TDengine默认日志文件目录,可通过[配置文件]修改位置     |
-| /usr/local/taos/bin    | TDengine可执行文件目录                            |
+| /etc/taos/taos.cfg     | 默认[配置文件]                            |
+| /usr/local/taos/driver | 动态链接库目录                            |
+| /var/lib/taos          | 默认数据文件目录,可通过[配置文件]修改位置.    |
+| /var/log/taos          | 默认日志文件目录,可通过[配置文件]修改位置     |
+| /usr/local/taos/bin    | 可执行文件目录                            |
 
 ### 可执行文件
 
@@ -25,22 +25,78 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下
 
 ## 服务端配置
 
-TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos目录，可以通过taosd命令行执行参数-c指定配置文件目录。比如taosd -c /home/user来指定配置文件位于/home/user这个目录。
+TDengine系统后台服务程序是`taosd`，其启动时候读取的配置文件缺省目录是`/etc/taos`。可以通过命令行执行参数-c指定配置文件目录，比如
+```
+taosd -c /home/user
+```
+指定`taosd`启动的时候读取`/home/user`目录下的配置文件taos.cfg。
 
 下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节。**注意：配置修改后，需要重启*taosd*服务才能生效。**
 
-- internalIp: 对外提供服务的IP地址，默认取第一个IP地址
-- mgmtShellPort：管理节点与客户端通信使用的TCP/UDP端口号（默认值是6030）。此端口号在内向后连续的5个端口都会被UDP通信占用，即UDP占用[6030-6034]，同时TCP通信也会使用端口[6030]。
-- vnodeShellPort：数据节点与客户端通信使用的TCP/UDP端口号（默认值是6035）。此端口号在内向后连续的5个端口都会被UDP通信占用，即UDP占用[6035-6039]，同时TCP通信也会使用端口[6035]
-- httpPort：数据节点对外提供RESTful服务使用TCP，端口号[6020]
-- dataDir: 数据文件目录，缺省是/var/lib/taos
-- maxUsers：用户的最大数量
-- maxDbs：数据库的最大数量
-- maxTables：数据表的最大数量
-- enableMonitor: 系统监测标志位，0：关闭，1：打开
-- logDir: 日志文件目录，缺省是/var/log/taos
-- numOfLogLines：日志文件的最大行数
-- debugFlag: 系统debug日志开关，131：仅错误和报警信息，135：所有
+**internalIp**
+- 默认值：操作配置的IP地址列表中的第一个IP地址
+
+对外提供服务的IP地址。
+
+**mgmtShellPort**
+- 默认值： _6030_
+
+数据库服务中管理节点与客户端通信使用的TCP/UDP端口号。
+> 此端口号在内向后连续的5个端口都会用于UDP通信，即使用的端口是 _6030_ - _6034_ 。此外，TCP还会使用端口 _6030_ 
+
+**vnodeShellPort**
+- 默认值： _6035_
+
+数据节点与客户端通信使用的TCP/UDP端口号。
+> 此端口号在内向后连续的5个端口都会用于UDP通信，即使用的端口是 _6035_ - _6039_ 。此外，TCP还会使用端口 _6035_ 
+
+**httpPort**
+- 默认值： _6020_
+
+RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求。
+
+**dataDir**
+- 默认值：/var/lib/taos
+
+数据文件目录，所有的数据文件都将写入该目录。
+
+**logDir**
+- 默认值：/var/log/taos
+
+日志文件目录，客户端和服务器的运行日志将写入该目录。
+
+**maxUsers**
+- 默认值：10,000
+
+系统允许创建用户数量的上限
+
+**maxDbs**
+- 默认值：1,000
+
+系统允许的创建数据库的上限
+
+**maxTables**
+- 默认值：650,000
+
+系统允许创建数据表的上限。
+>系统能够创建的表受到多种因素的限制，单纯地增大该参数并不能直接增加系统能够创建的表数量。例如，由于每个表创建均需要消耗一定量的缓存空间，系统可用内存一定的情况下，创建表的总数的上限是一个固定的值。
+
+**monitor**
+- 默认值：1（激活状态）
+
+服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况，包括CPU、内存、硬盘、网络带宽、HTTP请求量的监控记录，记录信息存储在`LOG`库中。0表示关闭监控服务，1表示激活监控服务。
+
+**numOfLogLines**
+- 默认值：10,000,000
+
+单个日志文件允许的最大行数（10,000,000行）。
+
+**debugFlag**
+- 默认值：131（仅输出错误和警告信息）
+
+系统（服务端和客户端）运行日志开关：
+- 131 仅输出错误和警告信息
+- 135 输入错误（ERROR）、警告（WARN）、信息（Info）
 
 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数：
 
@@ -66,15 +122,14 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
 
 ## 客户端配置 
 
-TDengine系统的前台交互客户端应用程序为taos（Windows平台上为taos.exe），可以使用taos.cfg来配置启动和运行配置项。可以使用 `taos -?` 来获取全部的可选项。
-启动的时候如果不指定taos加载配置文件路径，默认读取`/etc/taos/`路径下的`taos.cfg`文件来设置启动选项，如果在默认路径下找不到配置文件，则使用默认配置的设置来启动程序，并且会在启动的时候打印一行告警信息。指定配置文件来启动`taos`的命令如下：
+TDengine系统的前台交互客户端应用程序为taos（Windows平台上为taos.exe）。与服务端程序一样，也可以通过设置taos.cfg来配置`taos`启动和运行的配置项。启动的时候如果不指定taos加载配置文件路径，默认读取`/etc/taos/`路径下的`taos.cfg`文件。指定配置文件来启动`taos`的命令如下：
 ```
 taos -c /home/cfg/
 ```
 **注意：启动设置的是配置文件所在目录，而不是配置文件本身**
 
 如果`/home/cfg/`目录下没有配置文件，程序会继续启动并打印如下告警信息：
-```
+```markdown
 Welcome to the TDengine shell from linux, client version:1.6.4.0
 option file:/home/cfg/taos.cfg not found, all options are set to system default
 ```
@@ -83,11 +138,17 @@ option file:/home/cfg/taos.cfg not found, all options are set to system default
 客户端配置参数说明
 
 **masterIP**
-客户端默认发起请求的服务器的IP地址
+- 默认值：127.0.0.1
+
+客户端连接的TDengine服务器IP地址，如果不设置默认连接127.0.0.1的节点。以下两个命令等效：
+```
+taos
+taos -h 127.0.0.1
+```
+其中的IP地址是从配置文件中读取的masterIP的值。
 
 **locale**
 - 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
-- 是否必须设置：否
 
 TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型`nchar`。写入`nchar`字段的数据将统一采用`UCS4-LE`格式进行编码并发送到服务器。需要注意的是，**编码正确性**是客户端来保证。因此，如果用户想要正常使用`nchar`字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。
 
@@ -99,7 +160,6 @@ TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供
 
 **charset**
 - 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置
-- 是否必须设置：否
 
 如果配置文件中不设置`charset`，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，**则中断启动过程**。
 
@@ -127,7 +187,6 @@ locale zh_CN.UTF-8
 
 **sockettype**
 - 默认值：UDP
-- 是否必须设置：否
 
 客户端连接服务端的套接字的方式，可以使用`UDP`和`TCP`两种配置。
 在客户端和服务端之间的通讯需要经过恶劣的网络环境下（如公共网络、互联网）、客户端与数据库服务端连接不稳定（由于MTU的问题导致UDP丢包）的情况下，可以将连接的套接字类型调整为`TCP`
@@ -135,8 +194,7 @@ locale zh_CN.UTF-8
 >注意：客户端套接字的类型需要和服务端的套接字类型相同，否则无法连接数据库。
 
 **compressMsgSize**
-- 默认值：-1
-- 是否必须设置：否
+- 默认值：-1（不压缩）
 
 客户端与服务器之间进行消息通讯过程中，对通讯的消息进行压缩的阈值，默认值为-1（不压缩）。如果要压缩消息，建议设置为64330字节，即大于64330字节的消息体才进行压缩。在配置文件中增加如下配置项即可：
 ```
@@ -146,7 +204,6 @@ compressMsgSize 64330
 
 **timezone**
 - 默认值：从系统中动态获取当前的时区设置
-- 是否必须设置：否
 
 客户端运行系统所在的时区。为应对多时区的数据写入和查询问题，TDengine采用Unix时间戳([Unix Timestamp](https://en.wikipedia.org/wiki/Unix_time))来记录和存储时间戳。Unix时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间（）转换为正确的Unix时间戳，需要设置正确的时区。
 
@@ -175,7 +232,6 @@ SELECT count(*) FROM table_name WHERE TS<1554984068000;
 
 **defaultUser**
 - 默认值：root
-- 是否必须设置：否
 
 登录用户名，客户端登录的时候，如果不指定用户名，则自动使用该用户名登录。默认情况下，以下的两个命令等效
 ```
@@ -190,7 +246,6 @@ taos -u abc
 
 **defaultPass**
 - 默认值：taosdata
-- 是否必须设置：否
 
 登录用户名，客户端登录的时候，如果不指定密码，则自动使用该密码登录。默认情况下，以下的两个命令等效
 ```
@@ -198,8 +253,7 @@ taos
 taos -ptaosdata
 ```
 
-TCP/UDP端口，以及日志的配置参数，与server的配置参数完全一样。
-
+TCP/UDP端口，以及日志的配置参数，与server的配置参数完全一样。使用命令`taos -?` 可查看`taos`允许的可选项。
 
 ## 用户管理
 
@@ -312,6 +366,6 @@ KILL STREAM 
 
 ## 系统监控
 
-TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在SYS库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
+TDengine启动后，会自动创建一个监测数据库`LOG`，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在`LOG`库里。系统管理员可以通过客户端程序查看记录库中的运行负载信息，（在企业版中）还可以通过浏览器查看数据的图标可视化结果。
 
 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项`monitor`将其关闭或打开。

From 641c1e44d00b30b2dc0c35fbe3acf1546a6465c0 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Wed, 4 Dec 2019 16:07:25 +0800
Subject: [PATCH 33/42] [TBASE-816]

---
 packaging/tools/makeclient.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh
index b4948bb3a7..f8d21e202b 100755
--- a/packaging/tools/makeclient.sh
+++ b/packaging/tools/makeclient.sh
@@ -17,7 +17,6 @@ top_dir="$(readlink -m ${script_dir}/../..)"
 build_dir="${compile_dir}/build"
 code_dir="${top_dir}/src"
 release_dir="${top_dir}/release"
-community_dir="${script_dir}/../../../community/src"
 
 #package_name='linux'
 install_dir="${release_dir}/TDengine-client-${version}"
@@ -25,7 +24,7 @@ install_dir="${release_dir}/TDengine-client-${version}"
 # Directories and files.
 bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${script_dir}/remove_client.sh"
 lib_files="${build_dir}/lib/libtaos.so.${version}"
-header_files="${community_dir}/inc/taos.h ${community_dir}/inc/taoserror.h"
+header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h"
 cfg_dir="${top_dir}/packaging/cfg"
 install_files="${script_dir}/install_client.sh"
 
@@ -55,7 +54,7 @@ mkdir -p ${install_dir}/driver
 cp ${lib_files} ${install_dir}/driver
 
 # Copy connector
-connector_dir="${community_dir}/connector"
+connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
 cp ${build_dir}/lib/*.jar      ${install_dir}/connector
 cp -r ${connector_dir}/grafana ${install_dir}/connector/

From 886b9aac4b639da37840b5738949b332f4f41e56 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Wed, 4 Dec 2019 16:10:18 +0800
Subject: [PATCH 34/42] [TBASE-816]

---
 src/os/linux/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/os/linux/CMakeLists.txt b/src/os/linux/CMakeLists.txt
index 0f725947cc..a702cac759 100644
--- a/src/os/linux/CMakeLists.txt
+++ b/src/os/linux/CMakeLists.txt
@@ -6,5 +6,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(os ${SRC})
-  TARGET_LINK_LIBRARIES(os m rt atomic)
+  TARGET_LINK_LIBRARIES(os m rt)
 ENDIF ()

From a0b79bd771f35399dd3a622ba1de4a957f093acb Mon Sep 17 00:00:00 2001
From: fang 
Date: Thu, 5 Dec 2019 10:06:56 +0800
Subject: [PATCH 35/42] python connector user guide

---
 documentation/webdocs/markdowndocs/Connector.md    | 2 +-
 documentation/webdocs/markdowndocs/connector-ch.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/documentation/webdocs/markdowndocs/Connector.md b/documentation/webdocs/markdowndocs/Connector.md
index 2ecb14c710..014e88de42 100644
--- a/documentation/webdocs/markdowndocs/Connector.md
+++ b/documentation/webdocs/markdowndocs/Connector.md
@@ -345,7 +345,7 @@ c1.execute('select * from tb')
 data = c1.fetchall()
 # data is a list of returned rows with each row being a tuple
 numOfRows = c1.rowcount
-numOfCols = c1.descriptions
+numOfCols = len(c1.description)
 for irow in range(numOfRows):
   print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
   
diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md
index 1cc03703be..5313c07ace 100644
--- a/documentation/webdocs/markdowndocs/connector-ch.md
+++ b/documentation/webdocs/markdowndocs/connector-ch.md
@@ -341,7 +341,7 @@ c1.execute('select * from tb')
 data = c1.fetchall()
 # 返回的结果是一个列表，每一行构成列表的一个元素
 numOfRows = c1.rowcount
-numOfCols = c1.descriptions
+numOfCols = len(c1.description)
 for irow in range(numOfRows):
   print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
   

From 60629d31d62fb2915c93c9b8c11d4e2851141209 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Thu, 5 Dec 2019 11:05:14 +0800
Subject: [PATCH 36/42] [TBASE-1288]

---
 src/util/src/tcrc32c.c    |  4 ++-
 src/util/src/textbuffer.c | 52 ++++++++++++++++++++++-----------------
 src/util/src/thistogram.c |  4 +--
 src/util/src/tskiplist.c  |  6 ++---
 src/util/src/tstrbuild.c  |  4 +--
 src/util/src/ttypes.c     | 13 +++++-----
 6 files changed, 46 insertions(+), 37 deletions(-)

diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c
index 705ca5872a..c67773f1ca 100644
--- a/src/util/src/tcrc32c.c
+++ b/src/util/src/tcrc32c.c
@@ -26,7 +26,7 @@
 
 #include "tcrc32c.h"
 //todo : use the original source code
-#pragma GCC diagnostic ignored "-Wunused-function"
+//#pragma GCC diagnostic ignored "-Wunused-function"
 
 #define POLY 0x82f63b78
 #define LONG_SHIFT 8192
@@ -1093,6 +1093,7 @@ static uint32_t short_shifts[4][256] = {
      0xe1a734e7, 0xc41cc13c, 0x140cd014, 0x31b725cf, 0x5f7b3ba2, 0x7ac0ce79,
      0x82e30778, 0xa758f2a3, 0xc994ecce, 0xec2f1915}};
 
+#if 0
 static uint32_t append_trivial(uint32_t crc, crc_stream input, size_t length) {
   for (size_t i = 0; i < length; ++i) {
     crc = crc ^ input[i];
@@ -1130,6 +1131,7 @@ static uint32_t append_adler_table(uint32_t crci, crc_stream input,
   }
   return (uint32_t)(crc ^ 0xffffffff);
 }
+#endif
 
 /* Table-driven software version as a fall-back.  This is about 15 times slower
    than using the hardware instructions.  This assumes little-endian integers,
diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c
index ef55368081..42c2cc1ed0 100644
--- a/src/util/src/textbuffer.c
+++ b/src/util/src/textbuffer.c
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see .
  */
-
+#include 
 #include "os.h"
 #include "taos.h"
 #include "taosmsg.h"
@@ -23,7 +23,7 @@
 #include "ttypes.h"
 #include "tutil.h"
 
-#pragma GCC diagnostic ignored "-Wformat"
+//#pragma GCC diagnostic ignored "-Wformat"
 
 #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \
   (data + (schema)->colOffset[colId] * (allrow) + (rowId) * (schema)->pFields[colId].bytes)
@@ -1017,7 +1017,7 @@ static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx,
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:
     case TSDB_DATA_TYPE_BIGINT:
-      printf("%s:(%lld, %lld, %lld)\n", prefix, *(int64_t *)startx, *(int64_t *)midx, *(int64_t *)endx);
+      printf("%s:(%" PRId64 ", %" PRId64 ", %" PRId64 ")\n", prefix, *(int64_t *)startx, *(int64_t *)midx, *(int64_t *)endx);
       break;
     case TSDB_DATA_TYPE_FLOAT:
       printf("%s:(%f, %f, %f)\n", prefix, *(float *)startx, *(float *)midx, *(float *)endx);
@@ -1093,7 +1093,7 @@ static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t
         break;
       case TSDB_DATA_TYPE_TIMESTAMP:
       case TSDB_DATA_TYPE_BIGINT:
-        printf("%lld\t", *(int64_t *)startx);
+        printf("%" PRId64 "\t", *(int64_t *)startx);
         break;
       case TSDB_DATA_TYPE_BINARY:
         printf("%s\t", startx);
@@ -1264,7 +1264,7 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx,
         assert(pPage->numOfElems > 0);
 
         tColModelAppend(pDesc->pSchema, buffer, pPage->data, 0, pPage->numOfElems, pPage->numOfElems);
-        printf("id: %d  count: %d\n", j, buffer->numOfElems);
+        printf("id: %d  count: %" PRIu64 "\n", j, buffer->numOfElems);
       }
     }
     tfree(pPage);
@@ -1376,10 +1376,16 @@ static void printBinaryData(char *data, int32_t len) {
   }
 
   if (len == 50) {  // probably the avg intermediate result
-    printf("%lf,%d\t", *(double *)data, *(int64_t *)(data + sizeof(double)));
+    printf("%lf,%" PRId64 "\t", *(double *)data, *(int64_t *)(data + sizeof(double)));
   } else if (data[8] == ',') {  // in TSDB_FUNC_FIRST_DST/TSDB_FUNC_LAST_DST,
                                 // the value is seperated by ','
-    printf("%ld,%0x\t", *(int64_t *)data, data + sizeof(int64_t) + 1);
+    //printf("%" PRId64 ",%0x\t", *(int64_t *)data, data + sizeof(int64_t) + 1);
+    printf("%" PRId64 ", HEX: ", *(int64_t *)data);
+    int32_t tmp_len = len - sizeof(int64_t) - 1;
+    for (int32_t i = 0; i < tmp_len; ++i) {
+      printf("%0x ", *(data + sizeof(int64_t) + 1 + i));
+    }
+    printf("\t");
   } else if (isCharString) {
     printf("%s\t", data);
   }
@@ -1389,26 +1395,26 @@ static void printBinaryData(char *data, int32_t len) {
 static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) {
   if (param->functionId == TSDB_FUNC_LAST_DST) {
     switch (param->type) {
-      case TSDB_DATA_TYPE_TINYINT:printf("%lld,%d\t", *(int64_t *) data, *(int8_t *) (data + TSDB_KEYSIZE + 1));
+      case TSDB_DATA_TYPE_TINYINT:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int8_t *) (data + TSDB_KEYSIZE + 1));
         break;
-      case TSDB_DATA_TYPE_SMALLINT:printf("%lld,%d\t", *(int64_t *) data, *(int16_t *) (data + TSDB_KEYSIZE + 1));
+      case TSDB_DATA_TYPE_SMALLINT:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int16_t *) (data + TSDB_KEYSIZE + 1));
         break;
       case TSDB_DATA_TYPE_TIMESTAMP:
-      case TSDB_DATA_TYPE_BIGINT:printf("%lld,%lld\t", *(int64_t *) data, *(int64_t *) (data + TSDB_KEYSIZE + 1));
+      case TSDB_DATA_TYPE_BIGINT:printf("%" PRId64 ",%" PRId64 "\t", *(int64_t *) data, *(int64_t *) (data + TSDB_KEYSIZE + 1));
         break;
-      case TSDB_DATA_TYPE_FLOAT:printf("%lld,%d\t", *(int64_t *) data, *(float *) (data + TSDB_KEYSIZE + 1));
+      case TSDB_DATA_TYPE_FLOAT:printf("%" PRId64 ",%f\t", *(int64_t *) data, *(float *) (data + TSDB_KEYSIZE + 1));
         break;
-      case TSDB_DATA_TYPE_DOUBLE:printf("%lld,%d\t", *(int64_t *) data, *(double *) (data + TSDB_KEYSIZE + 1));
+      case TSDB_DATA_TYPE_DOUBLE:printf("%" PRId64 ",%f\t", *(int64_t *) data, *(double *) (data + TSDB_KEYSIZE + 1));
         break;
-      case TSDB_DATA_TYPE_BINARY:printf("%lld,%s\t", *(int64_t *) data, (data + TSDB_KEYSIZE + 1));
+      case TSDB_DATA_TYPE_BINARY:printf("%" PRId64 ",%s\t", *(int64_t *) data, (data + TSDB_KEYSIZE + 1));
         break;
 
       case TSDB_DATA_TYPE_INT:
-      default:printf("%lld,%d\t", *(int64_t *) data, *(int32_t *) (data + TSDB_KEYSIZE + 1));
+      default:printf("%" PRId64 ",%d\t", *(int64_t *) data, *(int32_t *) (data + TSDB_KEYSIZE + 1));
         break;
     }
   } else if (param->functionId == TSDB_FUNC_AVG) {
-      printf("%f,%lld\t", *(double *) data, *(int64_t *) (data + sizeof(double) + 1));
+      printf("%f,%" PRId64 "\t", *(double *) data, *(int64_t *) (data + sizeof(double) + 1));
   } else {
     // functionId == TSDB_FUNC_MAX_DST | TSDB_FUNC_TAG
     switch (param->type) {
@@ -1420,13 +1426,13 @@ static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) {
         break;
       case TSDB_DATA_TYPE_TIMESTAMP:
       case TSDB_DATA_TYPE_BIGINT:
-        printf("%lld\t", *(int64_t *)data);
+        printf("%" PRId64 "\t", *(int64_t *)data);
         break;
       case TSDB_DATA_TYPE_FLOAT:
-        printf("%d\t", *(float *)data);
+        printf("%f\t", *(float *)data);
         break;
       case TSDB_DATA_TYPE_DOUBLE:
-        printf("%d\t", *(double *)data);
+        printf("%f\t", *(double *)data);
         break;
       case TSDB_DATA_TYPE_BINARY:
         printf("%s\t", data);
@@ -1434,7 +1440,7 @@ static void printBinaryDataEx(char *data, int32_t len, SSrcColumnInfo *param) {
 
       case TSDB_DATA_TYPE_INT:
       default:
-        printf("%d\t", *(double *)data);
+        printf("%f\t", *(double *)data);
         break;
     }
   }
@@ -1450,7 +1456,7 @@ void tColModelDisplay(tColModel *pModel, void *pData, int32_t numOfRows, int32_t
 
       switch (type) {
         case TSDB_DATA_TYPE_BIGINT:
-          printf("%lld\t", *(int64_t *)val);
+          printf("%" PRId64 "\t", *(int64_t *)val);
           break;
         case TSDB_DATA_TYPE_INT:
           printf("%d\t", *(int32_t *)val);
@@ -1468,7 +1474,7 @@ void tColModelDisplay(tColModel *pModel, void *pData, int32_t numOfRows, int32_t
           printf("%lf\t", *(double *)val);
           break;
         case TSDB_DATA_TYPE_TIMESTAMP:
-          printf("%lld\t", *(int64_t *)val);
+          printf("%" PRId64 "\t", *(int64_t *)val);
           break;
         case TSDB_DATA_TYPE_TINYINT:
           printf("%d\t", *(int8_t *)val);
@@ -1501,7 +1507,7 @@ void tColModelDisplayEx(tColModel *pModel, void *pData, int32_t numOfRows, int32
 
       switch (pModel->pFields[j].type) {
         case TSDB_DATA_TYPE_BIGINT:
-          printf("%lld\t", *(int64_t *)val);
+          printf("%" PRId64 "\t", *(int64_t *)val);
           break;
         case TSDB_DATA_TYPE_INT:
           printf("%d\t", *(int32_t *)val);
@@ -1519,7 +1525,7 @@ void tColModelDisplayEx(tColModel *pModel, void *pData, int32_t numOfRows, int32
           printf("%lf\t", *(double *)val);
           break;
         case TSDB_DATA_TYPE_TIMESTAMP:
-          printf("%lld\t", *(int64_t *)val);
+          printf("%" PRId64 "\t", *(int64_t *)val);
           break;
         case TSDB_DATA_TYPE_TINYINT:
           printf("%d\t", *(int8_t *)val);
diff --git a/src/util/src/thistogram.c b/src/util/src/thistogram.c
index 5fef9077ea..03c2294938 100644
--- a/src/util/src/thistogram.c
+++ b/src/util/src/thistogram.c
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see .
  */
-
+#include 
 #include "os.h"
 
 #include "taosmsg.h"
@@ -447,7 +447,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) {
   printf("total entries: %d, elements: %d\n", pHisto->numOfEntries, pHisto->numOfElems);
 #if defined(USE_ARRAYLIST)
   for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
-    printf("%d: (%f, %lld)\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
+    printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
   }
 #else
   tSkipListNode* pNode = pHisto->pList->pHead.pForward[0];
diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c
index 72469fa75b..0440e5d5ab 100644
--- a/src/util/src/tskiplist.c
+++ b/src/util/src/tskiplist.c
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see .
  */
-
+#include 
 #include 
 #include 
 #include 
@@ -570,7 +570,7 @@ int32_t tSkipListIterateList(tSkipList *pSkipList, tSkipListNode ***pRes, bool (
     char* tmp = realloc((*pRes), num * POINTER_BYTES);
     assert(tmp != NULL);
 
-    *pRes = tmp;
+    *pRes = (tSkipListNode**)tmp;
   }
 
   return num;
@@ -688,7 +688,7 @@ void tSkipListPrint(tSkipList *pSkipList, int16_t nlevel) {
       case TSDB_DATA_TYPE_SMALLINT:
       case TSDB_DATA_TYPE_TINYINT:
       case TSDB_DATA_TYPE_BIGINT:
-        fprintf(stdout, "%d: %lld \n", id++, p->key.i64Key);
+        fprintf(stdout, "%d: %" PRId64 " \n", id++, p->key.i64Key);
         break;
       case TSDB_DATA_TYPE_BINARY:
         fprintf(stdout, "%d: %s \n", id++, p->key.pz);
diff --git a/src/util/src/tstrbuild.c b/src/util/src/tstrbuild.c
index 6914357493..439370ce07 100644
--- a/src/util/src/tstrbuild.c
+++ b/src/util/src/tstrbuild.c
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see .
  */
-
+#include 
 #include "os.h"
 #include "tstrbuild.h"
 
@@ -70,7 +70,7 @@ void taosStringBuilderAppendNull(SStringBuilder* sb) { taosStringBuilderAppendSt
 
 void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) {
   char   buf[64];
-  size_t len = sprintf(buf, "%lld", v);
+  size_t len = sprintf(buf, "%" PRId64, v);
   taosStringBuilderAppendStringLen(sb, buf, len);
 }
 
diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c
index 98f0741905..3f0c1732e3 100644
--- a/src/util/src/ttypes.c
+++ b/src/util/src/ttypes.c
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see .
  */
-
+#include 
 #include "os.h"
 #include "taos.h"
 #include "tsdb.h"
@@ -213,7 +213,7 @@ int32_t tVariantToString(tVariant *pVar, char *dst) {
       return sprintf(dst, "%d", (int32_t)pVar->i64Key);
 
     case TSDB_DATA_TYPE_BIGINT:
-      return sprintf(dst, "%lld", pVar->i64Key);
+      return sprintf(dst, "%" PRId64, pVar->i64Key);
 
     case TSDB_DATA_TYPE_FLOAT:
     case TSDB_DATA_TYPE_DOUBLE:
@@ -224,6 +224,7 @@ int32_t tVariantToString(tVariant *pVar, char *dst) {
   }
 }
 
+#if 0
 static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, bool releaseVariantPtr) {
   if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
     setNull(pDest, type, tDataTypeDesc[type].nSize);
@@ -337,7 +338,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type,
 
   return 0;
 }
-
+#endif
 static FORCE_INLINE int32_t convertToBoolImpl(char *pStr, int32_t len) {
   if ((strncasecmp(pStr, "true", len) == 0) && (len == 4)) {
     return TSDB_TRUE;
@@ -386,7 +387,7 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
 
   } else {
     if (pVariant->nType >= TSDB_DATA_TYPE_TINYINT && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) {
-      sprintf(pBuf == NULL ? *pDest : pBuf, "%lld", pVariant->i64Key);
+      sprintf(pBuf == NULL ? *pDest : pBuf, "%" PRId64, pVariant->i64Key);
     } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) {
       sprintf(pBuf == NULL ? *pDest : pBuf, "%lf", pVariant->dKey);
     } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL) {
@@ -411,7 +412,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   int32_t nLen = 0;
 
   if (pVariant->nType >= TSDB_DATA_TYPE_TINYINT && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) {
-    nLen = sprintf(pDst, "%lld", pVariant->i64Key);
+    nLen = sprintf(pDst, "%" PRId64, pVariant->i64Key);
   } else if (pVariant->nType == TSDB_DATA_TYPE_DOUBLE || pVariant->nType == TSDB_DATA_TYPE_FLOAT) {
     nLen = sprintf(pDst, "%lf", pVariant->dKey);
   } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
@@ -437,7 +438,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
     char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE);
     assert(tmp != NULL);
 
-    pVariant->wpz = tmp;
+    pVariant->wpz = (wchar_t *)tmp;
   } else {
     taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE);
   }

From cff3009a4174c00b135ead08705d2bb484ddfd4c Mon Sep 17 00:00:00 2001
From: Hongze Cheng 
Date: Thu, 5 Dec 2019 15:40:09 +0800
Subject: [PATCH 37/42] fix #856

---
 src/system/detail/src/vnodeShell.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c
index f4b3cdbbe7..66bede89b7 100644
--- a/src/system/detail/src/vnodeShell.c
+++ b/src/system/detail/src/vnodeShell.c
@@ -529,9 +529,11 @@ static int vnodeDoSubmitJob(SVnodeObj *pVnode, int import, int32_t *ssid, int32_
   int code = TSDB_CODE_SUCCESS;
   int32_t numOfPoints = 0;
   int32_t i = 0;
+  SShellSubmitBlock tBlock;
 
   for (i = *ssid; i < esid; i++) {
     numOfPoints = 0;
+    tBlock = *pBlocks;
 
     code = vnodeCheckSubmitBlockContext(pBlocks, pVnode);
     if (code != TSDB_CODE_SUCCESS) break;
@@ -565,6 +567,13 @@ static int vnodeDoSubmitJob(SVnodeObj *pVnode, int import, int32_t *ssid, int32_
 
   *ssid = i;
   *ppBlocks = pBlocks;
+  /* Since the pBlock part can be changed by the vnodeForwardToPeer interface,
+   * which is also possible to be used again. For that case, we just copy the original
+   * block content back.
+   */
+  if (import && (code == TSDB_CODE_ACTION_IN_PROGRESS)) {
+    memcpy((void *)pBlocks, (void *)&tBlock, sizeof(SShellSubmitBlock));
+  }
 
   return code;
 }

From 5a6d3762655be9dc9ffff8cf046b4a3b6a57f111 Mon Sep 17 00:00:00 2001
From: lihui 
Date: Thu, 5 Dec 2019 17:28:15 +0800
Subject: [PATCH 38/42] [1292]

---
 src/client/src/tscParseInsert.c | 65 ++++++++++++++++++++++++++-------
 src/kit/shell/src/shellEngine.c |  2 +
 src/util/src/ttokenizer.c       |  8 +++-
 3 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index 58cfcda17e..8873169e1e 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -392,7 +392,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start
 }
 
 int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error,
-                      int16_t timePrec) {
+                      int16_t timePrec, int32_t *code, char* tmpTokenBuf) {
   int32_t   index = 0;
   bool      isPrevOptr;
   SSQLToken sToken = {0};
@@ -418,6 +418,7 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
       }
       
       strcpy(error, "client out of memory");
+      *code = TSDB_CODE_CLI_OUT_OF_MEMORY;
       return -1;
     }
 
@@ -425,23 +426,42 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
          (sToken.type != TK_FLOAT) && (sToken.type != TK_BOOL) && (sToken.type != TK_NULL)) ||
         (sToken.n == 0) || (sToken.type == TK_RP)) {
       tscInvalidSQLErrMsg(error, "invalid data or symbol", sToken.z);
+      *code = TSDB_CODE_INVALID_SQL;
       return -1;
     }
 
     // Remove quotation marks
     if (TK_STRING == sToken.type) {
-      sToken.z++;
-      sToken.n -= 2;
+      // delete escape character: \\, \', \"
+      char delim = sToken.z[0];
+      int32_t cnt = 0;
+      int32_t j = 0;
+      for (int32_t i = 1; i < sToken.n - 1; ++i) {
+        if (sToken.z[i] == delim || sToken.z[i] == '\\') {
+          if (sToken.z[i + 1] == delim) {
+            cnt++;
+            continue;
+          }
+        }
+      
+        tmpTokenBuf[j] = sToken.z[i];
+        j++;
+      }
+      
+      sToken.z = tmpTokenBuf;
+      sToken.n -= 2 + cnt;    
     }
 
     bool    isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX);
     int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, error, str, isPrimaryKey, timePrec);
     if (ret != TSDB_CODE_SUCCESS) {
+      *code = TSDB_CODE_INVALID_SQL;
       return -1;  // NOTE: here 0 mean error!
     }
 
     if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, start) != TSDB_CODE_SUCCESS) {
       tscInvalidSQLErrMsg(error, "client time/server time can not be mixed up", sToken.z);
+      *code = TSDB_CODE_INVALID_TIME_STAMP;
       return -1;
     }
   }
@@ -476,7 +496,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) {
 }
 
 int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMeta, int maxRows,
-                  SParsedDataColInfo *spd, char *error) {
+                  SParsedDataColInfo *spd, char *error, int32_t *code, char* tmpTokenBuf) {
   int32_t   index = 0;
   SSQLToken sToken;
 
@@ -487,6 +507,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe
 
   if (spd->hasVal[0] == false) {
     strcpy(error, "primary timestamp column can not be null");
+    *code = TSDB_CODE_INVALID_SQL;
     return -1;
   }
 
@@ -500,12 +521,13 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe
       int32_t tSize = tscAllocateMemIfNeed(pDataBlock, pMeterMeta->rowSize);
       if (0 == tSize) {
         strcpy(error, "client out of memory");
+        *code = TSDB_CODE_CLI_OUT_OF_MEMORY;
         return -1;
       }
       maxRows += tSize;
     }
 
-    int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, error, precision);
+    int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, error, precision, code, tmpTokenBuf);
     if (len <= 0) { // error message has been set in tsParseOneRowData
       return -1;
     }
@@ -517,6 +539,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe
     *str += index;
     if (sToken.n == 0 || sToken.type != TK_RP) {
       tscInvalidSQLErrMsg(error, ") expected", *str);
+      *code = TSDB_CODE_INVALID_SQL;
       return -1;
     }
 
@@ -525,6 +548,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe
 
   if (numOfRows <= 0) {
     strcpy(error, "no any data points");
+    *code = TSDB_CODE_INVALID_SQL;
     return -1;
   } else {
     return numOfRows;
@@ -636,10 +660,17 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
   if (0 == maxNumOfRows) {
     return TSDB_CODE_CLI_OUT_OF_MEMORY;
   }
-
-  int32_t numOfRows = tsParseValues(str, dataBuf, pMeterMeta, maxNumOfRows, spd, pCmd->payload);
+  
+  int32_t code = TSDB_CODE_INVALID_SQL;
+  char*   tmpTokenBuf = calloc(1, 4096);  // used for deleting Escape character: \\, \', \"
+  if (NULL == tmpTokenBuf) {
+    return TSDB_CODE_CLI_OUT_OF_MEMORY;
+  }
+  
+  int32_t numOfRows = tsParseValues(str, dataBuf, pMeterMeta, maxNumOfRows, spd, pCmd->payload, &code, tmpTokenBuf);
+  free(tmpTokenBuf);
   if (numOfRows <= 0) {
-    return TSDB_CODE_INVALID_SQL;
+    return code;
   }
 
   for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) {
@@ -1173,7 +1204,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
   return TSDB_CODE_SUCCESS;
 }
 
-static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp) {
+static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
   size_t          readLen = 0;
   char *          line = NULL;
   size_t          n = 0;
@@ -1216,10 +1247,10 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp) {
       maxRows += tSize;    
     }
 
-    len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, pMeterMeta->precision);
+    len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, pMeterMeta->precision, &code, tmpTokenBuf);
     if (len <= 0 || pTableDataBlock->numOfParams > 0) {
-      pSql->res.code = TSDB_CODE_INVALID_SQL;
-      return -1;
+      pSql->res.code = code;
+      return (-code);
     }
     
     pTableDataBlock->size += len;
@@ -1348,8 +1379,16 @@ void tscProcessMultiVnodesInsertForFile(SSqlObj *pSql) {
       tscError("%p get meter meta failed, abort", pSql);
       continue;
     }
+    
+    char*   tmpTokenBuf = calloc(1, 4096);  // used for deleting Escape character: \\, \', \"
+    if (NULL == tmpTokenBuf) {
+      tscError("%p calloc failed", pSql);
+      continue;
+    }
 
-    int nrows = tscInsertDataFromFile(pSql, fp);
+    int nrows = tscInsertDataFromFile(pSql, fp, tmpTokenBuf);
+    free(tmpTokenBuf);
+    
     pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
 
     if (nrows < 0) {
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index ecda6912c3..c66f6f4be7 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -151,6 +151,8 @@ void shellReplaceCtrlChar(char *str) {
           }
           break;
         default:
+          *pstr = *str;
+          pstr++;
           break;
       }
       ctrlOn = false;
diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c
index af8174456c..f402963049 100644
--- a/src/util/src/ttokenizer.c
+++ b/src/util/src/ttokenizer.c
@@ -418,7 +418,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
       int  delim = z[0];
       bool strEnd = false;
       for (i = 1; z[i]; i++) {
-        if (z[i] == delim) {
+        if (z[i] == '\\') { 
+          i++;
+          continue;
+        }
+        
+        if (z[i] == delim ) {
           if (z[i + 1] == delim) {
             i++;
           } else {
@@ -427,6 +432,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) {
           }
         }
       }
+      
       if (z[i]) i++;
 
       if (strEnd) {

From 92a9613e7c72a0e43369de14d8027e6eb3a7d9f4 Mon Sep 17 00:00:00 2001
From: slguan 
Date: Fri, 6 Dec 2019 10:14:07 +0800
Subject: [PATCH 39/42] [TBASE-1284]

---
 src/sdb/src/sdbEngine.c           | 29 +++++++++++++++++++-----
 src/system/detail/src/mgmtMeter.c | 37 ++++++++++++++++++++++++++++---
 2 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c
index 3e7a6ac8ee..5e586de5ad 100644
--- a/src/sdb/src/sdbEngine.c
+++ b/src/sdb/src/sdbEngine.c
@@ -21,6 +21,10 @@
 
 #define abs(x) (((x) < 0) ? -(x) : (x))
 
+#ifndef max
+  #define max(a, b)  ((a) > (b) ? (a) : (b))
+#endif
+
 extern char   version[];
 const int16_t sdbFileVersion = 0;
 int           sdbExtConns = 0;
@@ -171,6 +175,7 @@ int sdbInitTableByFile(SSdbTable *pTable) {
   void *   pMetaRow = NULL;
   int      total_size = 0;
   int      real_size = 0;
+  int      maxAutoIndex = 0;
 
   oldId = pTable->id;
   if (sdbOpenSdbFile(pTable) < 0) return -1;
@@ -240,10 +245,18 @@ int sdbInitTableByFile(SSdbTable *pTable) {
         rowMeta.rowSize = rowHead->rowSize;
         rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL);
         (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta);
-        if (pTable->keyType == SDB_KEYTYPE_AUTO) pTable->autoIndex++;
+        if (pTable->keyType == SDB_KEYTYPE_AUTO) {
+          pTable->autoIndex++;
+          maxAutoIndex = max(maxAutoIndex, *(int32_t*)rowHead->data);
+        }
         pTable->numOfRows++;
       }
     } else {                  // already exists
+      if (pTable->keyType == SDB_KEYTYPE_AUTO) {
+        pTable->autoIndex++;
+        maxAutoIndex = max(maxAutoIndex, *(int32_t *) rowHead->data);
+      }
+
       if (rowHead->id < 0) {  // Delete the object
         (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data);
         (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL);
@@ -260,6 +273,10 @@ int sdbInitTableByFile(SSdbTable *pTable) {
     if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id);
   }
 
+  if (pTable->keyType == SDB_KEYTYPE_AUTO) {
+    pTable->autoIndex = maxAutoIndex;
+  }
+  
   sdbVersion += (pTable->id - oldId);
   if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable);
 
@@ -372,10 +389,10 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) {
             sdbError("table:%s, failed to insert record:%s sdbVersion:%ld id:%d", pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id);
             break;
           case SDB_KEYTYPE_AUTO:
-            sdbError("table:%s, failed to insert record:%s sdbVersion:%ld id:%d", pTable->name, *(int32_t *)row, sdbVersion, pTable->id);
+            sdbError("table:%s, failed to insert record:%d sdbVersion:%ld id:%d", pTable->name, *(int32_t *)row, sdbVersion, pTable->id);
             break;
           default:
-            sdbError("table:%s, failed to insert record:%s sdbVersion:%ld id:%d", pTable->name, sdbVersion, pTable->id);
+            sdbError("table:%s, failed to insert record sdbVersion:%ld id:%d", pTable->name, sdbVersion, pTable->id);
             break;
         }
         return -1;
@@ -593,15 +610,15 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) {
                 pTable->name, (char *) row, sdbVersion, pTable->id);
         break;
       case SDB_KEYTYPE_UINT32: //dnodes or mnodes
-        sdbError("table:%s, failed to update record:%s record is not there, sdbVersion:%ld id:%d",
+        sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%ld id:%d",
                 pTable->name, taosIpStr(*(int32_t *) row), sdbVersion, pTable->id);
         break;
       case SDB_KEYTYPE_AUTO:
-        sdbError("table:%s, failed to update record:F%s record is not there, sdbVersion:%ld id:%d",
+        sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%ld id:%d",
                 pTable->name, *(int32_t *) row, sdbVersion, pTable->id);
         break;
       default:
-        sdbError("table:%s, failed to update record:%s record is not there, sdbVersion:%ld id:%d",
+        sdbError("table:%s, failed to update record, record is not there, sdbVersion:%ld id:%d",
                 pTable->name, sdbVersion, pTable->id);
         break;
     }
diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c
index 0862edf3eb..4e1d62e60e 100644
--- a/src/system/detail/src/mgmtMeter.c
+++ b/src/system/detail/src/mgmtMeter.c
@@ -233,6 +233,10 @@ void *mgmtMeterActionDelete(void *row, char *str, int size, int *ssize) {
   pMeter = (STabObj *)row;
 
   if (mgmtIsNormalMeter(pMeter)) {
+    if (pMeter->gid.vgId == 0) {
+      return NULL;
+    }
+
     pVgroup = mgmtGetVgroup(pMeter->gid.vgId);
     if (pVgroup == NULL) {
       mError("id:%s not in vgroup:%d", pMeter->meterId, pMeter->gid.vgId);
@@ -426,6 +430,7 @@ void mgmtAddMeterStatisticToAcct(STabObj *pMeter, SAcctObj *pAcct) {
 
 int mgmtInitMeters() {
   void *    pNode = NULL;
+  void *    pLastNode = NULL;
   SVgObj *  pVgroup = NULL;
   STabObj * pMeter = NULL;
   STabObj * pMetric = NULL;
@@ -451,21 +456,47 @@ int mgmtInitMeters() {
 
   pNode = NULL;
   while (1) {
+    pLastNode = pNode;
     pNode = sdbFetchRow(meterSdb, pNode, (void **)&pMeter);
     if (pMeter == NULL) break;
 
     pDb = mgmtGetDbByMeterId(pMeter->meterId);
     if (pDb == NULL) {
-      mError("failed to get db: %s", pMeter->meterId);
+      mError("meter:%s, failed to get db, discard it", pMeter->meterId, pMeter->gid.vgId, pMeter->gid.sid);
+      pMeter->gid.vgId = 0;
+      sdbDeleteRow(meterSdb, pMeter);
+      pNode = pLastNode;
       continue;
     }
 
     if (mgmtIsNormalMeter(pMeter)) {
       pVgroup = mgmtGetVgroup(pMeter->gid.vgId);
-      if (pVgroup == NULL || pVgroup->meterList == NULL) {
-        mError("failed to get vgroup:%i", pMeter->gid.vgId);
+
+      if (pVgroup == NULL) {
+        mError("meter:%s, failed to get vgroup:%d sid:%d, discard it", pMeter->meterId, pMeter->gid.vgId, pMeter->gid.sid);
+        pMeter->gid.vgId = 0;
+        sdbDeleteRow(meterSdb, pMeter);
+        pNode = pLastNode;
         continue;
       }
+
+      if (strcmp(pVgroup->dbName, pDb->name) != 0) {
+        mError("meter:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
+               pMeter->meterId, pDb->name, pMeter->gid.vgId, pVgroup->dbName, pMeter->gid.sid);
+        pMeter->gid.vgId = 0;
+        sdbDeleteRow(meterSdb, pMeter);
+        pNode = pLastNode;
+        continue;
+      }
+
+      if ( pVgroup->meterList == NULL) {
+        mError("meter:%s, vgroup:%d meterlist is null", pMeter->meterId, pMeter->gid.vgId);
+        pMeter->gid.vgId = 0;
+        sdbDeleteRow(meterSdb, pMeter);
+        pNode = pLastNode;
+        continue;
+      }
+
       pVgroup->meterList[pMeter->gid.sid] = pMeter;
       taosIdPoolMarkStatus(pVgroup->idPool, pMeter->gid.sid, 1);
 

From a56b019a2524faf2d000e413506bb4140385dcbe Mon Sep 17 00:00:00 2001
From: slguan 
Date: Fri, 6 Dec 2019 10:21:27 +0800
Subject: [PATCH 40/42] [TBASE-1284]

---
 src/sdb/src/sdbEngine.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c
index 5e586de5ad..c115f3c3c0 100644
--- a/src/sdb/src/sdbEngine.c
+++ b/src/sdb/src/sdbEngine.c
@@ -21,10 +21,6 @@
 
 #define abs(x) (((x) < 0) ? -(x) : (x))
 
-#ifndef max
-  #define max(a, b)  ((a) > (b) ? (a) : (b))
-#endif
-
 extern char   version[];
 const int16_t sdbFileVersion = 0;
 int           sdbExtConns = 0;
@@ -247,14 +243,14 @@ int sdbInitTableByFile(SSdbTable *pTable) {
         (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta);
         if (pTable->keyType == SDB_KEYTYPE_AUTO) {
           pTable->autoIndex++;
-          maxAutoIndex = max(maxAutoIndex, *(int32_t*)rowHead->data);
+          maxAutoIndex = MAX(maxAutoIndex, *(int32_t*)rowHead->data);
         }
         pTable->numOfRows++;
       }
     } else {                  // already exists
       if (pTable->keyType == SDB_KEYTYPE_AUTO) {
         pTable->autoIndex++;
-        maxAutoIndex = max(maxAutoIndex, *(int32_t *) rowHead->data);
+        maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data);
       }
 
       if (rowHead->id < 0) {  // Delete the object
@@ -276,7 +272,7 @@ int sdbInitTableByFile(SSdbTable *pTable) {
   if (pTable->keyType == SDB_KEYTYPE_AUTO) {
     pTable->autoIndex = maxAutoIndex;
   }
-  
+
   sdbVersion += (pTable->id - oldId);
   if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable);
 

From 1e9f100473c9b934d3c195452c802438ae74710b Mon Sep 17 00:00:00 2001
From: lihui 
Date: Fri, 6 Dec 2019 17:03:53 +0800
Subject: [PATCH 41/42] [version pos]

---
 packaging/tools/install.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh
index 019e614e0b..e1bcce401d 100755
--- a/packaging/tools/install.sh
+++ b/packaging/tools/install.sh
@@ -313,9 +313,9 @@ vercomp () {
 
 function is_version_compatible() {
 
-    curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 2)
+    curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 3)
 
-    min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 4)
+    min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5)
 
     vercomp $curr_version $min_compatible_version
     case $? in

From 7e1cb6fd5572199fdecc6aaad4d6b2602fede435 Mon Sep 17 00:00:00 2001
From: localvar 
Date: Fri, 6 Dec 2019 09:47:23 +0000
Subject: [PATCH 42/42] fix wrong log in timer

---
 src/util/src/ttimer.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c
index e278f5b9f8..e276710afa 100644
--- a/src/util/src/ttimer.c
+++ b/src/util/src/ttimer.c
@@ -254,13 +254,13 @@ static void processExpiredTimer(void* handle, void* arg) {
   timer->executedBy = taosGetPthreadId();
   uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED);
   if (state == TIMER_STATE_WAITING) {
-    const char* fmt = "%s timer[id=" PRIuPTR ", fp=%p, param=%p] execution start.";
+    const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] execution start.";
     tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param);
 
     (*timer->fp)(timer->param, (tmr_h)timer->id);
     atomic_store_8(&timer->state, TIMER_STATE_STOPPED);
 
-    fmt = "%s timer[id=" PRIuPTR ", fp=%p, param=%p] execution end.";
+    fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] execution end.";
     tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param);
   }
   removeTimer(timer->id);
@@ -268,7 +268,7 @@ static void processExpiredTimer(void* handle, void* arg) {
 }
 
 static void addToExpired(tmr_obj_t* head) {
-  const char* fmt = "%s adding expired timer[id=" PRIuPTR ", fp=%p, param=%p] to queue.";
+  const char* fmt = "%s adding expired timer[id=%" PRIuPTR ", fp=%p, param=%p] to queue.";
 
   while (head != NULL) {
     uintptr_t id = head->id;
@@ -282,7 +282,7 @@ static void addToExpired(tmr_obj_t* head) {
     schedMsg.thandle = NULL;
     taosScheduleTask(tmrQhandle, &schedMsg);
 
-    tmrTrace("timer[id=" PRIuPTR "] has been added to queue.", id);
+    tmrTrace("timer[id=%" PRIuPTR "] has been added to queue.", id);
     head = next;
   }
 }
@@ -296,7 +296,7 @@ static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int msecon
   timer->ctrl = ctrl;
   addTimer(timer);
 
-  const char* fmt = "%s timer[id=" PRIuPTR ", fp=%p, param=%p] started";
+  const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] started";
   tmrTrace(fmt, ctrl->label, timer->id, timer->fp, timer->param);
 
   if (mseconds == 0) {
@@ -389,7 +389,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) {
       // we cannot guarantee the thread safety of the timr in all other cases.
       reusable = true;
     }
-    const char* fmt = "%s timer[id=" PRIuPTR ", fp=%p, param=%p] is cancelled.";
+    const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] is cancelled.";
     tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param);
     return reusable;
   }
@@ -409,7 +409,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) {
   // timer callback is executing in another thread, we SHOULD wait it to stop,
   // BUT this may result in dead lock if current thread are holding a lock which
   // the timer callback need to acquire. so, we HAVE TO return directly.
-  const char* fmt = "%s timer[id=" PRIuPTR ", fp=%p, param=%p] is executing and cannot be stopped.";
+  const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] is executing and cannot be stopped.";
   tmrTrace(fmt, timer->ctrl->label, timer->id, timer->fp, timer->param);
   return false;
 }
@@ -419,7 +419,7 @@ bool taosTmrStop(tmr_h timerId) {
 
   tmr_obj_t* timer = findTimer(id);
   if (timer == NULL) {
-    tmrTrace("timer[id=" PRIuPTR "] does not exist", id);
+    tmrTrace("timer[id=%" PRIuPTR "] does not exist", id);
     return false;
   }
 
@@ -446,7 +446,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle,
   bool       stopped = false;
   tmr_obj_t* timer = findTimer(id);
   if (timer == NULL) {
-    tmrTrace("%s timer[id=" PRIuPTR "] does not exist", ctrl->label, id);
+    tmrTrace("%s timer[id=%" PRIuPTR "] does not exist", ctrl->label, id);
   } else {
     uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED);
     if (!doStopTimer(timer, state)) {
@@ -461,7 +461,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle,
     return stopped;
   }
 
-  tmrTrace("%s timer[id=" PRIuPTR "] is reused", ctrl->label, timer->id);
+  tmrTrace("%s timer[id=%" PRIuPTR "] is reused", ctrl->label, timer->id);
 
   // wait until there's no other reference to this timer,
   // so that we can reuse this timer safely.