From 115d67e0926d7bb6d89cf2cea9f84c0ed5672473 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 6 Jan 2020 18:28:06 +0800 Subject: [PATCH 01/14] TBASE-1442 #1025 --- src/inc/taoserror.h | 2 +- src/inc/tstatus.h | 14 +++++++----- src/rpc/src/tstring.c | 4 ++-- src/system/detail/src/mgmtDnodeInt.c | 12 ++++------- src/system/detail/src/mgmtMeter.c | 12 +++-------- src/system/detail/src/vnodeFile.c | 2 +- src/system/detail/src/vnodeMeter.c | 14 ++++++++---- src/system/detail/src/vnodeStore.c | 32 ++++++++++++++++------------ src/util/src/tstatus.c | 12 ++++++----- 9 files changed, 55 insertions(+), 49 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5510212dbc..78aeaad475 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -91,7 +91,7 @@ extern "C" { #define TSDB_CODE_INVALID_QHANDLE 70 #define TSDB_CODE_RELATED_TABLES_EXIST 71 #define TSDB_CODE_MONITOR_DB_FORBEIDDEN 72 -#define TSDB_CODE_VG_COMMITLOG_INIT_FAILED 73 +#define TSDB_CODE_NO_DISK_PERMISSIONS 73 #define TSDB_CODE_VG_INIT_FAILED 74 #define TSDB_CODE_DATA_ALREADY_IMPORTED 75 #define TSDB_CODE_OPS_NOT_SUPPORT 76 diff --git a/src/inc/tstatus.h b/src/inc/tstatus.h index 53b5cb9547..58b98c59f8 100644 --- a/src/inc/tstatus.h +++ b/src/inc/tstatus.h @@ -16,16 +16,20 @@ #ifndef TDENGINE_TSTATUS_H #define TDENGINE_TSTATUS_H +#include "taoserror.h" + #ifdef __cplusplus extern "C" { #endif enum _TSDB_VG_STATUS { - TSDB_VG_STATUS_READY, - TSDB_VG_STATUS_IN_PROGRESS, - TSDB_VG_STATUS_COMMITLOG_INIT_FAILED, - TSDB_VG_STATUS_INIT_FAILED, - TSDB_VG_STATUS_FULL + TSDB_VG_STATUS_READY = TSDB_CODE_SUCCESS, + TSDB_VG_STATUS_IN_PROGRESS = TSDB_CODE_ACTION_IN_PROGRESS, + TSDB_VG_STATUS_NO_DISK_PERMISSIONS = TSDB_CODE_NO_DISK_PERMISSIONS, + TSDB_VG_STATUS_SERVER_NO_PACE = TSDB_CODE_SERVER_NO_SPACE, + TSDB_VG_STATUS_SERV_OUT_OF_MEMORY = TSDB_CODE_SERV_OUT_OF_MEMORY, + TSDB_VG_STATUS_INIT_FAILED = TSDB_CODE_VG_INIT_FAILED, + TSDB_VG_STATUS_FULL = TSDB_CODE_NO_ENOUGH_DNODES, }; enum _TSDB_DB_STATUS { diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index e3daca130a..cf40fb5058 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -197,7 +197,7 @@ char *tsError[] = {"success", "invalid query handle", // 70 "tables related to metric exist", "can't drop monitor database or tables", - "commit log init failed", + "no disk permissions", "vgroup init failed", "data is already imported", // 75 "not supported operation", @@ -234,7 +234,7 @@ char *tsError[] = {"success", "timestamp disordered in cache block", "timestamp disordered in file block", "invalid commit log", - "server no disk space", //110 + "no disk space on server", //110 "only super table has metric meta info", "tags value not unique for join", "invalid submit message", diff --git a/src/system/detail/src/mgmtDnodeInt.c b/src/system/detail/src/mgmtDnodeInt.c index 6b6571b06c..380cd2aff4 100644 --- a/src/system/detail/src/mgmtDnodeInt.c +++ b/src/system/detail/src/mgmtDnodeInt.c @@ -152,19 +152,15 @@ int mgmtProcessVPeersRsp(char *msg, int msgLen, SDnodeObj *pObj) { return 0; } - if (pRsp->code == 0) { + if (pRsp->code == TSDB_CODE_SUCCESS) { pDb->vgStatus = TSDB_VG_STATUS_READY; mTrace("dnode:%s, db:%s vgroup is created in dnode", taosIpStr(pObj->privateIp), pRsp->more); return 0; } - if (pRsp->code == TSDB_CODE_VG_COMMITLOG_INIT_FAILED) { - pDb->vgStatus = TSDB_VG_STATUS_COMMITLOG_INIT_FAILED; - mError("dnode:%s, db:%s vgroup commit log init failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); - } else { - pDb->vgStatus = TSDB_VG_STATUS_INIT_FAILED; - mError("dnode:%s, db:%s vgroup init failed, code:%d", taosIpStr(pObj->privateIp), pRsp->more, pRsp->code); - } + pDb->vgStatus = pRsp->code; + mError("dnode:%s, db:%s vgroup init failed, code:%d %s", + taosIpStr(pObj->privateIp), pRsp->more, pRsp->code, taosGetVgroupStatusStr(pDb->vgStatus)); return 0; } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 7a2cdabea5..70c482d40c 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -657,16 +657,10 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { return TSDB_CODE_NO_ENOUGH_DNODES; } - if (pDb->vgStatus == TSDB_VG_STATUS_COMMITLOG_INIT_FAILED) { + if (pDb->vgStatus < TSDB_VG_STATUS_FULL && pDb->vgStatus > TSDB_CODE_ACTION_IN_PROGRESS) { mgmtDestroyMeter(pMeter); - mError("table:%s, commit log init failed", pCreate->meterId); - return TSDB_CODE_VG_COMMITLOG_INIT_FAILED; - } - - if (pDb->vgStatus == TSDB_VG_STATUS_INIT_FAILED) { - mgmtDestroyMeter(pMeter); - mError("table:%s, vgroup init failed", pCreate->meterId); - return TSDB_CODE_VG_INIT_FAILED; + mError("table:%s, vgroup init failed, reason:%d %s", pCreate->meterId, pDb->vgStatus, taosGetVgroupStatusStr(pDb->vgStatus)); + return pDb->vgStatus; } if (pVgroup == NULL) { diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index 677a8d8a1e..860b7624da 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -1823,7 +1823,7 @@ int vnodeUpdateFileMagic(int vnode, int fileId) { } int vnodeInitFile(int vnode) { - int code = 0; + int code = TSDB_CODE_SUCCESS; SVnodeObj *pVnode = vnodeList + vnode; pVnode->maxFiles = pVnode->cfg.daysToKeep / pVnode->cfg.daysPerFile + 1; diff --git a/src/system/detail/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c index 77bafd50d4..a5b9db2d9c 100644 --- a/src/system/detail/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -79,8 +79,14 @@ int vnodeCreateMeterObjFile(int vnode) { sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode); fp = fopen(fileName, "w+"); if (fp == NULL) { - dError("failed to create vnode:%d file:%s", vnode, fileName); - return -1; + dError("failed to create vnode:%d file:%s, reason:%s", vnode, fileName, strerror(errno)); + if (errno == EACCES) { + return TSDB_CODE_NO_DISK_PERMISSIONS; + } else if (errno == ENOSPC) { + return TSDB_CODE_SERVER_NO_SPACE; + } else { + return TSDB_CODE_VG_INIT_FAILED; + } } else { vnodeCreateFileHeader(fp); vnodeUpdateVnodeFileHeader(fp, vnodeList + vnode); @@ -95,7 +101,7 @@ int vnodeCreateMeterObjFile(int vnode) { fclose(fp); } - return 0; + return TSDB_CODE_SUCCESS; } FILE *vnodeOpenMeterObjFile(int vnode) { @@ -273,7 +279,7 @@ int vnodeSaveVnodeCfg(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) { /* vnodeUpdateFileCheckSum(fp); */ fclose(fp); - return 0; + return TSDB_CODE_SUCCESS; } int vnodeSaveVnodeInfo(int vnode) { diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index f2617f09b6..8568163333 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -45,24 +45,24 @@ static int vnodeInitStoreVnode(int vnode) { pVnode->pCachePool = vnodeOpenCachePool(vnode); if (pVnode->pCachePool == NULL) { dError("vid:%d, cache pool init failed.", pVnode->vnode); - return -1; + return TSDB_CODE_SERV_OUT_OF_MEMORY; } - if (vnodeInitFile(vnode) < 0) { + if (vnodeInitFile(vnode) != TSDB_CODE_SUCCESS) { dError("vid:%d, files init failed.", pVnode->vnode); - return -1; + return TSDB_CODE_VG_INIT_FAILED; } - if (vnodeInitCommit(vnode) < 0) { + if (vnodeInitCommit(vnode) != TSDB_CODE_SUCCESS) { dError("vid:%d, commit init failed.", pVnode->vnode); - return -1; + return TSDB_CODE_VG_INIT_FAILED; } pthread_mutex_init(&(pVnode->vmutex), NULL); dPrint("vid:%d, storage initialized, version:%ld fileId:%d numOfFiles:%d", vnode, pVnode->version, pVnode->fileId, pVnode->numOfFiles); - return 0; + return TSDB_CODE_SUCCESS; } int vnodeOpenVnode(int vnode) { @@ -192,16 +192,19 @@ int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) { mkdir(fileName, 0755); vnodeList[vnode].cfg = *pCfg; - if (vnodeCreateMeterObjFile(vnode) != 0) { + int code = vnodeCreateMeterObjFile(vnode); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = vnodeSaveVnodeCfg(vnode, pCfg, pDesc); + if (code != TSDB_CODE_SUCCESS) { return TSDB_CODE_VG_INIT_FAILED; } - if (vnodeSaveVnodeCfg(vnode, pCfg, pDesc) != 0) { - return TSDB_CODE_VG_INIT_FAILED; - } - - if (vnodeInitStoreVnode(vnode) < 0) { - return TSDB_CODE_VG_COMMITLOG_INIT_FAILED; + code = vnodeInitStoreVnode(vnode); + if (code != TSDB_CODE_SUCCESS) { + return code; } return vnodeOpenVnode(vnode); @@ -294,7 +297,8 @@ int vnodeInitStore() { if (vnodeInitInfo() < 0) return -1; for (vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) { - if (vnodeInitStoreVnode(vnode) < 0) { + int code = vnodeInitStoreVnode(vnode); + if (code != TSDB_CODE_SUCCESS) { // one vnode is failed to recover from commit log, continue for remain return -1; } diff --git a/src/util/src/tstatus.c b/src/util/src/tstatus.c index 1ab007715d..21dbd8c67d 100644 --- a/src/util/src/tstatus.c +++ b/src/util/src/tstatus.c @@ -19,11 +19,13 @@ const char* taosGetVgroupStatusStr(int vgroupStatus) { switch (vgroupStatus) { - case TSDB_VG_STATUS_READY: return "ready"; - case TSDB_VG_STATUS_IN_PROGRESS: return "inprogress"; - case TSDB_VG_STATUS_COMMITLOG_INIT_FAILED: return "commitlog_init_failed"; - case TSDB_VG_STATUS_INIT_FAILED: return "init_failed"; - case TSDB_VG_STATUS_FULL: return "full"; + case TSDB_VG_STATUS_READY: return tsError[vgroupStatus]; + case TSDB_VG_STATUS_IN_PROGRESS: return tsError[vgroupStatus]; + case TSDB_VG_STATUS_NO_DISK_PERMISSIONS: return tsError[vgroupStatus]; + case TSDB_VG_STATUS_SERVER_NO_PACE: return tsError[vgroupStatus]; + case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY: return tsError[vgroupStatus]; + case TSDB_VG_STATUS_INIT_FAILED: return tsError[vgroupStatus]; + case TSDB_VG_STATUS_FULL: return tsError[vgroupStatus]; default: return "undefined"; } } From 9362bfe5024d022ec2a72d1220555a137fbf6ddc Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Jan 2020 18:01:06 +0800 Subject: [PATCH 02/14] TBASE-1398 #1073 --- src/system/detail/src/dnodeMgmt.c | 4 ++-- src/system/detail/src/mgmtMeter.c | 5 ++++- src/system/detail/src/vnodeMeter.c | 2 +- src/system/detail/src/vnodeStore.c | 21 ++++++++++++++++++++- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/system/detail/src/dnodeMgmt.c b/src/system/detail/src/dnodeMgmt.c index a2496c6a42..69d4dc2c30 100644 --- a/src/system/detail/src/dnodeMgmt.c +++ b/src/system/detail/src/dnodeMgmt.c @@ -415,10 +415,10 @@ int vnodeProcessVPeerCfgRsp(char *msg, int msgLen, SMgmtObj *pMgmtObj) { int32_t *pint = (int32_t *)pRsp->more; int vnode = htonl(*pint); if (vnode < TSDB_MAX_VNODES && vnodeList[vnode].lastKey != 0) { - dError("vnode:%d not configured, it shall be empty"); + dError("vnode:%d not configured, it shall be empty, code:%d", vnode, pRsp->code); vnodeRemoveVnode(vnode); } else { - dTrace("vnode:%d is invalid", vnode); + dError("vnode:%d is invalid, code:%d", vnode, pRsp->code); } } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 70c482d40c..05ee0762be 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -657,7 +657,10 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) { return TSDB_CODE_NO_ENOUGH_DNODES; } - if (pDb->vgStatus < TSDB_VG_STATUS_FULL && pDb->vgStatus > TSDB_CODE_ACTION_IN_PROGRESS) { + if (pDb->vgStatus == TSDB_VG_STATUS_NO_DISK_PERMISSIONS || + pDb->vgStatus == TSDB_VG_STATUS_SERVER_NO_PACE || + pDb->vgStatus == TSDB_VG_STATUS_SERV_OUT_OF_MEMORY || + pDb->vgStatus == TSDB_VG_STATUS_INIT_FAILED ) { mgmtDestroyMeter(pMeter); mError("table:%s, vgroup init failed, reason:%d %s", pCreate->meterId, pDb->vgStatus, taosGetVgroupStatusStr(pDb->vgStatus)); return pDb->vgStatus; diff --git a/src/system/detail/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c index a5b9db2d9c..a2da561434 100644 --- a/src/system/detail/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -79,7 +79,7 @@ int vnodeCreateMeterObjFile(int vnode) { sprintf(fileName, "%s/vnode%d/meterObj.v%d", tsDirectory, vnode, vnode); fp = fopen(fileName, "w+"); if (fp == NULL) { - dError("failed to create vnode:%d file:%s, reason:%s", vnode, fileName, strerror(errno)); + dError("failed to create vnode:%d file:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno)); if (errno == EACCES) { return TSDB_CODE_NO_DISK_PERMISSIONS; } else if (errno == ENOSPC) { diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index 8568163333..b0171fd073 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -186,10 +186,29 @@ int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) { vnodeList[vnode].vnodeStatus = TSDB_VN_STATUS_CREATING; sprintf(fileName, "%s/vnode%d", tsDirectory, vnode); - mkdir(fileName, 0755); + if (mkdir(fileName, 0755) != 0) { + dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno)); + if (errno == EACCES) { + return TSDB_CODE_NO_DISK_PERMISSIONS; + } else if (errno == ENOSPC) { + return TSDB_CODE_SERVER_NO_SPACE; + } else { + return TSDB_CODE_VG_INIT_FAILED; + } + } sprintf(fileName, "%s/vnode%d/db", tsDirectory, vnode); mkdir(fileName, 0755); + if (mkdir(fileName, 0755) != 0) { + dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno)); + if (errno == EACCES) { + return TSDB_CODE_NO_DISK_PERMISSIONS; + } else if (errno == ENOSPC) { + return TSDB_CODE_SERVER_NO_SPACE; + } else { + return TSDB_CODE_VG_INIT_FAILED; + } + } vnodeList[vnode].cfg = *pCfg; int code = vnodeCreateMeterObjFile(vnode); From fe582f9156c9b71e9fddd0b93eeb57117a03cc49 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 8 Jan 2020 18:35:34 +0800 Subject: [PATCH 03/14] TBASE-1442 #1025 --- src/system/detail/src/vnodeStore.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index b0171fd073..0edd182dca 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -192,19 +192,20 @@ int vnodeCreateVnode(int vnode, SVnodeCfg *pCfg, SVPeerDesc *pDesc) { return TSDB_CODE_NO_DISK_PERMISSIONS; } else if (errno == ENOSPC) { return TSDB_CODE_SERVER_NO_SPACE; + } else if (errno == EEXIST) { } else { return TSDB_CODE_VG_INIT_FAILED; } } sprintf(fileName, "%s/vnode%d/db", tsDirectory, vnode); - mkdir(fileName, 0755); if (mkdir(fileName, 0755) != 0) { dError("failed to create vnode:%d directory:%s, errno:%d, reason:%s", vnode, fileName, errno, strerror(errno)); if (errno == EACCES) { return TSDB_CODE_NO_DISK_PERMISSIONS; } else if (errno == ENOSPC) { return TSDB_CODE_SERVER_NO_SPACE; + } else if (errno == EEXIST) { } else { return TSDB_CODE_VG_INIT_FAILED; } From 97c6bebfdeb35b17e62fa93049d4b1847af4820c Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 9 Jan 2020 18:28:45 +0800 Subject: [PATCH 04/14] TBASE-926, data may be lost when the cache is full --- src/system/detail/src/vnodeMeter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/system/detail/src/vnodeMeter.c b/src/system/detail/src/vnodeMeter.c index a2da561434..190ddb1b09 100644 --- a/src/system/detail/src/vnodeMeter.c +++ b/src/system/detail/src/vnodeMeter.c @@ -576,6 +576,14 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi return code; } + /* + * please refer to TBASE-926, data may be lost when the cache is full + */ + if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) { + code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion); + if (code != TSDB_CODE_SUCCESS) return code; + } + SCachePool *pPool = (SCachePool *)pVnode->pCachePool; if (pObj->freePoints < numOfPoints || pObj->freePoints < (pObj->pointsPerBlock << 1) || pPool->notFreeSlots > pVnode->cfg.cacheNumOfBlocks.totalBlocks - 2) { @@ -593,11 +601,6 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi if (code != TSDB_CODE_SUCCESS) return code; } - if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) { - code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion); - if (code != TSDB_CODE_SUCCESS) return code; - } - if (pObj->sversion < sversion) { dTrace("vid:%d sid:%d id:%s, schema is changed, new:%d old:%d", pObj->vnode, pObj->sid, pObj->meterId, sversion, pObj->sversion); From 4000b0e9ad25eff882a52616c514ca50a8b89043 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 11 Jan 2020 13:56:05 +0800 Subject: [PATCH 05/14] #1120 TBASE-1154 --- src/modules/http/src/gcJson.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/modules/http/src/gcJson.c b/src/modules/http/src/gcJson.c index 0cb20ec7e1..7df8114162 100644 --- a/src/modules/http/src/gcJson.c +++ b/src/modules/http/src/gcJson.c @@ -79,7 +79,9 @@ void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) { if (jsonBuf == NULL) return; // write end of target - gcWriteTargetEndJson(jsonBuf); + if (cmd->numOfRows != 0) { + gcWriteTargetEndJson(jsonBuf); + } } bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) { @@ -116,8 +118,8 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (groupFields == -1 && cmd->numOfRows == 0) { gcWriteTargetStartJson(jsonBuf, refIdBuffer, aliasBuffer); - cmd->numOfRows += numOfRows; } + cmd->numOfRows += numOfRows; for (int i = 0; i < numOfRows; ++i) { TAOS_ROW row = taos_fetch_row(result); @@ -158,7 +160,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } break; default: - snprintf(target, HTTP_GC_TARGET_SIZE, "%s%s", aliasBuffer, "invalidcol"); + snprintf(target, HTTP_GC_TARGET_SIZE, "%s%s", aliasBuffer, "-"); break; } @@ -217,7 +219,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } break; default: - httpJsonString(jsonBuf, "invalidcol", 10); + httpJsonString(jsonBuf, "-", 1); break; } } From ded4a1bf6f41dd247caf8d33351b14f0673408f5 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Sat, 11 Jan 2020 15:47:08 +0800 Subject: [PATCH 06/14] [TBASE-952] add dockerfile and dockerbuild scripts --- packaging/Dockerfile | 18 ++++++++++++++++++ packaging/dockerbuild.sh | 12 ++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 packaging/Dockerfile create mode 100755 packaging/dockerbuild.sh diff --git a/packaging/Dockerfile b/packaging/Dockerfile new file mode 100644 index 0000000000..b01f375db0 --- /dev/null +++ b/packaging/Dockerfile @@ -0,0 +1,18 @@ +FROM centos:7 + +WORKDIR /root + +COPY tdengine.tar.gz /root/ +RUN tar -zxf tdengine.tar.gz +WORKDIR /root/tdengine/ +RUN sh install.sh + + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +EXPOSE 6020 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 +EXPOSE 6043 6044 6045 6046 6047 6048 6049 6050 +CMD ["taosd"] +VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] diff --git a/packaging/dockerbuild.sh b/packaging/dockerbuild.sh new file mode 100755 index 0000000000..280c27d7aa --- /dev/null +++ b/packaging/dockerbuild.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -x +$1 +tar -zxf $1 +DIR=`echo $1|awk -F . '{print($1"."$2"."$3"."$4)}'` +mv $DIR tdengine +tar -czf tdengine.tar.gz tdengine +TMP=`echo $1|awk -F . '{print($2"."$3"."$4)}'` +TAG="1."$TMP +docker build --rm -f "Dockerfile" -t tdengine/tdengine:$TAG "." +docker login -u tdengine -p ******** #replace the docker registry username and password +docker push tdengine/tdengine:$TAG \ No newline at end of file From f1cfc460d944e024547ab186c950620db5f71b16 Mon Sep 17 00:00:00 2001 From: Tao Liu <56432938+liu0x54@users.noreply.github.com> Date: Sat, 11 Jan 2020 15:51:05 +0800 Subject: [PATCH 07/14] Delete Dockerfile --- packaging/Dockerfile | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 packaging/Dockerfile diff --git a/packaging/Dockerfile b/packaging/Dockerfile deleted file mode 100644 index b01f375db0..0000000000 --- a/packaging/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM centos:7 - -WORKDIR /root - -COPY tdengine.tar.gz /root/ -RUN tar -zxf tdengine.tar.gz -WORKDIR /root/tdengine/ -RUN sh install.sh - - -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 -EXPOSE 6020 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 -EXPOSE 6043 6044 6045 6046 6047 6048 6049 6050 -CMD ["taosd"] -VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] From 7d5ca4d66f601a487bb73af08de3061cc3b09ea9 Mon Sep 17 00:00:00 2001 From: Tao Liu <56432938+liu0x54@users.noreply.github.com> Date: Sat, 11 Jan 2020 15:51:18 +0800 Subject: [PATCH 08/14] Delete dockerbuild.sh --- packaging/dockerbuild.sh | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100755 packaging/dockerbuild.sh diff --git a/packaging/dockerbuild.sh b/packaging/dockerbuild.sh deleted file mode 100755 index 280c27d7aa..0000000000 --- a/packaging/dockerbuild.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -x -$1 -tar -zxf $1 -DIR=`echo $1|awk -F . '{print($1"."$2"."$3"."$4)}'` -mv $DIR tdengine -tar -czf tdengine.tar.gz tdengine -TMP=`echo $1|awk -F . '{print($2"."$3"."$4)}'` -TAG="1."$TMP -docker build --rm -f "Dockerfile" -t tdengine/tdengine:$TAG "." -docker login -u tdengine -p ******** #replace the docker registry username and password -docker push tdengine/tdengine:$TAG \ No newline at end of file From fc1ee73f7374f56527eb95afe2e9cb307b037f9d Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Sat, 11 Jan 2020 16:00:52 +0800 Subject: [PATCH 09/14] [TBASE-952] add dockerfile and dockerbuild scripts --- packaging/docker/Dockerfile | 18 ++++++++++++++++++ packaging/docker/dockerbuild.sh | 12 ++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 packaging/docker/Dockerfile create mode 100755 packaging/docker/dockerbuild.sh diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 0000000000..b01f375db0 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM centos:7 + +WORKDIR /root + +COPY tdengine.tar.gz /root/ +RUN tar -zxf tdengine.tar.gz +WORKDIR /root/tdengine/ +RUN sh install.sh + + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +EXPOSE 6020 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 +EXPOSE 6043 6044 6045 6046 6047 6048 6049 6050 +CMD ["taosd"] +VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh new file mode 100755 index 0000000000..280c27d7aa --- /dev/null +++ b/packaging/docker/dockerbuild.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -x +$1 +tar -zxf $1 +DIR=`echo $1|awk -F . '{print($1"."$2"."$3"."$4)}'` +mv $DIR tdengine +tar -czf tdengine.tar.gz tdengine +TMP=`echo $1|awk -F . '{print($2"."$3"."$4)}'` +TAG="1."$TMP +docker build --rm -f "Dockerfile" -t tdengine/tdengine:$TAG "." +docker login -u tdengine -p ******** #replace the docker registry username and password +docker push tdengine/tdengine:$TAG \ No newline at end of file From 2c1dccd40faa501b463a4e6e6e309db88897b947 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Mon, 13 Jan 2020 11:57:16 +0800 Subject: [PATCH 10/14] Update tscServer.c fix bug #1124 [TBASE-1483] --- src/client/src/tscServer.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6981c16e9e..1a05ce374a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1090,7 +1090,9 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } - if (atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { + int32_t numOfTotal = pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { // pState may be released by otherthreads, so keep the value in a local variable. return tscFreeSubSqlObj(trsupport, pSql); } @@ -1215,7 +1217,14 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); } - if (atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1) < trsupport->pState->numOfTotal) { + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); return tscFreeSubSqlObj(trsupport, pSql); } @@ -1223,7 +1232,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { pDesc->pSchema->maxCapacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - trsupport->pState->numOfTotal, trsupport->pState->numOfCompleted); + trsupport->pState->numOfTotal, trsupport->pState->numOfRetrievedRows); tscClearInterpInfo(&pPObj->cmd); tscCreateLocalReducer(trsupport->pExtMemBuffer, trsupport->pState->numOfTotal, pDesc, trsupport->pFinalColModel, From 2cdc335d775d874820ccc426a6ffdfc703f4ab31 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Mon, 13 Jan 2020 13:46:30 +0800 Subject: [PATCH 11/14] Update tscServer.c --- src/client/src/tscServer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1a05ce374a..6bf0d74b6f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1090,8 +1090,8 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } - int32_t numOfTotal = pState->numOfTotal; - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + int32_t numOfTotal = trsupport->pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1); if (finished < numOfTotal) { // pState may be released by otherthreads, so keep the value in a local variable. return tscFreeSubSqlObj(trsupport, pSql); } @@ -1220,9 +1220,9 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. - int32_t numOfTotal = pState->numOfTotal; + int32_t numOfTotal = trsupport->pState->numOfTotal; - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + int32_t finished = atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1); if (finished < numOfTotal) { tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); return tscFreeSubSqlObj(trsupport, pSql); From 31ed41fc01c65019856a71fddb6213a8fedb9116 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Mon, 13 Jan 2020 13:52:06 +0800 Subject: [PATCH 12/14] Update tscServer.c --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6bf0d74b6f..9151159042 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1224,7 +1224,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { int32_t finished = atomic_add_fetch_32(&trsupport->pState->numOfCompleted, 1); if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->vnodeIdx, finished); return tscFreeSubSqlObj(trsupport, pSql); } From 8c05d36082850d830135c85555bfc160b0a22735 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 13 Jan 2020 19:03:01 +0800 Subject: [PATCH 13/14] for async test --- src/modules/http/src/restHandle.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/modules/http/src/restHandle.c b/src/modules/http/src/restHandle.c index 58509e693d..a307700866 100644 --- a/src/modules/http/src/restHandle.c +++ b/src/modules/http/src/restHandle.c @@ -67,10 +67,16 @@ bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) { return false; } + + /* + * for async test + * / + /* if (httpCheckUsedbSql(sql)) { httpSendErrorResp(pContext, HTTP_NO_EXEC_USEDB); return false; } + */ HttpSqlCmd* cmd = &(pContext->singleCmd); cmd->nativSql = sql; From 981d7c5e0231f4025ad5b3e73e9b65dd6ab39b00 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 14 Jan 2020 10:36:50 +0800 Subject: [PATCH 14/14] #1130 TBASE-1484 --- src/modules/http/src/httpJson.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/modules/http/src/httpJson.c b/src/modules/http/src/httpJson.c index 2bb768e801..ca88de59e6 100644 --- a/src/modules/http/src/httpJson.c +++ b/src/modules/http/src/httpJson.c @@ -310,7 +310,9 @@ void httpJsonInt(JsonBuf* buf, int num) { void httpJsonFloat(JsonBuf* buf, float num) { httpJsonItemToken(buf); httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - if (num > 1E10 || num < -1E10) { + if (isinf(num) || isnan(num)) { + buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null"); + } else if (num > 1E10 || num < -1E10) { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5e", num); } else { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5f", num); @@ -320,7 +322,9 @@ void httpJsonFloat(JsonBuf* buf, float num) { void httpJsonDouble(JsonBuf* buf, double num) { httpJsonItemToken(buf); httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - if (num > 1E10 || num < -1E10) { + if (isinf(num) || isnan(num)) { + buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null"); + } else if (num > 1E10 || num < -1E10) { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9e", num); } else { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9f", num);