From 9b0bfc895952a238f942a5658d38044efe60694f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 29 Jul 2020 17:37:49 +0800 Subject: [PATCH 01/18] make SDataCols buffer flexible --- src/common/inc/tdataformat.h | 15 ++++++++------- src/common/src/tdataformat.c | 29 +++++++++++++++++++++++++---- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMemTable.c | 7 +++++-- src/tsdb/src/tsdbRWHelper.c | 15 ++++++++++++--- 6 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 2ed4b81204..68875341ba 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -235,11 +235,11 @@ typedef struct { int maxPoints; // max number of points int bufSize; - int numOfRows; - int numOfCols; // Total number of cols - int sversion; // TODO: set sversion - void * buf; - SDataCol cols[]; + int numOfRows; + int numOfCols; // Total number of cols + int sversion; // TODO: set sversion + void * buf; + SDataCol *cols; } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column @@ -249,13 +249,14 @@ typedef struct { SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); -void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows); +void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows); // ----------------- K-V data row structure /* diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 12ea4ad78d..adfce5580e 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -310,9 +310,15 @@ void dataColSetOffset(SDataCol *pCol, int nEle) { } SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { - SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols); + SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); if (pCols == NULL) return NULL; + pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); + if (pCols->cols == NULL) { + tdFreeDataCols(pCols); + return NULL; + } + pCols->maxRowSize = maxRowSize; pCols->maxCols = maxCols; pCols->maxPoints = maxRows; @@ -320,15 +326,27 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->buf = malloc(pCols->bufSize); if (pCols->buf == NULL) { - free(pCols); + tdFreeDataCols(pCols); return NULL; } return pCols; } -void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { - // assert(schemaNCols(pSchema) <= pCols->numOfCols); +int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { + if (schemaNCols(pSchema) > pCols->maxCols) { + pCols->maxCols = schemaNCols(pSchema); + pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + if (pCols->cols == NULL) return -1; + } + + if (schemaTLen(pSchema) > pCols->maxRowSize) { + pCols->maxRowSize = schemaTLen(pSchema); + pCols->bufSize = schemaTLen(pSchema) * pCols->maxPoints; + pCols->buf = realloc(pCols->buf, pCols->bufSize); + if (pCols->buf == NULL) return -1; + } + tdResetDataCols(pCols); pCols->numOfCols = schemaNCols(pSchema); @@ -337,11 +355,14 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), &ptr, pCols->maxPoints); ASSERT((char *)ptr - (char *)(pCols->buf) <= pCols->bufSize); } + + return 0; } void tdFreeDataCols(SDataCols *pCols) { if (pCols) { tfree(pCols->buf); + tfree(pCols->cols); free(pCols); } } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 3e92c01765..6a54cb2399 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -483,7 +483,7 @@ void tsdbDestroyHelper(SRWHelper* pHelper); void tsdbResetHelper(SRWHelper* pHelper); int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError); -void tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); +int tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey); int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); int tsdbWriteCompInfo(SRWHelper* pHelper); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 3a453249a9..7f674f09e5 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -788,7 +788,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; - tsdbSetHelperTable(&rhelper, pTable, pRepo); + if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err; SCompIdx *pIdx = &(rhelper.curCompIdx); if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index a65d956eb0..506f3c11c1 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -610,10 +610,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe taosRLockLatch(&(pIter->pTable->latch)); - tsdbSetHelperTable(pHelper, pIter->pTable, pRepo); + if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err; if (pIter->pIter != NULL) { - tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)); + if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) { taosRUnLockLatch(&(pIter->pTable->latch)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9456996071..8d3908666e 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -262,7 +262,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { return 0; } -void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { +int tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); // Clear members and state used by previous table @@ -273,8 +273,15 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { pHelper->tableInfo.uid = pTable->tableId.uid; STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - tdInitDataCols(pHelper->pDataCols[0], pSchema); - tdInitDataCols(pHelper->pDataCols[1], pSchema); + if (tdInitDataCols(pHelper->pDataCols[0], pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tdInitDataCols(pHelper->pDataCols[1], pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } if (pHelper->idxH.numOfIdx > 0) { while (true) { @@ -309,6 +316,8 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { helperSetState(pHelper, TSDB_HELPER_TABLE_SET); ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1)); + + return 0; } int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { From ddcfa4c798feff307307438ec09c30ec0a7a92ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 29 Jul 2020 18:33:15 +0800 Subject: [PATCH 02/18] fix a coverity scan problem --- src/util/inc/tchecksum.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/util/inc/tchecksum.h b/src/util/inc/tchecksum.h index 493c993c25..54043d68e1 100644 --- a/src/util/inc/tchecksum.h +++ b/src/util/inc/tchecksum.h @@ -35,8 +35,6 @@ extern "C" { typedef uint32_t TSCKSUM; static FORCE_INLINE TSCKSUM taosCalcChecksum(TSCKSUM csi, const uint8_t *stream, uint32_t ssize) { - assert(ssize >= 0 && stream != NULL); - return (*crc32c)(csi, stream, (size_t)ssize); } From 0bc21b284950906d711abc5e5f6a0667ff346882 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 30 Jul 2020 11:08:29 +0800 Subject: [PATCH 03/18] [modify install] --- packaging/cfg/taos.cfg | 7 +-- packaging/tools/install.sh | 46 +++++++------------- tests/script/sh/deploy.sh | 88 +++++++++++++++++++------------------- tests/script/test.sh | 45 +++++++++---------- 4 files changed, 84 insertions(+), 102 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index f2dbc04353..3a03972e68 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -6,10 +6,10 @@ ######################################################## # first full-qualified domain name (FQDN) for TDengine system -# first hostname1:6030 +# firstEp hostname1:6030 # second full-qualified domain name (FQDN) for TDengine system, for cluster edition only -# second cluster_hostname2:6030 +# secondEp cluster_hostname2:6030 # the full-qualified domain name (FQDN) of dnode # fqdn hostname @@ -23,9 +23,6 @@ # log file's directory # logDir /var/log/taos -# scriptDir file's directory -# scriptDir /var/log/taos - # data file's directory # dataDir /var/lib/taos diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index b2613a7cb6..2191667242 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -241,36 +241,17 @@ function install_config() { # first full-qualified domain name (FQDN) for TDengine cluster system echo - echo -e -n "${GREEN}Enter the FQDN of an existing TDengine cluster node to join${NC} OR ${GREEN}leave it blank to build one${NC} :" - read firstFqdn + echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join OR leave it blank to build one${NC} :" + read firstEp while true; do - if [ ! -z "$firstFqdn" ]; then - # check the format of the firstFqdn - #if [[ $firstFqdn == $FQDN_PATTERN ]]; then - # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(first\s*).*/\1$firstFqdn/" ${cfg_install_dir}/taos.cfg - - # Get the second FQDN - echo - echo -e -n "${GREEN}Enter the FQDN of another node in cluster${NC} OR ${GREEN}leave it blank to skip${NC}: " - read secondFqdn - while true; do - if [ ! -z "$secondFqdn" ]; then - #if [[ $secondFqdn == $FQDN_PATTERN ]]; then - # Write the second FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(second\s*).*/\1$secondFqdn/" ${cfg_install_dir}/taos.cfg - break - #else - # read -p "Please enter the correct FQDN: " secondFqdn - #fi - else - break - fi - done - + if [ ! -z "$firstEp" ]; then + # check the format of the firstEp + #if [[ $firstEp == $FQDN_PATTERN ]]; then + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg break #else - # read -p "Please enter the correct FQDN: " firstFqdn + # read -p "Please enter the correct FQDN:port: " firstEp #fi else break @@ -584,7 +565,7 @@ function update_TDengine() { install_service install_config - if [ "$verMode" == "cluster" ]; then + if [ "$verMode" == "cluster" ]; then # Check if openresty is installed openresty_work=false @@ -597,7 +578,7 @@ function update_TDengine() { echo -e "\033[44;31;5mNginx for TDengine does not work! Please try again!\033[0m" fi fi - fi + fi echo echo -e "\033[44;32;1mTDengine is updated successfully!${NC}" @@ -678,8 +659,8 @@ function install_TDengine() { install_config # Ask if to start the service - echo - echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" + #echo + #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then @@ -700,8 +681,11 @@ function install_TDengine() { echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" fi + echo + echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp ${GREEN_DARK} to login into cluster, then execute ${NC}: create dnode 'selfEp'; ${GREEN_DARK}in TAOS shell to add this new node into the clsuter${NC}" echo echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" + echo else # Only install client install_bin install_config diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index ac65237319..90ca14277b 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -104,48 +104,48 @@ elif [ $NODE -eq 8 ]; then NODE=7800 fi -echo " " >> $TAOS_CFG -echo "first ${HOSTNAME}:7100" >> $TAOS_CFG -echo "second ${HOSTNAME}:7200" >> $TAOS_CFG -echo "serverPort ${NODE}" >> $TAOS_CFG -echo "dataDir $DATA_DIR" >> $TAOS_CFG -echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 135" >> $TAOS_CFG -echo "sdbDebugFlag 135" >> $TAOS_CFG -echo "dDebugFlag 135" >> $TAOS_CFG -echo "vDebugFlag 135" >> $TAOS_CFG -echo "tsdbDebugFlag 135" >> $TAOS_CFG -echo "cDebugFlag 135" >> $TAOS_CFG -echo "jnidebugFlag 135" >> $TAOS_CFG -echo "odbcdebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 143" >> $TAOS_CFG -echo "monitorDebugFlag 135" >> $TAOS_CFG -echo "mqttDebugFlag 135" >> $TAOS_CFG -echo "qdebugFlag 135" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG -echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 135" >> $TAOS_CFG -echo "sdebugFlag 135" >> $TAOS_CFG -echo "wdebugFlag 135" >> $TAOS_CFG -echo "monitor 0" >> $TAOS_CFG -echo "monitorInterval 1" >> $TAOS_CFG -echo "http 0" >> $TAOS_CFG -echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG -echo "defaultPass taosdata" >> $TAOS_CFG -echo "numOfLogLines 20000000" >> $TAOS_CFG -echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG -echo "clog 2" >> $TAOS_CFG -#echo "cache 1" >> $TAOS_CFG -#echo "block 2" >> $TAOS_CFG -echo "statusInterval 1" >> $TAOS_CFG -echo "maxVgroupsPerDb 4" >> $TAOS_CFG -echo "minTablesPerVnode 4" >> $TAOS_CFG -echo "maxTablesPerVnode 1000" >> $TAOS_CFG -echo "tableIncStepPerVnode 10000" >> $TAOS_CFG -echo "asyncLog 0" >> $TAOS_CFG -echo "numOfMnodes 1" >> $TAOS_CFG -echo "locale en_US.UTF-8" >> $TAOS_CFG -echo "fsync 0" >> $TAOS_CFG - +echo " " >> $TAOS_CFG +echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG +echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG +echo "serverPort ${NODE}" >> $TAOS_CFG +echo "dataDir $DATA_DIR" >> $TAOS_CFG +echo "logDir $LOG_DIR" >> $TAOS_CFG +echo "debugFlag 0" >> $TAOS_CFG +echo "mDebugFlag 135" >> $TAOS_CFG +echo "sdbDebugFlag 135" >> $TAOS_CFG +echo "dDebugFlag 135" >> $TAOS_CFG +echo "vDebugFlag 135" >> $TAOS_CFG +echo "tsdbDebugFlag 135" >> $TAOS_CFG +echo "cDebugFlag 135" >> $TAOS_CFG +echo "jnidebugFlag 135" >> $TAOS_CFG +echo "odbcdebugFlag 135" >> $TAOS_CFG +echo "httpDebugFlag 143" >> $TAOS_CFG +echo "monitorDebugFlag 135" >> $TAOS_CFG +echo "mqttDebugFlag 135" >> $TAOS_CFG +echo "qdebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "tmrDebugFlag 131" >> $TAOS_CFG +echo "udebugFlag 135" >> $TAOS_CFG +echo "sdebugFlag 135" >> $TAOS_CFG +echo "wdebugFlag 135" >> $TAOS_CFG +echo "monitor 0" >> $TAOS_CFG +echo "monitorInterval 1" >> $TAOS_CFG +echo "http 0" >> $TAOS_CFG +echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG +echo "defaultPass taosdata" >> $TAOS_CFG +echo "numOfLogLines 20000000" >> $TAOS_CFG +echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "clog 2" >> $TAOS_CFG +#echo "cache 1" >> $TAOS_CFG +#echo "block 2" >> $TAOS_CFG +echo "statusInterval 1" >> $TAOS_CFG +echo "maxVgroupsPerDb 4" >> $TAOS_CFG +echo "minTablesPerVnode 4" >> $TAOS_CFG +echo "maxTablesPerVnode 1000" >> $TAOS_CFG +echo "tableIncStepPerVnode 10000" >> $TAOS_CFG +echo "asyncLog 0" >> $TAOS_CFG +echo "numOfMnodes 1" >> $TAOS_CFG +echo "locale en_US.UTF-8" >> $TAOS_CFG +echo "fsync 0" >> $TAOS_CFG +echo " " >> $TAOS_CFG diff --git a/tests/script/test.sh b/tests/script/test.sh index d9a738749f..96e4ffe689 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -101,28 +101,29 @@ TAOS_FLAG=$PRG_DIR/flag HOSTNAME=`hostname -f` -echo " " >> $TAOS_CFG -echo "first ${HOSTNAME}:7100" >> $TAOS_CFG -echo "second ${HOSTNAME}:7200" >> $TAOS_CFG -echo "serverPort 7100" >> $TAOS_CFG -echo "dataDir $DATA_DIR" >> $TAOS_CFG -echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "scriptDir ${CODE_DIR}/../script">> $TAOS_CFG -echo "numOfLogLines 100000000" >> $TAOS_CFG -echo "dDebugFlag 135" >> $TAOS_CFG -echo "mDebugFlag 135" >> $TAOS_CFG -echo "sdbDebugFlag 135" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG -echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "cDebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 135" >> $TAOS_CFG -echo "monitorDebugFlag 135" >> $TAOS_CFG -echo "udebugFlag 135" >> $TAOS_CFG -echo "tablemetakeeptimer 5" >> $TAOS_CFG -echo "wal 0" >> $TAOS_CFG -echo "asyncLog 0" >> $TAOS_CFG -echo "locale en_US.UTF-8" >> $TAOS_CFG -echo " " >> $TAOS_CFG +echo " " >> $TAOS_CFG +echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG +echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG +echo "serverPort 7100" >> $TAOS_CFG +echo "dataDir $DATA_DIR" >> $TAOS_CFG +echo "logDir $LOG_DIR" >> $TAOS_CFG +echo "scriptDir ${CODE_DIR}/../script" >> $TAOS_CFG +echo "numOfLogLines 100000000" >> $TAOS_CFG +echo "dDebugFlag 135" >> $TAOS_CFG +echo "mDebugFlag 135" >> $TAOS_CFG +echo "sdbDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "tmrDebugFlag 131" >> $TAOS_CFG +echo "cDebugFlag 135" >> $TAOS_CFG +echo "httpDebugFlag 135" >> $TAOS_CFG +echo "monitorDebugFlag 135" >> $TAOS_CFG +echo "udebugFlag 135" >> $TAOS_CFG +echo "tablemetakeeptimer 5" >> $TAOS_CFG +echo "wal 0" >> $TAOS_CFG +echo "asyncLog 0" >> $TAOS_CFG +echo "locale en_US.UTF-8" >> $TAOS_CFG +echo "enableCoreFile 1" >> $TAOS_CFG +echo " " >> $TAOS_CFG ulimit -n 600000 ulimit -c unlimited From 92cd04847433c03d9fbe2ce4544181f324abdbaa Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 30 Jul 2020 11:12:36 +0800 Subject: [PATCH 04/18] [modify install] --- packaging/tools/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 2191667242..47a8febe35 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -682,7 +682,7 @@ function install_TDengine() { fi echo - echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp ${GREEN_DARK} to login into cluster, then execute ${NC}: create dnode 'selfEp'; ${GREEN_DARK}in TAOS shell to add this new node into the clsuter${NC}" + echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp ${GREEN_DARK} to login into cluster, then execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}in TAOS shell to add this new node into the clsuter${NC}" echo echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo From 9581b7a95c6e3ccded9867d5d0190c208c52e77b Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 30 Jul 2020 14:43:37 +0800 Subject: [PATCH 05/18] TD-1030: fix JDBC url parser issue --- .../java/com/taosdata/jdbc/TSDBDriver.java | 66 ++++++++++++------- .../com/taosdata/jdbc/TSDBDriverTest.java | 47 +++++++++++++ 2 files changed, 88 insertions(+), 25 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index f4c9ec765a..02d642d643 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -260,50 +260,66 @@ public class TSDBDriver implements java.sql.Driver { if (url.indexOf("?") != -1) { String dbName = url.substring(0, url.indexOf("?")); urlProps.setProperty(PROPERTY_KEY_DBNAME, dbName); - url = url.trim().substring(1); + url = url.trim().substring(url.indexOf("?") + 1); } else { // without user & password so return - String dbName = url.trim(); - urlProps.setProperty(PROPERTY_KEY_DBNAME, dbName); + if(!url.trim().isEmpty()) { + String dbName = url.trim(); + urlProps.setProperty(PROPERTY_KEY_DBNAME, dbName); + } this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty("user")); return urlProps; } - String[] queryStrings = url.trim().split("&"); String user = ""; + + if (url.indexOf("&") == -1) { + String[] kvPair = url.trim().split("="); + if (kvPair.length == 2) { + setPropertyValue(urlProps, kvPair); + return urlProps; + } + } + + String[] queryStrings = url.trim().split("&"); for (String queryStr : queryStrings) { String[] kvPair = queryStr.trim().split("="); if (kvPair.length < 2){ continue; } - switch (kvPair[0].toLowerCase()) { - case PROPERTY_KEY_USER: - urlProps.setProperty(PROPERTY_KEY_USER, kvPair[1]); - user = kvPair[1]; - break; - case PROPERTY_KEY_PASSWORD: - urlProps.setProperty(PROPERTY_KEY_PASSWORD, kvPair[1]); - break; - case PROPERTY_KEY_TIME_ZONE: - urlProps.setProperty(PROPERTY_KEY_TIME_ZONE, kvPair[1]); - break; - case PROPERTY_KEY_LOCALE: - urlProps.setProperty(PROPERTY_KEY_LOCALE, kvPair[1]); - break; - case PROPERTY_KEY_CHARSET: - urlProps.setProperty(PROPERTY_KEY_CHARSET, kvPair[1]); - break; - case PROPERTY_KEY_CONFIG_DIR: - urlProps.setProperty(PROPERTY_KEY_CONFIG_DIR, kvPair[1]); - break; - } + setPropertyValue(urlProps, kvPair); } + user = urlProps.getProperty(PROPERTY_KEY_USER).toString(); this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, user); return urlProps; } + public void setPropertyValue(Properties property, String[] keyValuePair) { + switch (keyValuePair[0].toLowerCase()) { + case PROPERTY_KEY_USER: + property.setProperty(PROPERTY_KEY_USER, keyValuePair[1]); + break; + case PROPERTY_KEY_PASSWORD: + property.setProperty(PROPERTY_KEY_PASSWORD, keyValuePair[1]); + break; + case PROPERTY_KEY_TIME_ZONE: + property.setProperty(PROPERTY_KEY_TIME_ZONE, keyValuePair[1]); + break; + case PROPERTY_KEY_LOCALE: + property.setProperty(PROPERTY_KEY_LOCALE, keyValuePair[1]); + break; + case PROPERTY_KEY_CHARSET: + property.setProperty(PROPERTY_KEY_CHARSET, keyValuePair[1]); + break; + case PROPERTY_KEY_CONFIG_DIR: + property.setProperty(PROPERTY_KEY_CONFIG_DIR, keyValuePair[1]); + break; + } + } + + public int getMajorVersion() { return 1; } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java new file mode 100644 index 0000000000..9ea5a431a5 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -0,0 +1,47 @@ +package com.taosdata.jdbc; + +import org.junit.Test; + +import java.sql.SQLException; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class TSDBDriverTest { + + @Test + public void urlParserTest() throws SQLException { + TSDBDriver driver = new TSDBDriver(); + String url = "jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password"; + + Properties properties = new Properties(); + driver.parseURL(url, properties); + assertEquals(properties.get("host"), "127.0.0.1"); + assertEquals(properties.get("port"), "0"); + assertEquals(properties.get("dbname"), "db"); + assertEquals(properties.get("user"), "root"); + assertEquals(properties.get("password"), "your_password"); + + url = "jdbc:TSDB://127.0.0.1:0/log?charset=UTF-8"; + properties = new Properties(); + driver.parseURL(url, properties); + assertEquals(properties.get("host"), "127.0.0.1"); + assertEquals(properties.get("port"), "0"); + assertEquals(properties.get("dbname"), "log"); + assertEquals(properties.get("charset"), "UTF-8"); + + url = "jdbc:TSDB://127.0.0.1:0/"; + properties = new Properties(); + driver.parseURL(url, properties); + assertEquals(properties.get("host"), "127.0.0.1"); + assertEquals(properties.get("port"), "0"); + assertEquals(properties.get("dbname"), null); + + url = "jdbc:TSDB://127.0.0.1:0/db"; + properties = new Properties(); + driver.parseURL(url, properties); + assertEquals(properties.get("host"), "127.0.0.1"); + assertEquals(properties.get("port"), "0"); + assertEquals(properties.get("dbname"), "db"); + } +} \ No newline at end of file From 3cc384a03c68787763093ad93d38f343af50154e Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 30 Jul 2020 14:49:18 +0800 Subject: [PATCH 06/18] [fix bug] --- src/common/src/tglobal.c | 4 ++-- src/kit/shell/src/shellEngine.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 79ea2b2d46..1a510299c1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -297,7 +297,7 @@ static void doInitGlobalConfig() { SGlobalCfg cfg = {0}; // ip address - cfg.option = "first"; + cfg.option = "firstEp"; cfg.ptr = tsFirst; cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; @@ -307,7 +307,7 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "second"; + cfg.option = "secondEp"; cfg.ptr = tsSecond; cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 01dcc73ef6..616a3bfd7f 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -73,6 +73,8 @@ TAOS *shellInit(SShellArguments *args) { // Connect to the database. TAOS *con = taos_connect(args->host, args->user, args->password, args->database, args->port); if (con == NULL) { + printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); + fflush(stdout); return con; } From fb9146ad76b17b23df7bca446565d6e94a03df2f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Jul 2020 15:37:44 +0800 Subject: [PATCH 07/18] change alter dnode ep to id. [TD-1031] --- tests/pytest/client/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/client/client.py b/tests/pytest/client/client.py index 6ac12c6795..36c06ac09d 100644 --- a/tests/pytest/client/client.py +++ b/tests/pytest/client/client.py @@ -42,8 +42,8 @@ class TDTestCase: ret = tdSql.query('show dnodes') - ret = tdSql.execute('alter dnode "%s" debugFlag 135' % tdSql.getData(0,1)) - tdLog.info('alter dnode "%s" debugFlag 135 -> ret: %d' % (tdSql.getData(0, 1), ret)) + ret = tdSql.execute('alter dnode "%s" debugFlag 135' % tdSql.getData(0,0)) + tdLog.info('alter dnode "%s" debugFlag 135 -> ret: %d' % (tdSql.getData(0, 0), ret)) def stop(self): tdSql.close() From 4c024aab37f8ec4b5e67d1e20acc8ebb5c50e1d8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Jul 2020 07:43:54 +0000 Subject: [PATCH 08/18] Ensure that the htpp context is valid before release --- src/mnode/src/mnodeProfile.c | 2 +- src/plugins/http/src/httpContext.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 30a292f522..ff4ddf1b2a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -120,7 +120,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po } if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { - mError("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user, + mDebug("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user, taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port); taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); return NULL; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index ca65f65608..527e56ea0e 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 3, true, httpDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; @@ -108,7 +108,7 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 3); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 5); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); @@ -133,13 +133,22 @@ HttpContext *httpGetContext(void *ptr) { } void httpReleaseContext(HttpContext *pContext) { + // Ensure that the context is valid before release + HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &pContext, sizeof(HttpContext *)); + if (ppContext == NULL) { + httpError("context:%p, is already released", pContext); + return; + } + int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); assert(refCount >= 0); + assert(ppContext == pContext->ppContext); - HttpContext **ppContext = pContext->ppContext; httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount); if (tsHttpServer.contextCache != NULL) { + // and release context twice + taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); } else { httpDebug("context:%p, won't be destroyed for cache is already released", pContext); From 0dcee21a19216041e20b12e50375b12f2582c592 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Jul 2020 15:56:13 +0800 Subject: [PATCH 09/18] return success --- src/tsdb/src/tsdbMeta.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index ddb935b7e5..d389a96fee 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -69,9 +69,9 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) { if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]), - TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid])); - return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), + TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid])); + return 0; } else { tsdbError("vgId:%d table %s at tid %d uid %" PRIu64 " exists, replace it with new table, this can be not reasonable", From 6c7eab55e5064f9d9a0c5a3e7157c677610223ac Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Jul 2020 16:50:32 +0800 Subject: [PATCH 10/18] fix stable/query-after-reset.py [TD-1032] --- tests/pytest/stable/query_after_reset.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/pytest/stable/query_after_reset.py b/tests/pytest/stable/query_after_reset.py index 61f6558b83..0e34fade5f 100644 --- a/tests/pytest/stable/query_after_reset.py +++ b/tests/pytest/stable/query_after_reset.py @@ -72,8 +72,12 @@ class Test: def query_stable(self): tdLog.info("query super table") - tdSql.query("select * from st") - tdSql.checkRows(1) + try: + tdSql.query("select * from st") + except Exception as e: + tdLog.info("Exception catched: %s" % repr(e)) + if ('mnode invalid table name' not in repr(e)): + raise Exception(repr(e)) def create_stable(self): tdLog.info("create a super table and sub-table and insert data") @@ -162,10 +166,15 @@ class TDTestCase: 12: test.stop_database, } + tdLog.info("create stable") switch.get(4, lambda: "ERROR")() + tdLog.info("stop database") switch.get(12, lambda: "ERROR")() + tdLog.info("delete datafiles") switch.get(10, lambda: "ERROR")() + tdLog.info("restart database") switch.get(5, lambda: "ERROR")() + tdLog.info("query stable") switch.get(11, lambda: "ERROR")() def stop(self): From 61635dc03879aaa8fdd53a412e6a986cb06beb2a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Jul 2020 09:09:15 +0000 Subject: [PATCH 11/18] context log --- src/plugins/http/src/httpContext.c | 9 +++++---- src/plugins/http/src/httpServer.c | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 527e56ea0e..8c4cd743c8 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -52,13 +52,14 @@ static void httpDestroyContext(void *data) { // avoid double free httpFreeJsonBuf(pContext); httpFreeMultiCmds(pContext); - - httpDebug("context:%p, is destroyed, refCount:%d data:%p", pContext, pContext->refCount, data); + + httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount, + data, pContext->pThread->label, pContext->pThread->numOfContexts); tfree(pContext); } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 3, true, httpDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; @@ -108,7 +109,7 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 5); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 3); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 177d447f10..6806a3524c 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -299,12 +299,14 @@ static void *httpAcceptHttpConnection(void *arg) { totalFds += pServer->pThreads[i].numOfContexts; } +#if 0 if (totalFds > tsHttpCacheSessions * 100) { httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions); taosCloseSocket(connFd); continue; } +#endif taosKeepTcpAlive(connFd); taosSetNonblocking(connFd, 1); From 5dd0158eda3b57b00c392c68a98b731230eef312 Mon Sep 17 00:00:00 2001 From: localvar Date: Tue, 7 Apr 2020 11:07:34 +0800 Subject: [PATCH 12/18] td-40: move 'alert' project into tdengine --- alert/.gitignore | 21 + alert/README.md | 180 ++++++++ alert/README_cn.md | 177 ++++++++ alert/app/app.go | 175 ++++++++ alert/app/expr/expr.go | 792 +++++++++++++++++++++++++++++++++++ alert/app/expr/expr_test.go | 198 +++++++++ alert/app/expr/funcs.go | 198 +++++++++ alert/app/expr/funcs_test.go | 329 +++++++++++++++ alert/app/route.go | 44 ++ alert/app/rule.go | 627 +++++++++++++++++++++++++++ alert/app/rule_test.go | 81 ++++ alert/cmd/alert/alert.cfg | 13 + alert/cmd/alert/main.go | 183 ++++++++ alert/cmd/alert/rule.json | 13 + alert/go.mod | 12 + alert/go.sum | 80 ++++ alert/models/models.go | 131 ++++++ alert/models/rule.go | 75 ++++ alert/release.sh | 45 ++ alert/test/prepare.sim | 17 + alert/test/rule.json | 13 + alert/test/step1.sim | 10 + alert/test/step2.sim | 5 + alert/test/step3.sim | 12 + alert/test/test.sh | 91 ++++ alert/utils/config.go | 41 ++ alert/utils/log/log.go | 94 +++++ 27 files changed, 3657 insertions(+) create mode 100644 alert/.gitignore create mode 100644 alert/README.md create mode 100644 alert/README_cn.md create mode 100644 alert/app/app.go create mode 100644 alert/app/expr/expr.go create mode 100644 alert/app/expr/expr_test.go create mode 100644 alert/app/expr/funcs.go create mode 100644 alert/app/expr/funcs_test.go create mode 100644 alert/app/route.go create mode 100644 alert/app/rule.go create mode 100644 alert/app/rule_test.go create mode 100644 alert/cmd/alert/alert.cfg create mode 100644 alert/cmd/alert/main.go create mode 100644 alert/cmd/alert/rule.json create mode 100644 alert/go.mod create mode 100644 alert/go.sum create mode 100644 alert/models/models.go create mode 100644 alert/models/rule.go create mode 100755 alert/release.sh create mode 100644 alert/test/prepare.sim create mode 100644 alert/test/rule.json create mode 100644 alert/test/step1.sim create mode 100644 alert/test/step2.sim create mode 100644 alert/test/step3.sim create mode 100755 alert/test/test.sh create mode 100644 alert/utils/config.go create mode 100644 alert/utils/log/log.go diff --git a/alert/.gitignore b/alert/.gitignore new file mode 100644 index 0000000000..332fefb3ae --- /dev/null +++ b/alert/.gitignore @@ -0,0 +1,21 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +vendor/ + +# Project specific files +cmd/alert/alert +cmd/alert/alert.log +*.db +*.gz \ No newline at end of file diff --git a/alert/README.md b/alert/README.md new file mode 100644 index 0000000000..aa3738a394 --- /dev/null +++ b/alert/README.md @@ -0,0 +1,180 @@ +# Alert [DRAFT] + +The Alert application reads data from [TDEngine](https://www.taosdata.com/), calculating according to predefined rules to generate alerts, and pushes alerts to downstream applications like [AlertManager](https://github.com/prometheus/alertmanager). + +## Install + +### From Binary + +TODO: 安装包具体位置 + +Precompiled binaries is available at [taosdata website](https://www.taosdata.com/), please download and unpack it by below shell command. + +``` +$ tar -xzf alert-$version-$OS-$ARCH.tar.gz +``` + +### From Source Code + +Two prerequisites are required to install from source. + +1. TDEngine server or client must be installed. +2. Latest [Go](https://golang.org) language must be installed. + +When these two prerequisites are ready, please follow steps below to build the application: + +``` +$ mkdir taosdata +$ cd taosdata +$ git clone https://github.com/taosdata/tdengine.git +$ cd tdengine/alert/cmd/alert +$ go build +``` + +If `go build` fails because some of the dependency packages cannot be downloaded, please follow steps in [goproxy.io](https://goproxy.io) to configure `GOPROXY` and try `go build` again. + +## Configure + +The configuration file format of Alert application is standard `json`, below is its default content, please revise according to actual scenario. + +```json +{ + "port": 8100, + "database": "file:alert.db", + "tdengine": "root:taosdata@/tcp(127.0.0.1:0)/", + "log": { + "level": "production", + "path": "alert.log" + }, + "receivers": { + "alertManager": "http://127.0.0.1:9093/api/v1/alerts", + "console": true + } +} +``` + +The use of each configuration item is: + +* **port**: This is the `http` service port which enables other application to manage rules by `restful API`. +* **database**: rules are stored in a `sqlite` database, this is the path of the database file (if the file does not exist, the alert application creates it automatically). +* **tdengine**: connection string of `TDEngine` server, note in most cases the database information should be put in a rule, thus it should NOT be included here. +* **log > level**: log level, could be `production` or `debug`. +* **log > path**: log output file path. +* **receivers > alertManager**: the alert application pushes alerts to `AlertManager` at this URL. +* **receivers > console**: print out alerts to console (stdout) or not. + +When the configruation file is ready, the alert application can be started with below command (`alert.cfg` is the path of the configuration file): + +``` +$ ./alert -cfg alert.cfg +``` + +## Prepare an alert rule + +From technical aspect, an alert could be defined as: query and filter recent data from `TDEngine`, and calculating out a boolean value from these data according to a formula, and trigger an alert if the boolean value last for a certain duration. + +This is a rule example in `json` format: + +```json +{ + "name": "rule1", + "sql": "select sum(col1) as sumCol1 from test.meters where ts > now - 1h group by areaid", + "expr": "sumCol1 > 10", + "for": "10m", + "period": "1m", + "labels": { + "ruleName": "rule1" + }, + "annotations": { + "summary": "sum of rule {{$labels.ruleName}} of area {{$values.areaid}} is {{$values.sumCol1}}" + } +} +``` + +The fields of the rule is explained below: + +* **name**: the name of the rule, must be unique. +* **sql**: this is the `sql` statement used to query data from `TDEngine`, columns of the query result are used in later processing, so please give the column an alias if aggregation functions are used. +* **expr**: an expression whose result is a boolean value, arithmatic and logical calculations can be included in the expression, and builtin functions (see below) are also supported. Alerts are only triggered when the expression evaluates to `true`. +* **for**: this item is a duration which default value is zero second. when `expr` evaluates to `true` and last at least this duration, an alert is triggered. +* **period**: the interval for the alert application to check the rule, default is 1 minute. +* **labels**: a label list, labels are used to generate alert information. note if the `sql` statement includes a `group by` clause, the `group by` columns are inserted into this list automatically. +* **annotations**: the template of alert information which is in [go template](https://golang.org/pkg/text/template) syntax, labels can be referenced by `$labels.