From 89d87300d3b725f54c0cfe1621b5a48a2bb041bd Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 1 Sep 2020 18:04:05 +0800 Subject: [PATCH 1/7] fix td-804 --- src/client/inc/tsclient.h | 2 +- src/client/src/tscAsync.c | 2 +- src/client/src/tscParseInsert.c | 24 ++++++++++++++++++++---- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 12 ++++++------ 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 900d4955e9..53266d2c97 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -416,7 +416,7 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscDestroyResPointerInfo(SSqlRes *pRes); -void tscResetSqlCmdObj(SSqlCmd *pCmd); +void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); /** * free query result of the sql object diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 4643d255dc..3d04cae1c4 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -482,7 +482,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_SELECT) { tscDebug("%p redo parse sql string and proceed", pSql); pCmd->parseFinished = false; - tscResetSqlCmdObj(pCmd); + tscResetSqlCmdObj(pCmd, false); code = tsParseSql(pSql, true); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 47bfe0fcdc..5e3fdbc5e7 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1328,14 +1328,30 @@ int tsParseSql(SSqlObj *pSql, bool initial) { pSql->fp = (void(*)())tscHandleMultivnodeInsert; } - if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { - return ret; + for (int i = 0; true; i++) { + if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { + return ret; + } + + // make a backup as tsParseInsertSql may modify the string + char* sqlstr = strdup(pSql->sqlstr); + ret = tsParseInsertSql(pSql); + if (sqlstr == NULL || i >= 2 || ret != TSDB_CODE_TSC_INVALID_SQL) { + free(sqlstr); + break; + } + tscResetSqlCmdObj(pCmd, true); + pCmd->pQueryInfo = NULL; + free(pSql->sqlstr); + pSql->sqlstr = sqlstr; } - - ret = tsParseInsertSql(pSql); } else { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); + if (ret == TSDB_CODE_TSC_INVALID_SQL) { + tscResetSqlCmdObj(pCmd, true); + ret = tscToSQLCmd(pSql, &SQLInfo); + } SQLInfoDestroy(&SQLInfo); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f63923e046..aeeba18a95 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -817,7 +817,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { // must before clean the sqlcmd object - tscResetSqlCmdObj(&pSql->cmd); + tscResetSqlCmdObj(&pSql->cmd, false); SSqlCmd *pCmd = &pSql->cmd; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 49f7c91397..f7e8c6a5c9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -33,7 +33,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); - SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { +SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { return NULL; } @@ -296,7 +296,7 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -static void tscFreeQueryInfo(SSqlCmd* pCmd) { +static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } @@ -306,7 +306,7 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); + clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); taosTFree(pQueryInfo); } @@ -314,7 +314,7 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd) { taosTFree(pCmd->pQueryInfo); } -void tscResetSqlCmdObj(SSqlCmd* pCmd) { +void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -328,7 +328,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd); + tscFreeQueryInfo(pCmd, removeFromCache); } void tscFreeSqlResult(SSqlObj* pSql) { @@ -366,7 +366,7 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->pSubs); pSql->numOfSubs = 0; - tscResetSqlCmdObj(pCmd); + tscResetSqlCmdObj(pCmd, false); } void tscFreeSqlObj(SSqlObj* pSql) { From 79833f924e9195a0ab18ee2aeb7b79b5682091ce Mon Sep 17 00:00:00 2001 From: root Date: Wed, 2 Sep 2020 17:42:53 +0800 Subject: [PATCH 2/7] minor changes --- tests/pytest/crash_gen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 7588e03e17..768e401be8 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -2350,7 +2350,7 @@ class ServiceManagerThread: self._thread2.start() # wait for service to start - for i in range(0, 10): + for i in range(0, 100): time.sleep(1.0) # self.procIpcBatch() # don't pump message during start up print("_zz_", end="", flush=True) @@ -2358,7 +2358,7 @@ class ServiceManagerThread: logger.info("[] TDengine service READY to process requests") return # now we've started # TODO: handle this better? - self.procIpcBatch(20, True) # display output before cronking out, trim to last 20 msgs, force output + self.procIpcBatch(100, True) # display output before cronking out, trim to last 20 msgs, force output raise RuntimeError("TDengine service did not start successfully") def stop(self): From d18b1e9339fb66ad0d9c4eaaab2018f09c5e2809 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 2 Sep 2020 19:00:52 +0000 Subject: [PATCH 3/7] Fixed port number for crash_gen tool --- tests/pytest/crash_gen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 7588e03e17..8c8f44b016 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -693,7 +693,7 @@ class DbConnRest(DbConn): def __init__(self): super().__init__() self._type = self.TYPE_REST - self._url = "http://localhost:6020/rest/sql" # fixed for now + self._url = "http://localhost:6041/rest/sql" # fixed for now self._result = None def openByType(self): # Open connection @@ -2768,7 +2768,7 @@ class MainExec: try: ret = self._clientMgr.run(self._svcMgr) # stop TAOS service inside except requests.exceptions.ConnectionError as err: - logger.warning("Failed to open REST connection to DB") + logger.warning("Failed to open REST connection to DB: {}".format(err.getMessage())) # don't raise return ret From 89b3b7912494ac61355ae10b3b3adcbea584a215 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 3 Sep 2020 00:25:03 +0000 Subject: [PATCH 4/7] Added last_row() and spread() functions to crash_gen tool --- tests/pytest/crash_gen.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 0ecc227402..c0a8fd1f00 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1306,6 +1306,7 @@ class DbManager(): "Cannot establish DB connection, please re-run script without parameter, and follow the instructions.") sys.exit(2) else: + print("Failed to connect to DB, errno = {}, msg: {}".format(Helper.convertErrno(err.errno), err.msg)) raise except BaseException: print("[=] Unexpected exception") @@ -1910,10 +1911,19 @@ class TaskReadData(StateTransitionTask): # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable 'sum(speed)', 'stddev(speed)', + # SELECTOR functions 'min(speed)', 'max(speed)', 'first(speed)', - 'last(speed)']) # TODO: add more from 'top' + 'last(speed)', + # 'top(speed)', # TODO: not supported? + # 'bottom(speed)', # TODO: not supported? + # 'percentile(speed, 10)', # TODO: TD-1316 + 'last_row(speed)', + # Transformation Functions + # 'diff(speed)', # TODO: no supported?! + 'spread(speed)' + ]) # TODO: add more from 'top' filterExpr = Dice.choice([ # TODO: add various kind of WHERE conditions None ]) From 9bda0b39a0f589937ae9f665dd7b01fdf1e6b5e3 Mon Sep 17 00:00:00 2001 From: Bo Xiao <69349626+boxiaobj@users.noreply.github.com> Date: Thu, 3 Sep 2020 08:53:16 +0800 Subject: [PATCH 5/7] Modify wallevel description in section 2 of Master/Slave vnode writing process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改了Master/Slave vnode 的第二段关于wallevel和日志文件WAL的描述：如果系统配置参数walLevel大于0，vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2，而且fsync设置为0，TDengine还将WAL数据立即落盘，以保证即使宕机，也能从数据库日志文件中恢复数据，避免数据的丢失； --- documentation20/webdocs/markdowndocs/architecture-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index 7ab4b5d096..a279875649 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -162,7 +162,7 @@ Master Vnode遵循下面的写入流程：
图 3 TDengine Master写入流程
1. Master vnode收到应用的数据插入请求，验证OK，进入下一步； -2. 如果系统配置参数walLevel打开（设置为2），vnode将把该请求的原始数据包写入数据库日志文件WAL，以保证TDengine能够在断电等因素导致的服务重启时从数据库日志文件中恢复数据，避免数据的丢失； +2. 如果系统配置参数walLevel大于0，vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2，而且fsync设置为0，TDengine还将WAL数据立即落盘，以保证即使宕机，也能从数据库日志文件中恢复数据，避免数据的丢失； 3. 如果有多个副本，vnode将把数据包转发给同一虚拟节点组内slave vnodes, 该转发包带有数据的版本号(version)； 4. 写入内存，并加记录加入到skip list； 5. Master vnode返回确认信息给应用，表示写入成功。 @@ -174,7 +174,7 @@ Master Vnode遵循下面的写入流程：
图 4 TDengine Slave写入流程
1. Slave vnode收到Master vnode转发了的数据插入请求。 -2. 如果系统配置参数walLevl设置为2，vnode将把该请求的原始数据包写入日志(WAL)； +2. 如果系统配置参数walLevel大于0，vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2，而且fsync设置为0，TDengine还将WAL数据立即落盘，以保证即使宕机，也能从数据库日志文件中恢复数据，避免数据的丢失； 3. 写入内存，更新内存中的skip list。 与Master vnode相比，slave vnode不存在转发环节，也不存在回复确认环节，少了两步。但写内存与WAL是完全一样的。 From ab0d29f9a141442fbd8d7f74bf9abe8db83645a7 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 3 Sep 2020 09:43:35 +0800 Subject: [PATCH 6/7] td-804: pass all smoke test --- src/client/inc/tsclient.h | 2 +- src/client/src/tscParseInsert.c | 34 +++++++++++++++++++-------------- src/client/src/tscServer.c | 10 ++++------ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 53266d2c97..7691419c76 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -401,7 +401,7 @@ int tsParseSql(SSqlObj *pSql, bool initial); void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); -int tscRenewTableMeta(SSqlObj *pSql, char *tableId); +int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); void tscQueueAsyncRes(SSqlObj *pSql); void tscQueueAsyncError(void(*fp), void *param, int32_t code); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 5e3fdbc5e7..fb9acddd2a 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1327,34 +1327,40 @@ int tsParseSql(SSqlObj *pSql, bool initial) { pSql->fetchFp = pSql->fp; pSql->fp = (void(*)())tscHandleMultivnodeInsert; } - - for (int i = 0; true; i++) { - if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { - return ret; - } - // make a backup as tsParseInsertSql may modify the string - char* sqlstr = strdup(pSql->sqlstr); - ret = tsParseInsertSql(pSql); - if (sqlstr == NULL || i >= 2 || ret != TSDB_CODE_TSC_INVALID_SQL) { - free(sqlstr); - break; - } + if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { + return ret; + } + + // make a backup as tsParseInsertSql may modify the string + char* sqlstr = strdup(pSql->sqlstr); + ret = tsParseInsertSql(pSql); + if (sqlstr == NULL || pSql->retry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { + free(sqlstr); + } else { tscResetSqlCmdObj(pCmd, true); - pCmd->pQueryInfo = NULL; free(pSql->sqlstr); pSql->sqlstr = sqlstr; + pSql->retry++; + if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { + ret = tsParseInsertSql(pSql); + } } } else { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); - if (ret == TSDB_CODE_TSC_INVALID_SQL) { + if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->retry == 0 && SQLInfo.type == TSDB_SQL_NULL) { tscResetSqlCmdObj(pCmd, true); + pSql->retry++; ret = tscToSQLCmd(pSql, &SQLInfo); } SQLInfoDestroy(&SQLInfo); } + if (ret == TSDB_CODE_SUCCESS) { + pSql->retry = 0; + } + /* * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function, * so do NOT use pRes->code to determine if the getTableMeta function diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b36767dbb4..0b8fa83239 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -280,8 +280,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } } - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - int32_t cmd = pCmd->command; if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || @@ -306,7 +304,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { taosMsleep(duration); } - rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); + rpcMsg->code = tscRenewTableMeta(pSql, 0); // if there is an error occurring, proceed to the following error handling procedure. if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -2208,14 +2206,14 @@ int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create /** * retrieve table meta from mnode, and update the local table meta cache. * @param pSql sql object - * @param tableId table full name + * @param tableIndex table index * @return status code */ -int tscRenewTableMeta(SSqlObj *pSql, char *tableId) { +int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { From 19b0a7ccb70dca87ca8d86669c5f8a25413b6a2a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Sep 2020 17:05:07 +0800 Subject: [PATCH 7/7] [td-1322] --- src/query/src/qPercentile.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index c6eb836c61..3a484bf955 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -154,9 +154,14 @@ int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) { // todo refactor to more generic int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { - int32_t v = *(int32_t *)value; - int32_t index = -1; + int32_t v = 0; + switch(pBucket->type) { + case TSDB_DATA_TYPE_SMALLINT: v = *(int16_t*) value; break; + case TSDB_DATA_TYPE_TINYINT: v = *(int8_t*) value; break; + default: v = *(int32_t*) value;break; + } + int32_t index = -1; if (pBucket->range.iMaxVal == INT32_MIN) { /* * taking negative integer into consideration,