diff --git a/README.md b/README.md index 411dddb9f7..522fc0ebc1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine) [![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) diff --git a/cmake/install.inc b/cmake/install.inc index 7a92a396e3..0531d40048 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -13,7 +13,7 @@ ELSEIF (TD_WINDOWS) IF (NOT TD_GODLL) #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector) - #INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) diff --git a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md b/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md index d62a525e28..4e55f3c5e8 100644 --- a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md +++ b/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md @@ -11,7 +11,7 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/) ### 配置Grafana -TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafana目录下。 +TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。 以CentOS 7.2操作系统为例，将tdengine目录拷贝到/var/lib/grafana/plugins目录下，重新启动grafana即可。 diff --git a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md index 6ba0decfd3..a9a3dee432 100644 --- a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md +++ b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md @@ -1,7 +1,7 @@ # TDengine 2.0 错误码以及对应的十进制码 -| Code | bit | error code | 错误描述 | 十进制错误码 | +| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | |-----------------------| :---: | :---------: | :------------------------ | ---------------- | |TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647| |TSDB_CODE_RPC_AUTH_REQUIRED| 0 | 0x0002 | "Authentication required"| -2147483646| @@ -87,7 +87,7 @@ |TSDB_CODE_MND_INVALID_ACCT_OPTION| 0 | 0x0342 | "Invalid account options"| -2147482814| |TSDB_CODE_MND_USER_ALREADY_EXIST| 0 | 0x0350 | "User already exists"| -2147482800| |TSDB_CODE_MND_INVALID_USER |0 | 0x0351 | "Invalid user" |-2147482799| -|TSDB_CODE_MND_INVALID_USER_FORMAT| |0 |0x0352 |"Invalid user format" |-2147482798| +|TSDB_CODE_MND_INVALID_USER_FORMAT| 0 |0x0352 |"Invalid user format" |-2147482798| |TSDB_CODE_MND_INVALID_PASS_FORMAT| 0| 0x0353 | "Invalid password format"| -2147482797| |TSDB_CODE_MND_NO_USER_FROM_CONN| 0 | 0x0354 | "Can not get user from conn"| -2147482796| |TSDB_CODE_MND_TOO_MANY_USERS| 0 | 0x0355| "Too many users"| -2147482795| @@ -107,7 +107,7 @@ |TSDB_CODE_MND_DB_NOT_SELECTED| 0 | 0x0380 | "Database not specified or available"| -2147482752| |TSDB_CODE_MND_DB_ALREADY_EXIST| 0 | 0x0381 | "Database already exists"| -2147482751| |TSDB_CODE_MND_INVALID_DB_OPTION| 0 | 0x0382 | "Invalid database options"| -2147482750| -|TSDB_CODE_MND_INVALID_DB| |0 | 0x0383 | "Invalid database name"| -2147482749| +|TSDB_CODE_MND_INVALID_DB| 0 | 0x0383 | "Invalid database name"| -2147482749| |TSDB_CODE_MND_MONITOR_DB_FORBIDDEN| 0 | 0x0384 | "Cannot delete monitor database"| -2147482748| |TSDB_CODE_MND_TOO_MANY_DATABASES| 0| 0x0385 | "Too many databases for account"| -2147482747| |TSDB_CODE_MND_DB_IN_DROPPING| 0 | 0x0386| "Database not available" |-2147482746| diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 9b31b8fc6a..748b19069b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -69,17 +69,12 @@ typedef struct SJoinSupporter { SSubqueryState* pState; SSqlObj* pObj; // parent SqlObj int32_t subqueryIndex; // index of sub query - char intervalTimeUnit; - char slidingTimeUnit; - int64_t intervalTime; // interval time - int64_t slidingTime; // sliding time SLimitVal limit; // limit info - uint64_t uid; // query meter uid + uint64_t uid; // query table uid SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution SArray* exprList; SFieldInfo fieldsInfo; STagCond tagCond; - SSqlGroupbyExpr groupbyExpr; struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array FILE* f; // temporary file in order to create TSBuf char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 5f4a46ddad..74a0e8c11c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -29,6 +29,7 @@ extern "C" { #include "tglobal.h" #include "tsqlfunction.h" #include "tutil.h" +#include "tcache.h" #include "qExecutor.h" #include "qSqlparser.h" @@ -333,6 +334,7 @@ typedef struct STscObj { struct SSqlStream *streamList; void* pDnodeConn; pthread_mutex_t mutex; + T_REF_DECLARE(); } STscObj; typedef struct SSqlObj { @@ -359,6 +361,8 @@ typedef struct SSqlObj { uint16_t numOfSubs; struct SSqlObj **pSubs; struct SSqlObj * prev, *next; + + struct SSqlObj **self; } SSqlObj; typedef struct SSqlStream { @@ -413,7 +417,6 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); -void tscDestroyResPointerInfo(SSqlRes *pRes); void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); @@ -425,17 +428,19 @@ void tscFreeSqlResult(SSqlObj *pSql); /** * only free part of resources allocated during query. + * TODO remove it later * Note: this function is multi-thread safe. * @param pObj */ -void tscPartiallyFreeSqlObj(SSqlObj *pObj); +void tscPartiallyFreeSqlObj(SSqlObj *pSql); /** * free sql object, release allocated resource - * @param pObj Free metric/meta information, dynamically allocated payload, and - * response buffer, object itself + * @param pObj */ -void tscFreeSqlObj(SSqlObj *pObj); +void tscFreeSqlObj(SSqlObj *pSql); + +void tscFreeSqlObjInCache(void *pSql); void tscCloseTscObj(STscObj *pObj); @@ -451,9 +456,6 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) bool tscIsUpdateQuery(SSqlObj* pSql); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); -// todo remove this function. -bool tscResultsetFetchCompleted(TAOS_RES *result); - char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); @@ -502,7 +504,8 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern void * tscCacheHandle; +extern SCacheObj* tscMetaCache; +extern SCacheObj* tscObjCache; extern void * tscTmr; extern void * tscQhandle; extern int tscKeepConn[]; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index d07089539a..09610575f6 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -18,6 +18,7 @@ #include "tnote.h" #include "trpc.h" +#include "tcache.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscLocalMerge.h" @@ -40,6 +41,8 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) { + SSqlCmd* pCmd = &pSql->cmd; + pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; @@ -48,6 +51,11 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->fp = fp; pSql->fetchFp = fp; + uint64_t handle = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); + + T_REF_INC(pSql->pTscObj); + pSql->sqlstr = calloc(1, sqlLen + 1); if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); @@ -59,7 +67,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen); tscDebugL("%p SQL: %s", pSql, pSql->sqlstr); - pSql->cmd.curSql = pSql->sqlstr; + pCmd->curSql = pSql->sqlstr; int32_t code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; @@ -69,7 +77,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const tscQueueAsyncRes(pSql); return; } - + tscDoQuery(pSql); } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 4b31a8001f..1356b98c67 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -117,6 +117,10 @@ typedef struct SFirstLastInfo { typedef struct SFirstLastInfo SLastrowInfo; typedef struct SPercentileInfo { tMemBucket *pMemBucket; + int32_t stage; + double minval; + double maxval; + int64_t numOfElems; } SPercentileInfo; typedef struct STopBotInfo { @@ -302,7 +306,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_PERCT) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = (int16_t)sizeof(double); - *interBytes = (int16_t)sizeof(double); + *interBytes = (int16_t)sizeof(SPercentileInfo); } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string @@ -2428,12 +2432,14 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = - tMemBucketCreate(pCtx->inputBytes, pCtx->inputType); - + // in the first round, get the min-max value of all involved data + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = pResInfo->interResultBuf; + pInfo->minval = DBL_MAX; + pInfo->maxval = -DBL_MAX; + pInfo->numOfElems = 0; + return true; } @@ -2442,7 +2448,65 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SResultInfo * pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = pResInfo->interResultBuf; - + + // the first stage, only acquire the min/max value + if (pInfo->stage == 0) { + if (pCtx->preAggVals.isSet) { + if (pInfo->minval > pCtx->preAggVals.statis.min) { + pInfo->minval = pCtx->preAggVals.statis.min; + } + + if (pInfo->maxval < pCtx->preAggVals.statis.max) { + pInfo->maxval = pCtx->preAggVals.statis.max; + } + + pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); + } else { + for (int32_t i = 0; i < pCtx->size; ++i) { + char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { + continue; + } + + // TODO extract functions + double v = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = GET_INT8_VAL(data); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = GET_INT16_VAL(data); + break; + case TSDB_DATA_TYPE_BIGINT: + v = (double)(GET_INT64_VAL(data)); + break; + case TSDB_DATA_TYPE_FLOAT: + v = GET_FLOAT_VAL(data); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = GET_DOUBLE_VAL(data); + break; + default: + v = GET_INT32_VAL(data); + break; + } + + if (v < pInfo->minval) { + pInfo->minval = v; + } + + if (v > pInfo->maxval) { + pInfo->maxval = v; + } + + pInfo->numOfElems += 1; + } + } + + return; + } + + // the second stage, calculate the true percentile value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { @@ -2462,10 +2526,47 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; + + if (pInfo->stage == 0) { + // TODO extract functions + double v = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: + v = GET_INT8_VAL(pData); + break; + case TSDB_DATA_TYPE_SMALLINT: + v = GET_INT16_VAL(pData); + break; + case TSDB_DATA_TYPE_BIGINT: + v = (double)(GET_INT64_VAL(pData)); + break; + case TSDB_DATA_TYPE_FLOAT: + v = GET_FLOAT_VAL(pData); + break; + case TSDB_DATA_TYPE_DOUBLE: + v = GET_DOUBLE_VAL(pData); + break; + default: + v = GET_INT32_VAL(pData); + break; + } + + if (v < pInfo->minval) { + pInfo->minval = v; + } + + if (v > pInfo->maxval) { + pInfo->maxval = v; + } + + pInfo->numOfElems += 1; + return; + } + tMemBucketPut(pInfo->pMemBucket, pData, 1); SET_VAL(pCtx, 1, 1); @@ -2488,6 +2589,23 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +static void percentile_next_step(SQLFunctionCtx *pCtx) { + SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = pResInfo->interResultBuf; + + if (pInfo->stage == 0) { + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + } + + pInfo->stage += 1; + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } else { + pResInfo->complete = true; + } +} + ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4513,7 +4631,7 @@ SQLAggFuncElem aAggs[] = {{ percentile_function_setup, percentile_function, percentile_function_f, - no_next_step, + percentile_next_step, percentile_finalizer, noop1, noop1, diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index b240d357a8..50f82ae662 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -16,7 +16,6 @@ #include "os.h" #include "taosmsg.h" -#include "qExtbuffer.h" #include "taosdef.h" #include "tcache.h" #include "tname.h" @@ -430,7 +429,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { pRes->qhandle = 0x1; pRes->numOfRows = 0; } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosCacheEmpty(tscCacheHandle); + taosCacheEmpty(tscMetaCache); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 39a757795e..af6a546ff4 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -472,10 +472,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { return; } - tscDebug("%p start to free local reducer", pSql); SSqlRes *pRes = &(pSql->res); if (pRes->pLocalReducer == NULL) { - tscDebug("%p local reducer has been freed, abort", pSql); return; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 09eb8f167e..18926a84b9 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -526,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe int32_t index = 0; SStrToken sToken; - int16_t numOfRows = 0; + int32_t numOfRows = 0; SSchema *pSchema = tscGetTableSchema(pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMeta); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c248b08ddd..517fb32bb3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -92,7 +92,6 @@ static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQueryS static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** exprString); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateEp(char* ep); @@ -103,6 +102,7 @@ static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killTy static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); +static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); @@ -360,8 +360,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + // additional msg has been attached already if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return TSDB_CODE_TSC_INVALID_SQL; } return tscGetTableMeta(pSql, pTableMetaInfo); @@ -712,8 +713,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg1 = "name too long"; - const char* msg2 = "invalid db name"; - const char *msg = msg1; + const char* msg2 = "current database name is invalid"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; @@ -728,17 +728,24 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa if (hasSpecifyDB(pzTableName)) { // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); + if (code != 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } else { // get current DB name first, then set it into path SStrToken t = {0}; getCurrentDBName(pSql, &t); if (t.n == 0) { - msg = msg2; + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); + if (code != 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } + if (code != TSDB_CODE_SUCCESS) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - free(oldName); + taosTFree(oldName); return code; } @@ -1181,13 +1188,14 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } END_TRY len = tbufTell(&bw); - char* c = tbufGetData(&bw, true); + char* c = tbufGetData(&bw, false); // set the serialized binary string as the parameter of arithmetic expression addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex); insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); + tbufCloseWriter(&bw); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { @@ -1331,7 +1339,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel /* * transfer sql functions that need secondary merge into another format - * in dealing with metric queries such as: count/first/last + * in dealing with super table queries such as: count/first/last */ if (isSTable) { tscTansformSQLFuncForSTableQuery(pQueryInfo); @@ -1516,16 +1524,16 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex) { + const char* msg1 = "not support column types"; + int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; - const char* msg1 = "not support column types"; + char columnName[TSDB_COL_NAME_LEN] = {0}; int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { - if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY || - pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_NCHAR || - pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BOOL) { + int32_t t1 = pSchema[pColIndex->columnIndex].type; + if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; } else { @@ -1542,8 +1550,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS } else { getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); } - - + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); @@ -1800,10 +1807,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } + + // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID}; - if (functionID == TSDB_FUNC_LAST_ROW && TSWINDOW_IS_EQUAL(pQueryInfo->window,TSWINDOW_INITIALIZER)) { + if (functionID == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { cvtFunc.execFuncId = TSDB_FUNC_LAST; } + if (!requireAllFields) { if (pItem->pNode->pParam->nExpr < 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -2591,7 +2601,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { continue; } - if (functionId == TSDB_FUNC_PRJ && pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (functionId == TSDB_FUNC_PRJ && (pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->colInfo.flag))) { continue; } @@ -2803,6 +2813,12 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, case TK_LIKE: pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE; break; + case TK_ISNULL: + pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL; + break; + case TK_NOTNULL: + pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL; + break; default: return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -2848,19 +2864,19 @@ static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { return TSDB_CODE_SUCCESS; } +// pExpr->nSQLOptr == 0 while handling "is null" query static bool isExprLeafNode(tSQLExpr* pExpr) { return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || - pExpr->nSQLOptr == TK_SET); + (pExpr->nSQLOptr == 0 || pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || pExpr->nSQLOptr == TK_SET); } -static bool isExprDirectParentOfLeaftNode(tSQLExpr* pExpr) { +static bool isExprDirectParentOfLeafNode(tSQLExpr* pExpr) { return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && (isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight)); } static int32_t tSQLExprLeafToString(tSQLExpr* pExpr, bool addParentheses, char** output) { - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3071,7 +3087,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) { return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { *(*str) = '('; *str += 1; @@ -3127,7 +3143,7 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { // internal node + if (!isExprDirectParentOfLeafNode(pExpr)) { // internal node int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->nSQLOptr); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3153,7 +3169,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -3225,19 +3241,6 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { - char* start = *str; - - int32_t code = doArithmeticExprToString(pExpr, str); - if (code == TSDB_CODE_SUCCESS) { // remove out the parenthesis - int32_t len = (int32_t)strlen(start); - memmove(start, start + 1, len - 2); - start[len - 2] = 0; - } - - return code; -} - static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { @@ -3485,7 +3488,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - assert(isExprDirectParentOfLeaftNode(*pExpr)); + assert(isExprDirectParentOfLeafNode(*pExpr)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -3531,7 +3534,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL } } - // in case of in operator, keep it in a seperate attribute + // in case of in operator, keep it in a seprate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); @@ -3552,7 +3555,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL *type = TSQL_EXPR_TBNAME; *pExpr = NULL; } else { - if (pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query + if (pRight != NULL && pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3605,7 +3608,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr int32_t leftType = -1; int32_t rightType = -1; - if (!isExprDirectParentOfLeaftNode(*pExpr)) { + if (!isExprDirectParentOfLeafNode(*pExpr)) { int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3636,7 +3639,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr } static void doCompactQueryExpr(tSQLExpr** pExpr) { - if (*pExpr == NULL || isExprDirectParentOfLeaftNode(*pExpr)) { + if (*pExpr == NULL || isExprDirectParentOfLeafNode(*pExpr)) { return; } @@ -3667,7 +3670,7 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) { } static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* pQueryInfo, tSQLExpr** pOut, int32_t tableIndex) { - if (isExprDirectParentOfLeaftNode(*pExpr)) { + if (isExprDirectParentOfLeafNode(*pExpr)) { tSQLExpr* pLeft = (*pExpr)->pLeft; SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -3826,7 +3829,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeaftNode(pExpr)) { + if (!isExprDirectParentOfLeafNode(pExpr)) { if (pExpr->nSQLOptr == TK_OR) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -3996,7 +3999,6 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql const char* msg2 = "invalid filter expression"; int32_t ret = TSDB_CODE_SUCCESS; - pQueryInfo->window = TSWINDOW_INITIALIZER; // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb; memset(&sb, 0, sizeof(sb)); @@ -5244,7 +5246,8 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { + if ((pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) && + !(pExpr->functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) { SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, (int32_t)pExpr->param[0].i64Key, &pExpr->resType, &pExpr->resBytes, &pExpr->interBytes, tagLength, true); @@ -5252,12 +5255,13 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } } -static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { +static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId == TSDB_FUNC_PRJ) { + + if (pExpr->functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) { bool qualifiedCol = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); @@ -5270,9 +5274,14 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { } } - assert(qualifiedCol); + // it is not a tag column/tbname column/user-defined column, return error + if (!qualifiedCol) { + return TSDB_CODE_TSC_INVALID_SQL; + } } } + + return TSDB_CODE_SUCCESS; } static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { @@ -5346,16 +5355,23 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) const char* msg1 = "only one selectivity function allowed in presence of tags function"; const char* msg3 = "aggregation function should not be mixed up with projection"; - bool tagColExists = false; + bool tagTsColExists = false; int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; + // todo is 0?? + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + if (!isSTable) { + return TSDB_CODE_SUCCESS; + } + size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { - tagColExists = true; + tagTsColExists = true; // selectivity + ts/tag column break; } } @@ -5376,7 +5392,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) } } - if (tagColExists) { // check if the selectivity function exists + if (tagTsColExists) { // check if the selectivity function exists // When the tag projection function on tag column that is not in the group by clause, aggregation function and // selectivity function exist in select clause is not allowed. if (numOfAggregation > 0) { @@ -5388,26 +5404,36 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) */ if (numOfSelectivity == 1) { doUpdateSqlFunctionForTagPrj(pQueryInfo); - doUpdateSqlFunctionForColPrj(pQueryInfo); + int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else if (numOfSelectivity > 1) { /* * If more than one selectivity functions exist, all the selectivity functions must be last_row. * Otherwise, return with error code. */ for (int32_t i = 0; i < numOfExprs; ++i) { - - int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + int16_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) { continue; } - if (((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) && (functionId != TSDB_FUNC_LAST_ROW)) { + if ((functionId == TSDB_FUNC_LAST_ROW) || + (functionId == TSDB_FUNC_LAST_DST && (pExpr->colInfo.flag & TSDB_COL_NULL) != 0)) { + // do nothing + } else { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } doUpdateSqlFunctionForTagPrj(pQueryInfo); - doUpdateSqlFunctionForColPrj(pQueryInfo); + int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else { if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { @@ -5418,7 +5444,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) if (numOfAggregation > 0 || numOfSelectivity > 0) { // clear the projection type flag pQueryInfo->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY); - doUpdateSqlFunctionForColPrj(pQueryInfo); + int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -6128,11 +6157,27 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } + // set where info + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + if (pQuerySql->pWhere != NULL) { + if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + pQuerySql->pWhere = NULL; + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; + pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; + } + } else { // set the time rang + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } + } + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); - if (pQuerySql->pWhere) { - pQueryInfo->window = TSWINDOW_INITIALIZER; - } if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6152,26 +6197,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - // set where info - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - if (pQuerySql->pWhere != NULL) { - if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - pQuerySql->pWhere = NULL; - if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; - pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; - } - } else { // set the time rang - pQueryInfo->window = TSWINDOW_INITIALIZER; - if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); - } - } - // user does not specified the query time window, twa is not allowed in such case. if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX || (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { @@ -6250,6 +6275,11 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS return ret; } } + + if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->nSQLOptr == 0) { + *pExpr = calloc(1, sizeof(tExprNode)); + return TSDB_CODE_SUCCESS; + } if (pSqlExpr->pLeft == NULL) { if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) { @@ -6322,7 +6352,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); - + + // check for dividing by 0 if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { if (pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64Key == 0) { @@ -6335,7 +6366,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS // NOTE: binary|nchar data allows the >|< type filter if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { - if (pRight->nodeType == TSQL_NODE_VALUE) { + if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6345,3 +6376,15 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS return TSDB_CODE_SUCCESS; } + +bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); + if (pCol->numOfFilters > 0) { + return true; + } + } + + return false; +} \ No newline at end of file diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c9fe5fbd2c..5639719c15 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -27,10 +27,7 @@ #include "tutil.h" #include "tlockfree.h" -#define TSC_MGMT_VNODE 999 - SRpcCorEpSet tscMgmtEpSet; -SRpcEpSet tscDnodeEpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -236,20 +233,27 @@ int tscSendMsgToServer(SSqlObj *pSql) { } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; - if (pSql == NULL || pSql->signature != pSql) { - tscError("%p sql is already released", pSql); + uint64_t handle = (uint64_t) rpcMsg->ahandle; + + void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(uint64_t)); + if (p == NULL) { + rpcFreeCont(rpcMsg->pCont); return; } + SSqlObj* pSql = *p; + assert(pSql != NULL); + STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + assert(*pSql->self == pSql); + if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); - tscFreeSqlObj(pSql); + taosCacheRelease(tscObjCache, (void**) &p, true); rpcFreeCont(rpcMsg->pCont); return; } @@ -261,18 +265,21 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - tscFreeSqlObj(pSql); + void** p1 = p; + taosCacheRelease(tscObjCache, (void**) &p1, false); + + taosCacheRelease(tscObjCache, (void**) &p, true); rpcFreeCont(rpcMsg->pCont); return; } - if (pEpSet) { + if (pEpSet) { if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { - if(pCmd->command < TSDB_SQL_MGMT) { - tscUpdateVgroupInfo(pSql, pEpSet); + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pEpSet); } else { tscUpdateMgmtEpSet(pEpSet); - } + } } } @@ -294,7 +301,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pSql->retry > pSql->maxRetry) { tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); } else { - // wait for a little bit moment and then retry + // wait for a little bit moment and then retry, todo do not sleep in rpc callback thread if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { int32_t duration = getWaitingTimeInterval(pSql->retry); taosMsleep(duration); @@ -304,6 +311,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { // if there is an error occurring, proceed to the following error handling procedure. if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosCacheRelease(tscObjCache, (void**) &p, false); rpcFreeCont(rpcMsg->pCont); return; } @@ -365,16 +373,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); } + bool shouldFree = tscShouldBeFreed(pSql); if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - - bool shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, pSql, rpcMsg->code); + } - if (shouldFree) { - tscDebug("%p sqlObj is automatically freed", pSql); - tscFreeSqlObj(pSql); - } + void** p1 = p; + taosCacheRelease(tscObjCache, (void**) &p1, false); + + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it + taosCacheRelease(tscObjCache, (void **)&p, true); + tscDebug("%p sqlObj is automatically freed", pSql); } rpcFreeCont(rpcMsg->pCont); @@ -1667,8 +1677,10 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) { - tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid); + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && + (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->sid, pMetaMsg->tableId); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1708,7 +1720,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, + pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case @@ -1820,7 +1832,7 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { // int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache // // pMeta->index = 0; - // (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer); + // (void)taosCachePut(tscMetaCache, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer); // } } @@ -1907,12 +1919,14 @@ int tscProcessShowRsp(SSqlObj *pSql) { key[0] = pCmd->msgType + 'a'; strcpy(key + 1, "showlist"); - taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false); - + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false); + } + size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, + pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size, tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); @@ -1971,6 +1985,8 @@ static void createHBObj(STscObj* pObj) { pSql->pTscObj = pObj; pSql->signature = pSql; pObj->pHb = pSql; + T_REF_INC(pObj); + tscAddSubqueryInfo(&pObj->pHb->cmd); tscDebug("%p HB is allocated, pObj:%p", pObj->pHb, pObj); @@ -2000,7 +2016,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { createHBObj(pObj); - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); +// taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); return 0; } @@ -2015,14 +2031,14 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { int tscProcessDropDbRsp(SSqlObj *pSql) { pSql->pTscObj->db[0] = 0; - taosCacheEmpty(tscCacheHandle); + taosCacheEmpty(tscMetaCache); return 0; } int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -2035,10 +2051,10 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { * instead. */ tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); + taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); if (pTableMetaInfo->pTableMeta) { - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); } return 0; @@ -2047,21 +2063,21 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); + taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); if (pTableMetaInfo->pTableMeta) { bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); if (isSuperTable) { // if it is a super table, reset whole query cache tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); - taosCacheEmpty(tscCacheHandle); + taosCacheEmpty(tscMetaCache); } } @@ -2146,6 +2162,12 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; + T_REF_INC(pNew->pTscObj); + + // TODO add test case on x86 platform + uint64_t adr = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000); + tscAddSubqueryInfo(&pNew->cmd); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); @@ -2182,10 +2204,10 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { // If this STableMetaInfo owns a table meta, release it first if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false); } - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); + pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, @@ -2220,7 +2242,7 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); } - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); return getTableMetaFromMgmt(pSql, pTableMetaInfo); } @@ -2250,7 +2272,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - + + // TODO TEST IT SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); @@ -2260,7 +2283,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta); + STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList); } @@ -2270,6 +2293,10 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { } pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + T_REF_INC(pNew->pTscObj); + + uint64_t p = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); pNew->fp = tscTableMetaCallBack; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 9fa4db999f..69bc69cd4a 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -26,6 +26,7 @@ #include "tsclient.h" #include "ttokendef.h" #include "tutil.h" +#include "tscProfile.h" static bool validImpl(const char* str, size_t maxsize) { if (str == NULL) { @@ -100,6 +101,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } pObj->signature = pObj; + pObj->pDnodeConn = pDnodeConn; + T_REF_INIT_VAL(pObj, 1); tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); @@ -132,20 +135,15 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } - pSql->pTscObj = pObj; + pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_MAX_REPLICA; - tsem_init(&pSql->rspSem, 0, 0); - - pObj->pDnodeConn = pDnodeConn; - - pSql->fp = fp; - pSql->param = param; - if (taos != NULL) { - *taos = pObj; - } - + pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->fp = fp; + pSql->param = param; pSql->cmd.command = TSDB_SQL_CONNECT; + + tsem_init(&pSql->rspSem, 0, 0); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; rpcClose(pDnodeConn); @@ -154,7 +152,16 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } + if (taos != NULL) { + *taos = pObj; + } + + T_REF_INC(pSql->pTscObj); + + uint64_t key = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + return pSql; } @@ -257,6 +264,33 @@ void taos_close(TAOS *taos) { tscFreeSqlObj(pObj->pHb); } + // free all sqlObjs created by using this connect before free the STscObj +// while(1) { +// pthread_mutex_lock(&pObj->mutex); +// void* p = pObj->sqlList; +// pthread_mutex_unlock(&pObj->mutex); +// +// if (p == NULL) { +// break; +// } +// +// tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p); +// taosMsleep(100); +// +// // todo fix me!! two threads call taos_free_result will cause problem. +// tscDebug("%p free :%p", pObj, p); +// taos_free_result(p); +// } + + int32_t ref = T_REF_DEC(pObj); + assert(ref >= 0); + + if (ref > 0) { + tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn); + return; + } + + tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); tscCloseTscObj(pObj); } @@ -533,57 +567,51 @@ int taos_select_db(TAOS *taos, const char *db) { } // send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscKillQueryInVnode(SSqlObj* pSql) { +static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) && - (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_SHOW || - pCmd->command == TSDB_SQL_RETRIEVE || - pCmd->command == TSDB_SQL_FETCH) && - (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) { - - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]); - tscProcessSql(pSql); + if (pRes == NULL || pRes->qhandle == 0) { return true; } - return false; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + return true; + } + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + tscRemoveFromSqlList(pSql); + + int32_t cmd = pCmd->command; + if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && + (cmd == TSDB_SQL_SELECT || + cmd == TSDB_SQL_SHOW || + cmd == TSDB_SQL_RETRIEVE || + cmd == TSDB_SQL_FETCH)) { + pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql, sqlCmd[pCmd->command]); + + tscProcessSql(pSql); + return false; + } + + return true; } void taos_free_result(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - + SSqlObj* pSql = (SSqlObj*) res; if (pSql == NULL || pSql->signature != pSql) { - tscDebug("%p sqlObj has been freed", pSql); - return; - } - - // The semaphore can not be changed while freeing async sub query objects. - SSqlRes *pRes = &pSql->res; - if (pRes == NULL || pRes->qhandle == 0) { - tscFreeSqlObj(pSql); - tscDebug("%p SqlObj is freed by app, qhandle is null", pSql); + tscError("%p already released sqlObj", res); return; } - // set freeFlag to 1 in retrieve message if there are un-retrieved results data in node - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - if (pQueryInfo == NULL) { - tscFreeSqlObj(pSql); - tscDebug("%p SqlObj is freed by app", pSql); - return; - } - - pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - if (!tscKillQueryInVnode(pSql)) { - tscFreeSqlObj(pSql); - tscDebug("%p sqlObj is freed by app", pSql); + bool freeNow = tscKillQueryInDnode(pSql); + if (freeNow) { + tscDebug("%p free sqlObj in cache", pSql); + SSqlObj** p = pSql->self; + taosCacheRelease(tscObjCache, (void**) &p, true); } } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 79e0011093..71ba76dc6c 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -167,7 +167,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); taosTFree(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); @@ -275,7 +275,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf // release the metric/meter meta information reference, so data in cache can be updated - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); taosTFree(pSql->pSubs); pSql->numOfSubs = 0; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e264fa9b33..4246abf52d 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -92,7 +92,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); #ifdef _DEBUG_VIEW - tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); + tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key); #endif int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); @@ -178,10 +178,6 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in pSupporter->subqueryIndex = index; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); - pSupporter->intervalTimeUnit = pQueryInfo->intervalTimeUnit; - pSupporter->slidingTime = pQueryInfo->slidingTimeUnit; - pSupporter->intervalTime = pQueryInfo->intervalTime; - pSupporter->slidingTime = pQueryInfo->slidingTime; pSupporter->limit = pQueryInfo->limit; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); @@ -311,18 +307,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); - pQueryInfo->intervalTimeUnit = pSupporter->intervalTimeUnit; - pQueryInfo->slidingTimeUnit = pSupporter->slidingTimeUnit; - pQueryInfo->intervalTime = pSupporter->intervalTime; - pQueryInfo->slidingTime = pSupporter->slidingTime; - pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); pQueryInfo->colList = pSupporter->colList; pQueryInfo->exprList = pSupporter->exprList; pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; - + pSupporter->exprList = NULL; pSupporter->colList = NULL; memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); @@ -1221,7 +1211,6 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter pNewQueryInfo->limit.offset = 0; // backup the data and clear it in the sqlcmd object - pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); tscInitQueryInfo(pNewQueryInfo); @@ -1523,9 +1512,9 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { SSqlObj *pParentSql = trsupport->pParentSql; assert(pSql == pParentSql->pSubs[index]); - pParentSql->pSubs[index] = NULL; - - taos_free_result(pSql); +// pParentSql->pSubs[index] = NULL; +// +// taos_free_result(pSql); taosTFree(trsupport->localBuffer); taosTFree(trsupport); } @@ -1739,10 +1728,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; -// if (pSql == NULL) { // sql object has been released in error process, return immediately -// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); -// return; -// } SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); @@ -1918,9 +1903,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; } - taos_free_result(tres); taosTFree(pSupporter); - if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { return; } @@ -1964,28 +1947,27 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) { } int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - - size_t size = taosArrayGetSize(pCmd->pDataBlocks); - assert(size > 0); + SSqlRes *pRes = &pSql->res; + + pSql->numOfSubs = taosArrayGetSize(pCmd->pDataBlocks); + assert(pSql->numOfSubs > 0); + + pRes->code = TSDB_CODE_SUCCESS; // the number of already initialized subqueries int32_t numOfSub = 0; - pSql->numOfSubs = (uint16_t)size; - pSql->pSubs = calloc(size, POINTER_BYTES); + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + pState->numOfTotal = pSql->numOfSubs; + pState->numOfRemain = pSql->numOfSubs; + + pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; } - tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); - - SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = pSql->numOfSubs; - pState->numOfRemain = pSql->numOfSubs; - - pRes->code = TSDB_CODE_SUCCESS; + tscDebug("%p submit data to %d vnode(s)", pSql, pSql->numOfSubs); while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); @@ -2016,8 +1998,8 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { tscDebug("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub); numOfSub++; } else { - tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%" PRIzu ", code:%s", pSql, numOfSub, - size, tstrerror(pRes->code)); + tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub, + pSql->numOfSubs, tstrerror(pRes->code)); goto _error; } } @@ -2040,11 +2022,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; _error: - for(int32_t j = 0; j < numOfSub; ++j) { - taosTFree(pSql->pSubs[j]->param); - taos_free_result(pSql->pSubs[j]); - } - taosTFree(pState); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2220,16 +2197,15 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { if (pRes->pArithSup == NULL) { - SArithmeticSupport *sas = (SArithmeticSupport *) calloc(1, sizeof(SArithmeticSupport)); - sas->offset = 0; - sas->pArithExpr = pSup->pArithExprInfo; - sas->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - sas->exprList = pQueryInfo->exprList; - sas->data = calloc(sas->numOfCols, POINTER_BYTES); - - pRes->pArithSup = sas; + pRes->pArithSup = (SArithmeticSupport*)calloc(1, sizeof(SArithmeticSupport)); } + pRes->pArithSup->offset = 0; + pRes->pArithSup->pArithExpr = pSup->pArithExprInfo; + pRes->pArithSup->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + pRes->pArithSup->exprList = pQueryInfo->exprList; + pRes->pArithSup->data = calloc(pRes->pArithSup->numOfCols, POINTER_BYTES); + if (pRes->buffer[i] == NULL) { TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pRes->buffer[i] = malloc(field->bytes); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 72f23881d2..82ce1d3679 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -30,7 +30,8 @@ #include "tlocale.h" // global, not configurable -void * tscCacheHandle; +SCacheObj* tscMetaCache; +SCacheObj* tscObjCache; void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; @@ -144,8 +145,9 @@ void taos_init_imp(void) { refreshTime = refreshTime > 10 ? 10 : refreshTime; refreshTime = refreshTime < 10 ? 10 : refreshTime; - if (tscCacheHandle == NULL) { - tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); + if (tscMetaCache == NULL) { + tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); + tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime/2, false, tscFreeSqlObjInCache, "sqlObj"); } tscDebug("client is initialized successfully"); @@ -154,9 +156,12 @@ void taos_init_imp(void) { void taos_init() { pthread_once(&tscinit, taos_init_imp); } void taos_cleanup() { - if (tscCacheHandle != NULL) { - taosCacheCleanup(tscCacheHandle); - tscCacheHandle = NULL; + if (tscMetaCache != NULL) { + taosCacheCleanup(tscMetaCache); + tscMetaCache = NULL; + + taosCacheCleanup(tscObjCache); + tscObjCache = NULL; } if (tscQhandle != NULL) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 80e3828c9d..9b3f5927ab 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -252,11 +252,11 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { if (pRes->tsrow == NULL) { int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; pRes->numOfCols = numOfOutput; - + pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); pRes->length = calloc(numOfOutput, sizeof(int32_t)); pRes->buffer = calloc(numOfOutput, POINTER_BYTES); - + // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { taosTFree(pRes->tsrow); @@ -268,7 +268,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { return TSDB_CODE_SUCCESS; } -void tscDestroyResPointerInfo(SSqlRes* pRes) { +static void tscDestroyResPointerInfo(SSqlRes* pRes) { if (pRes->buffer != NULL) { // free all buffers containing the multibyte string for (int i = 0; i < pRes->numOfCols; i++) { taosTFree(pRes->buffer[i]); @@ -344,8 +344,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { } SSqlCmd* pCmd = &pSql->cmd; - STscObj* pObj = pSql->pTscObj; - int32_t cmd = pCmd->command; if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { @@ -353,26 +351,61 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { } // pSql->sqlstr will be used by tscBuildQueryStreamDesc - if (pObj->signature == pObj) { +// if (pObj->signature == pObj) { //pthread_mutex_lock(&pObj->mutex); taosTFree(pSql->sqlstr); //pthread_mutex_unlock(&pObj->mutex); - } +// } tscFreeSqlResult(pSql); taosTFree(pSql->pSubs); pSql->numOfSubs = 0; + pSql->self = 0; tscResetSqlCmdObj(pCmd, false); } +static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) { + if (pSql->numOfSubs == 0) { + return; + } + + tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->numOfSubs); + + for(int32_t i = 0; i < pSql->numOfSubs; ++i) { + tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i); + taos_free_result(pSql->pSubs[i]); + pSql->pSubs[i] = NULL; + } + + pSql->numOfSubs = 0; +} + +/** + * The free operation will cause the pSql to be removed from hash table and free it in + * the function of processmsgfromserver is impossible in this case, since it will fail + * to retrieve pSqlObj in hashtable. + * + * @param pSql + */ +void tscFreeSqlObjInCache(void *pSql) { + assert(pSql != NULL); + SSqlObj** p = (SSqlObj**)pSql; + + assert((*p)->self != 0 && (*p)->self == (p)); + tscFreeSqlObj(*p); +} + void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - tscDebug("%p start to free sql object", pSql); + tscDebug("%p start to free sqlObj", pSql); + STscObj* pTscObj = pSql->pTscObj; + + tscFreeSubobj(pSql); tscPartiallyFreeSqlObj(pSql); pSql->signature = NULL; @@ -388,6 +421,14 @@ void tscFreeSqlObj(SSqlObj* pSql) { tsem_destroy(&pSql->rspSem); free(pSql); + tscDebug("%p free sqlObj completed", pSql); + + int32_t ref = T_REF_DEC(pTscObj); + assert(ref >= 0); + + if (ref == 0) { + tscCloseTscObj(pTscObj); + } } void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { @@ -399,7 +440,10 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { taosTFree(pDataBlock->params); // free the refcount for metermeta - taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false); + if (pDataBlock->pTableMeta != NULL) { + taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); + } + taosTFree(pDataBlock); } @@ -454,9 +498,12 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); - pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta); + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); + } + + pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta); } else { assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); } @@ -527,7 +574,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff * due to operation such as drop database. So here we add the reference count directly instead of invoke * taosGetDataFromCache, which may return NULL value. */ - dataBuf->pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMeta); + dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -721,17 +768,19 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { // TODO: all subqueries should be freed correctly before close this connection. void tscCloseTscObj(STscObj* pObj) { assert(pObj != NULL); - + pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - pthread_mutex_destroy(&pObj->mutex); - + + void* p = pObj->pDnodeConn; if (pObj->pDnodeConn != NULL) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } - - tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn); + + pthread_mutex_destroy(&pObj->mutex); + + tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); taosTFree(pObj); } @@ -1531,8 +1580,8 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); assert(pQueryInfo->exprList == NULL); - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; } @@ -1554,6 +1603,8 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { } tscInitQueryInfo(pQueryInfo); + + pQueryInfo->window = TSWINDOW_INITIALIZER; pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; @@ -1665,7 +1716,10 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) return; } - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); + } + taosTFree(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); @@ -1689,6 +1743,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm } pNew->pTscObj = pSql->pTscObj; + T_REF_INC(pNew->pTscObj); + pNew->signature = pNew; SSqlCmd* pCmd = &pNew->cmd; @@ -1719,6 +1775,11 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); + + T_REF_INC(pNew->pTscObj); + + uint64_t p = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000); return pNew; } @@ -1807,6 +1868,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; + T_REF_INC(pNew->pTscObj); pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { @@ -1911,14 +1973,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup + STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - STableMeta* pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); + STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); @@ -1958,6 +2020,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex); } + T_REF_INC(pNew->pTscObj); + + uint64_t p = (uint64_t) pNew; + pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10); return pNew; _error: @@ -2069,6 +2135,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } + int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { const char* msgFormat1 = "invalid SQL: %s"; const char* msgFormat2 = "invalid SQL: \'%s\' (%s)"; @@ -2099,11 +2166,6 @@ bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); } -bool tscResultsetFetchCompleted(TAOS_RES *result) { - SSqlRes* pRes = result; - return pRes->completed; -} - char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; } /** diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 96e8fb26c6..7e46f58a93 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -957,17 +957,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - // http configs - cfg.option = "httpCacheSessions"; - cfg.ptr = &tsHttpCacheSessions; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 49c9e6b726..bb18cb6847 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -62,10 +62,9 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t len = strdequote(exprStr->z); - size_t tlen = MIN(sizeof(s.name), len + 1); - + size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); tstrncpy(s.name, exprStr->z, tlen); + strdequote(s.name); } return s; diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 3910d80ecf..ff417b6cde 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -540,9 +540,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } } -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) { - char tmpBuf[4096] = {0}; - +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: { SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); @@ -575,10 +573,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) { } default: { - assert(size <= 4096); - memcpy(tmpBuf, pLeft, size); + memcpy(buf, pLeft, size); memcpy(pLeft, pRight, size); - memcpy(pRight, tmpBuf, size); + memcpy(pRight, buf, size); break; } } diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index da0c5b12d4..0fed63fc80 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -48,11 +48,6 @@ - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - junit junit diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 02d642d643..e25bd64c73 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -14,8 +14,6 @@ *****************************************************************************/ package com.taosdata.jdbc; -import org.apache.commons.lang3.StringUtils; - import java.sql.*; import java.util.Properties; import java.util.logging.Logger; @@ -42,9 +40,8 @@ import java.util.logging.Logger; public class TSDBDriver implements java.sql.Driver { @Deprecated - private static final String URL_PREFIX1 = "jdbc:TSDB://"; - - private static final String URL_PREFIX = "jdbc:TAOS://"; + private static final String URL_PREFIX1 = "jdbc:tsdb://"; + private static final String URL_PREFIX = "jdbc:taos://"; /** * Key used to retrieve the database value from the properties instance passed @@ -188,7 +185,7 @@ public class TSDBDriver implements java.sql.Driver { } public boolean acceptsURL(String url) throws SQLException { - return StringUtils.isNotBlank(url) && url.startsWith(URL_PREFIX); + return (url != null && url.length() > 0 && url.trim().length() > 0) && url.toLowerCase().startsWith(URL_PREFIX); } public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { @@ -238,7 +235,8 @@ public class TSDBDriver implements java.sql.Driver { return null; } - if (!StringUtils.startsWithIgnoreCase(url, URL_PREFIX) && !StringUtils.startsWithIgnoreCase(url, URL_PREFIX1)) { + String lowerUrl = url.toLowerCase(); + if (!lowerUrl.startsWith(URL_PREFIX) && !lowerUrl.startsWith(URL_PREFIX1)) { return null; } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 1a40f3b56d..17cbbca834 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -198,7 +198,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); void* getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 @@ -209,21 +209,24 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_IN 8 +#define TSDB_RELATION_ISNULL 8 +#define TSDB_RELATION_NOTNULL 9 +#define TSDB_RELATION_IN 10 -#define TSDB_RELATION_AND 9 -#define TSDB_RELATION_OR 10 -#define TSDB_RELATION_NOT 11 +#define TSDB_RELATION_AND 11 +#define TSDB_RELATION_OR 12 +#define TSDB_RELATION_NOT 13 -#define TSDB_BINARY_OP_ADD 12 -#define TSDB_BINARY_OP_SUBTRACT 13 -#define TSDB_BINARY_OP_MULTIPLY 14 -#define TSDB_BINARY_OP_DIVIDE 15 -#define TSDB_BINARY_OP_REMAINDER 16 +#define TSDB_BINARY_OP_ADD 30 +#define TSDB_BINARY_OP_SUBTRACT 31 +#define TSDB_BINARY_OP_MULTIPLY 32 +#define TSDB_BINARY_OP_DIVIDE 33 +#define TSDB_BINARY_OP_REMAINDER 34 #define TS_PATH_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 #define TSDB_USER_LEN TSDB_UNI_LEN + // ACCOUNT is a 32 bit positive integer // this is the length of its string representation // including the terminator zero diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 277dc45f8e..1d1ca1c42b 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -765,7 +765,9 @@ void read_history() { FILE *f = fopen(f_history, "r"); if (f == NULL) { #ifndef WINDOWS - fprintf(stderr, "Failed to open file %s\n", f_history); + if (errno != ENOENT) { + fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); + } #endif return; } @@ -792,7 +794,7 @@ void write_history() { FILE *f = fopen(f_history, "w"); if (f == NULL) { #ifndef WINDOWS - fprintf(stderr, "Failed to open file %s for write\n", f_history); + fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif return; } diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index c0c0c494de..9238869375 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -131,8 +131,6 @@ HttpContext *httpCreateContext(int32_t fd) { HttpContext *httpGetContext(void *ptr) { uint64_t handleVal = (uint64_t)ptr; HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(HttpContext *)); - ASSERT(ppContext); - ASSERT(*ppContext); if (ppContext) { HttpContext *pContext = *ppContext; diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index cad5dd8f91..0c3204687a 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -238,6 +238,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } +#if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { parser->contentChunked = 1; @@ -245,8 +246,9 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } + #endif - else if (0 == strcasecmp(key, "Transfer-Encoding")) { + else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { parser->transferGzip = 1; ehttp_gzip_conf_t conf = {0}; diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index 26f0441519..f8912331a3 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -87,15 +87,12 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - cmd->numOfRows += numOfRows; - int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); for (int32_t k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); if (row == NULL) { - cmd->numOfRows--; continue; } int32_t* length = taos_fetch_lengths(result); @@ -151,24 +148,23 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - } + httpJsonToken(jsonBuf, JsonArrEnd); + cmd->numOfRows ++; - if (cmd->numOfRows >= tsRestRowLimit) { - httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, - pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit); - return false; - } else { if (pContext->fd <= 0) { - httpError("context:%p, fd:%d, user:%s, connection is closed, abort retrieve", pContext, pContext->fd, - pContext->user); + httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); + return false; + } + + if (cmd->numOfRows >= tsRestRowLimit) { + httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, + pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit); return false; - } else { - httpDebug("context:%p, fd:%d, user:%s, total rows:%d retrieved", pContext, pContext->fd, pContext->user, - cmd->numOfRows); - return true; } } + + httpDebug("context:%p, fd:%d, user:%s, retrieved row:%d", pContext, pContext->fd, pContext->user, cmd->numOfRows); + return true; } bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 2d95a0ac72..f0a7249b51 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -315,45 +315,48 @@ static bool httpReadData(HttpContext *pContext) { pContext->accessTimes++; pContext->lastAccessTime = taosGetTimestampSec(); + char buf[HTTP_STEP_SIZE + 1] = {0}; - char buf[HTTP_STEP_SIZE + 1] = {0}; - int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, sizeof(buf)); - if (nread > 0) { - buf[nread] = '\0'; - httpTrace("context:%p, fd:%d, nread:%d", pContext, pContext->fd, nread); - int32_t ok = httpParseBuf(pParser, buf, nread); + while (1) { + int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); + if (nread > 0) { + buf[nread] = '\0'; + httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); + int32_t ok = httpParseBuf(pParser, buf, nread); - if (ok) { - httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, pParser->parseCode); - httpSendErrorResp(pContext, pParser->parseCode); - httpNotifyContextClose(pContext); - return false; - } + if (ok) { + httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, + pParser->parseCode); + httpSendErrorResp(pContext, pParser->parseCode); + httpNotifyContextClose(pContext); + return false; + } - if (pParser->parseCode) { - httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); - httpSendErrorResp(pContext, pParser->parseCode); - httpNotifyContextClose(pContext); - return false; - } + if (pParser->parseCode) { + httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); + httpSendErrorResp(pContext, pParser->parseCode); + httpNotifyContextClose(pContext); + return false; + } - if (!pParser->parsed) { - httpTrace("context:%p, fd:%d, read not over yet, len:%d", pContext, pContext->fd, pParser->body.pos); - return false; + if (!pParser->parsed) { + httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd); + continue; + } else { + httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos); + return true; + } + } else if (nread < 0) { + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); + return false; // later again + } else { + httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); + return false; + } } else { - httpTraceL("context:%p, fd:%d, len:%d, body:%s", pContext, pContext->fd, pParser->body.pos, pParser->body.str); - return true; - } - } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { - httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); - return false; // later again - } else { - httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); + httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread); return false; } - } else { - httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread); - return false; } } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 207f5d7923..881fa55fb7 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -51,10 +51,6 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n } } - // if (tscResultsetFetchCompleted(result)) { - // isContinue = false; - // } - if (isContinue) { // retrieve next batch of rows httpDebug("context:%p, fd:%d, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", pContext, @@ -224,14 +220,6 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int } } -#if 0 - // todo refactor - if (tscResultsetFetchCompleted(result)) { - httpDebug("context:%p, fd:%d, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->user); - isContinue = false; - } -#endif - if (isContinue) { // retrieve next batch of rows httpDebug("context:%p, fd:%d, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 547616dee6..d3e60c21dc 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -32,6 +32,7 @@ struct tExprNode; struct SSchema; enum { + TSQL_NODE_DUMMY = 0x0, TSQL_NODE_EXPR = 0x1, TSQL_NODE_COL = 0x2, TSQL_NODE_VALUE = 0x4, diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 25fb04fb9a..169bf907c6 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -57,7 +57,7 @@ typedef struct SWindowResult { uint16_t numOfRows; // number of rows of current time window bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - TSKEY skey; // start key of current time window + union {STimeWindow win; char* key;}; // start key of current time window } SWindowResult; /** diff --git a/src/query/inc/qPercentile.h b/src/query/inc/qPercentile.h index 0a52d4f205..c34c24c5b2 100644 --- a/src/query/inc/qPercentile.h +++ b/src/query/inc/qPercentile.h @@ -64,11 +64,11 @@ typedef struct tMemBucket { __perc_hash_func_t hashFunc; } tMemBucket; -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType); +tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval); void tMemBucketDestroy(tMemBucket *pBucket); -void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); +int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size); double getPercentile(tMemBucket *pMemBucket, double percent); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 6de3c7c0e5..314159484d 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -50,14 +50,16 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); - SQuery *pQuery = pRuntimeEnv->pQuery; -// tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + SQuery *pQuery = pRuntimeEnv->pQuery; int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } +bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); +bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); + __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 79aec2f349..e0dd9d98f0 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -567,53 +567,53 @@ where_opt(A) ::= WHERE expr(X). {A = X;} %type expr {tSQLExpr*} %destructor expr {tSQLExprDestroy($$);} -expr(A) ::= LP expr(X) RP. {A = X; } +expr(A) ::= LP expr(X) RP. {A = X; } -expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} +expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} -expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);} expr(A) ::= MINUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} expr(A) ::= PLUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);} -expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);} -// normal functions: min(x) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { - A = tSQLExprCreateFunction(Y, &X, &E, X.type); -} +expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);} +expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); } +expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} +expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);} -// this is for: count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { - A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); -} +// ordinary functions: min(x), max(x), top(k, 20) +expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); } -//binary expression: a+2, b+3 -expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } +// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation +expr(A) ::= ID(X) LP STAR RP(Y). { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); } -//binary relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} +// is (not) null expression +expr(A) ::= expr(X) IS NULL. {A = tSQLExprCreate(X, NULL, TK_ISNULL);} +expr(A) ::= expr(X) IS NOT NULL. {A = tSQLExprCreate(X, NULL, TK_NOTNULL);} -//binary arithmetic expression +// relational expression +expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} +expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} +expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} +expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} +expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} +expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} + +expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} +expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } + +// binary arithmetic expression expr(A) ::= expr(X) PLUS expr(Y). {A = tSQLExprCreate(X, Y, TK_PLUS); } expr(A) ::= expr(X) MINUS expr(Y). {A = tSQLExprCreate(X, Y, TK_MINUS); } expr(A) ::= expr(X) STAR expr(Y). {A = tSQLExprCreate(X, Y, TK_STAR); } expr(A) ::= expr(X) SLASH expr(Y). {A = tSQLExprCreate(X, Y, TK_DIVIDE);} expr(A) ::= expr(X) REM expr(Y). {A = tSQLExprCreate(X, Y, TK_REM); } -//like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } +// like expression +expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } //in expression expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); } @@ -625,9 +625,9 @@ expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, %destructor expritem {tSQLExprDestroy($$);} exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);} -exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} -expritem(A) ::= expr(X). {A = X;} -expritem(A) ::= . {A = 0;} +exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} +expritem(A) ::= expr(X). {A = X;} +expritem(A) ::= . {A = 0;} ///////////////////////////////////reset query cache////////////////////////////////////// cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c314087179..a78bb7ec51 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -177,7 +177,7 @@ typedef struct SQLFunctionCtx { int16_t outputType; int16_t outputBytes; // size of results, determined by function and input column data type bool hasNull; // null value exist in current block - bool requireNull; // require null in some function + bool requireNull; // require null in some function int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 634f014d97..63411aaf3f 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -188,6 +188,10 @@ uint8_t getBinaryExprOptr(SStrToken *pToken) { return TSDB_BINARY_OP_REMAINDER; case TK_LIKE: return TSDB_RELATION_LIKE; + case TK_ISNULL: + return TSDB_RELATION_ISNULL; + case TK_NOTNULL: + return TSDB_RELATION_NOTNULL; default: { return 0; } } } @@ -486,29 +490,42 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr } else { int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - bool comp = true; + bool comp = true; int32_t ret = 0; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + if (comp) { ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); assert(ret <= 0); } - + if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; // no need to compare anymore } } + } else { + assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); + if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || + (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { + STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } } } - free(cond.start); + free(cond.start); free(cond.end); tSkipListDestroyIter(iter); } @@ -683,6 +700,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, char * pData = SL_GET_NODE_DATA(pNode); tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); + // todo speed up by using hash if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { @@ -714,7 +732,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S // column project if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { - assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d48d7d5ea1..b873714c49 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -200,14 +200,28 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; char *pElem = (char*)pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; - if (isNull(pElem, pFilterInfo->info.type)) { - return false; - } bool qualified = false; for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + bool isnull = isNull(pElem, pFilterInfo->info.type); + if (isnull) { + if (pFilterElem->fp == isNull_filter) { + qualified = true; + break; + } else { + continue; + } + } else { + if (pFilterElem->fp == notNull_filter) { + qualified = true; + break; + } else if (pFilterElem->fp == isNull_filter) { + continue; + } + } + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { qualified = true; break; @@ -504,7 +518,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); - w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); + w = pWindowRes->win; } if (w.skey > ts || w.ekey < ts) { @@ -610,7 +624,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->skey = win->skey; + pWindowRes->win = (*win); setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; @@ -683,12 +697,12 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe continue; } - TSKEY ekey = pResult->skey + pWindowResInfo->interval; + TSKEY ekey = pResult->win.ekey; if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + (pResult->win.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->skey; + skey = pResult->win.skey; break; } } @@ -701,7 +715,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { @@ -729,9 +743,9 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - int32_t num = -1; + int32_t num = -1; int32_t order = pQuery->order.order; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; @@ -765,27 +779,24 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, - int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, + int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].base.functionId; - pCtx[k].nStartQueryTimestamp = pWin->skey; pCtx[k].size = forwardStep; pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsBuf[offset]; + pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; } // not a whole block involved in query processing, statistics data can not be used - if (forwardStep != numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } + pCtx[k].preAggVals.isSet = (forwardStep == numOfTotal); if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); @@ -910,19 +921,11 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas char *dataBlock = NULL; SQuery *pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; int32_t functionId = pQuery->pSelectExpr[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pSelectExpr[col]; - // set the start offset to be the lowest start position, no matter asc/desc query order - if (QUERY_IS_ASC_QUERY(pQuery)) { - pCtx->startOffset = pQuery->pos; - } else { - pCtx->startOffset = pQuery->pos - (size - 1); - } - sas->offset = 0; sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; @@ -1002,7 +1005,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (QUERY_IS_INTERVAL_QUERY(pQuery)/* && tsCols != NULL*/) { + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { TSKEY ts = TSKEY_INITIAL_VAL; if (tsCols == NULL) { @@ -1094,8 +1097,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int64_t v = -1; // not assign result buffer yet, add new result buffer + char* d = pData; + int16_t len = bytes; + if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) { + d = varDataVal(pData); + len = varDataLen(pData); + } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); + qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo); + + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true); + if (pWindowRes == NULL) { + return -1; + } + + int64_t v = -1; switch(type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(pData); break; @@ -1104,12 +1124,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true); - if (pWindowRes == NULL) { - return -1; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + pWindowRes->key = malloc(varDataTLen(pData)); + varDataCopy(pWindowRes->key, pData); + } else { + pWindowRes->win.skey = v; + pWindowRes->win.ekey = v; } - pWindowRes->skey = v; assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { @@ -1180,7 +1202,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { #if defined(_DEBUG_VIEW) printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); #endif @@ -1461,12 +1483,15 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->preAggVals.dataBlockLoaded = (inputData != NULL); // limit/offset query will affect this value - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0; pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; + // minimum value no matter ascending/descending order query + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1); + assert(pCtx->startOffset >= 0); + uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = tsCol; + pCtx->ptsList = &tsCol[pCtx->startOffset]; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { @@ -1951,36 +1976,36 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo // todo handle the case the the order irrelevant query type mixed up with order critical query type // descending order query for last_row query - if (isFirstLastRowQuery(pQuery) && !QUERY_IS_ASC_QUERY(pQuery)) { + if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), pQuery->order.order, TSDB_ORDER_ASC); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); pQuery->order.order = TSDB_ORDER_ASC; - assert (pQuery->window.skey <= pQuery->window.ekey); + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + return; + } + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } doExchangeTimeWindow(pQInfo, &pQuery->window); return; } - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && !QUERY_IS_ASC_QUERY(pQuery)) { - pQuery->order.order = TSDB_ORDER_ASC; - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - assert (pQuery->window.skey <= pQuery->window.ekey); - - doExchangeTimeWindow(pQInfo, &pQuery->window); - return; - } - - if (isPointInterpoQuery(pQuery) && (pQuery->intervalTime == 0) && !QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, - pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } pQuery->order.order = TSDB_ORDER_ASC; - - assert (pQuery->window.skey <= pQuery->window.ekey); - doExchangeTimeWindow(pQInfo, &pQuery->window); return; } @@ -2185,43 +2210,43 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { return false; } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) { +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { SQuery *pQuery = pRuntimeEnv->pQuery; - uint32_t status = 0; + *status = 0; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) { - status = BLK_DATA_ALL_NEEDED; + *status = BLK_DATA_ALL_NEEDED; } else { // check if this data block is required to load // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { - status = BLK_DATA_ALL_NEEDED; + *status = BLK_DATA_ALL_NEEDED; } - if (status != BLK_DATA_ALL_NEEDED) { + if ((*status) != BLK_DATA_ALL_NEEDED) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base; int32_t functionId = pSqlFunc->functionId; int32_t colId = pSqlFunc->colInfo.colId; - status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); - if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { + (*status) |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); + if (((*status) & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { break; } } } } - if (status == BLK_DATA_NO_NEEDED) { + if ((*status) == BLK_DATA_NO_NEEDED) { qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pRuntimeEnv->summary.discardBlocks += 1; - } else if (status == BLK_DATA_STATIS_NEEDED) { - if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - // return DISK_DATA_LOAD_FAILED; - } + } else if ((*status) == BLK_DATA_STATIS_NEEDED) { + + // this function never returns error? + tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); pRuntimeEnv->summary.loadBlockStatis += 1; @@ -2230,24 +2255,26 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { - assert(status == BLK_DATA_ALL_NEEDED); + assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; - if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - } + tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - return BLK_DATA_DISCARD; + (*status) = BLK_DATA_DISCARD; } pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); + if (*pDataBlock == NULL) { + return terrno; + } } return TSDB_CODE_SUCCESS; @@ -2365,16 +2392,16 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; } - + // set the pCtx output buffer position pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; - + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } } - + qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); @@ -2431,15 +2458,18 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { ensureOutputBuffer(pRuntimeEnv, &blockInfo); SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; - if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { - pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; - continue; + SArray * pDataBlock = NULL; + uint32_t status = 0; + + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); + if (ret != TSDB_CODE_SUCCESS) { + break; } - if (terrno != TSDB_CODE_SUCCESS) { // load data block failed, abort query - longjmp(pRuntimeEnv->env, terrno); - break; + if (status == BLK_DATA_DISCARD) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; } // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition @@ -2806,6 +2836,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // all results have been return to client, try next group if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.pageId = 0; pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group @@ -2920,11 +2951,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { pTableList[numOfTables++] = item; - tid = TSDB_TABLEID(item->pTable)->tid; - pageList = list; } } @@ -2993,7 +3024,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->skey); + assert(ts == pWindowRes->win.skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -3357,7 +3388,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - + memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes)); pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; @@ -4354,32 +4385,6 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { return true; } -static void freeTableQueryInfo(STableGroupInfo* pTableGroupInfo) { - if (pTableGroupInfo->pGroupList == NULL) { - assert(pTableGroupInfo->numOfTables == 0); - } else { - size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *p = taosArrayGetP(pTableGroupInfo->pGroupList, i); - - size_t num = taosArrayGetSize(p); - for(int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(p, j); - destroyTableQueryInfo(item); - } - - taosArrayDestroy(p); - } - - taosArrayDestroy(pTableGroupInfo->pGroupList); - pTableGroupInfo->pGroupList = NULL; - pTableGroupInfo->numOfTables = 0; - } - - taosHashCleanup(pTableGroupInfo->map); - pTableGroupInfo->map = NULL; -} - static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4415,22 +4420,20 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); - if (pRuntimeEnv->pQueryHandle == NULL) { // no data in current stable, clear all - freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo); - } else { // update the query time window - pQuery->window = cond.twindow; - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); + // update the query time window + pQuery->window = cond.twindow; - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); - pCheckInfo->win = pQuery->window; - pCheckInfo->lastKey = pCheckInfo->win.skey; - } + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; } } } else if (isPointInterpoQuery(pQuery)) { @@ -4484,12 +4487,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); - setQueryStatus(pQuery, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4612,7 +4609,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; SQueryCostInfo* summary = &pRuntimeEnv->summary; - + int64_t st = taosGetTimestampMs(); TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; @@ -4622,7 +4619,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { while (tsdbNextDataBlock(pQueryHandle)) { summary->totalBlocks += 1; - + if (IS_QUERY_KILLED(pQInfo)) { longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -4651,15 +4648,23 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; - if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) { - pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step; + SArray * pDataBlock = NULL; + uint32_t status = 0; + + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); + if (ret != TSDB_CODE_SUCCESS) { + break; + } + + if (status == BLK_DATA_DISCARD) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; continue; } summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " "lastKey:%" PRId64, pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, @@ -6383,10 +6388,25 @@ static void freeQInfo(SQInfo *pQInfo) { taosTFree(pQuery); } - freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo); + // todo refactor, extract method to destroytableDataInfo + if (pQInfo->tableqinfoGroupInfo.pGroupList != NULL) { + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = GET_TABLEGROUP(pQInfo, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(p, j); + destroyTableQueryInfo(item); + } + + taosArrayDestroy(p); + } + } taosTFree(pQInfo->pBuf); - + taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList); + taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 192a31ebf5..1d3120ead4 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -502,22 +502,22 @@ FORCE_INLINE int32_t compare_sd(tOrderDescriptor *pDescriptor, int32_t numOfRows return compare_d(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data); } -static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2) { +static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2, void* buf) { for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) { void *first = COLMODEL_GET_VAL(data1, pColumnModel, count, s1, i); void *second = COLMODEL_GET_VAL(data1, pColumnModel, count, s2, i); SSchema* pSchema = &pColumnModel->pFields[i].field; - tsDataSwap(first, second, pSchema->type, pSchema->bytes); + tsDataSwap(first, second, pSchema->type, pSchema->bytes, buf); } } static void tColDataInsertSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn) { + __col_compar_fn_t compareFn, void* buf) { for (int32_t i = start + 1; i <= end; ++i) { for (int32_t j = i; j > start; --j) { if (compareFn(pDescriptor, numOfRows, j, j - 1, data) == -1) { - swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j); + swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j, buf); } else { break; } @@ -553,7 +553,7 @@ static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx, } static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - __col_compar_fn_t compareFn) { + __col_compar_fn_t compareFn, void* buf) { int32_t midIdx = ((end - start) >> 1) + start; #if defined(_DEBUG_VIEW) @@ -567,15 +567,16 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx); #endif + SColumnModel* pModel = pDescriptor->pColumnModel; if (compareFn(pDescriptor, numOfRows, midIdx, start, data) == 1) { - swap(pDescriptor->pColumnModel, numOfRows, start, data, midIdx); + swap(pModel, numOfRows, start, data, midIdx, buf); } if (compareFn(pDescriptor, numOfRows, midIdx, end, data) == 1) { - swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, start); - swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, end); + swap(pModel, numOfRows, midIdx, data, start, buf); + swap(pModel, numOfRows, midIdx, data, end, buf); } else if (compareFn(pDescriptor, numOfRows, start, end, data) == 1) { - swap(pDescriptor->pColumnModel, numOfRows, start, data, end); + swap(pModel, numOfRows, start, data, end, buf); } assert(compareFn(pDescriptor, numOfRows, midIdx, start, data) <= 0 && @@ -626,32 +627,20 @@ static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t printf("\n"); } -static int32_t qsort_call = 0; - -void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, - int32_t orderType) { - // short array sort, incur another sort procedure instead of quick sort process - __col_compar_fn_t compareFn = (orderType == TSDB_ORDER_ASC) ? compare_sa : compare_sd; - - if (end - start + 1 <= 8) { - tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn); - return; - } - +static void columnwiseQSortImpl(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, + int32_t orderType, __col_compar_fn_t compareFn, void* buf) { #ifdef _DEBUG_VIEW -// printf("before sort:\n"); -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + printf("before sort:\n"); + tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif int32_t s = start, e = end; - median(pDescriptor, numOfRows, start, end, data, compareFn); + median(pDescriptor, numOfRows, start, end, data, compareFn, buf); #ifdef _DEBUG_VIEW -// printf("%s called: %d\n", __FUNCTION__, qsort_call++); + // printf("%s called: %d\n", __FUNCTION__, qsort_call++); #endif - UNUSED(qsort_call); - int32_t end_same = end; int32_t start_same = start; @@ -663,17 +652,17 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta } if (ret == 0 && e != end_same) { - swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--); + swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--, buf); } e--; } if (e != s) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e); + swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); } #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif while (s < e) { @@ -683,16 +672,16 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta } if (ret == 0 && s != start_same) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++); + swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++, buf); } s++; } if (s != e) { - swap(pDescriptor->pColumnModel, numOfRows, s, data, e); + swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf); } #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif } @@ -702,14 +691,14 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta int32_t right = end; while (right > end_same && left <= end_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--); + swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); } // (pivotal+1) + steps of number that are identical pivotal rightx += (end - end_same); #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif } @@ -719,26 +708,52 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta int32_t right = e - 1; while (left < start_same && right >= start_same) { - swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--); + swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf); } // (pivotal-1) - steps of number that are identical pivotal leftx -= (start_same - start); #ifdef _DEBUG_VIEW -// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); + // tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1); #endif } if (leftx > start) { - tColDataQSort(pDescriptor, numOfRows, start, leftx, data, orderType); + columnwiseQSortImpl(pDescriptor, numOfRows, start, leftx, data, orderType, compareFn, buf); } if (rightx < end) { - tColDataQSort(pDescriptor, numOfRows, rightx, end, data, orderType); + columnwiseQSortImpl(pDescriptor, numOfRows, rightx, end, data, orderType, compareFn, buf); } } +void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t order) { + // short array sort, incur another sort procedure instead of quick sort process + __col_compar_fn_t compareFn = (order == TSDB_ORDER_ASC) ? compare_sa : compare_sd; + + SColumnModel* pModel = pDescriptor->pColumnModel; + + size_t width = 0; + for(int32_t i = 0; i < pModel->numOfCols; ++i) { + SSchema* pSchema = &pModel->pFields[i].field; + if (width < pSchema->bytes) { + width = pSchema->bytes; + } + } + + char* buf = malloc(width); + assert(width > 0 && buf != NULL); + + if (end - start + 1 <= 8) { + tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn, buf); + } else { + columnwiseQSortImpl(pDescriptor, numOfRows, start, end, data, order, compareFn, buf); + } + + free(buf); +} + /* * deep copy of sschema */ diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index 7e9f5c7da5..b6050dddd8 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -284,6 +284,14 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; } +//////////////////////////////////////////////////////////////// +bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) { + return true; +} + +bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) { + return true; +} //////////////////////////////////////////////////////////////// @@ -398,6 +406,8 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i8, nequal_i8, NULL, + isNull_filter, + notNull_filter, }; bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -409,6 +419,8 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i16, nequal_i16, NULL, + isNull_filter, + notNull_filter, }; bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -420,6 +432,8 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i32, nequal_i32, NULL, + isNull_filter, + notNull_filter, }; bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -431,6 +445,8 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_i64, nequal_i64, NULL, + isNull_filter, + notNull_filter, }; bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -442,6 +458,8 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_ds, nequal_ds, NULL, + isNull_filter, + notNull_filter, }; bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { @@ -453,6 +471,8 @@ bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) largeEqual_dd, nequal_dd, NULL, + isNull_filter, + notNull_filter, }; bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { @@ -464,6 +484,8 @@ bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) NULL, nequal_str, like_str, + isNull_filter, + notNull_filter, }; bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { @@ -475,6 +497,8 @@ bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxva NULL, nequal_nchar, like_nchar, + isNull_filter, + notNull_filter, }; bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 9629f24cc2..c1b0bd5706 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -179,7 +179,7 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (pRight != NULL && pLeft != NULL) { + if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { char* endPos = pRight->token.z + pRight->token.n; pExpr->token.z = pLeft->token.z; pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); @@ -275,6 +275,11 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { } else { pExpr->nSQLOptr = optrType; pExpr->pLeft = pLeft; + + if (pRight == NULL) { + pRight = calloc(1, sizeof(tSQLExpr)); + } + pExpr->pRight = pRight; } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 1ce5861e52..3a8be781d5 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -70,6 +70,33 @@ static void resetBoundingBox(MinMaxEntry* range, int32_t type) { } } +static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) { + if (minval > maxval) { + return -1; + } + + switch(type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + range->iMinVal = (int32_t) minval; + range->iMaxVal = (int32_t) maxval; + break; + + case TSDB_DATA_TYPE_BIGINT: + range->i64MinVal = (int64_t) minval; + range->i64MaxVal = (int64_t) maxval; + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + range->dMinVal = minval; + range->dMaxVal = maxval; + break; + } + + return 0; +} + static void resetPosInfo(SSlotInfo* pInfo) { pInfo->size = 0; pInfo->pageId = -1; @@ -135,6 +162,11 @@ int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) { return index; } else { + // out of range + if (v < pBucket->range.i64MinVal || v > pBucket->range.i64MaxVal) { + return -1; + } + // todo hash for bigint and float and double int64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; if (span < pBucket->numOfSlots) { @@ -179,6 +211,11 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { return index; } else { + // out of range + if (v < pBucket->range.iMinVal || v > pBucket->range.iMaxVal) { + return -1; + } + // divide a range of [iMinVal, iMaxVal] into 1024 buckets int32_t span = pBucket->range.iMaxVal - pBucket->range.iMinVal; if (span < pBucket->numOfSlots) { @@ -209,6 +246,12 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { double posx = (v + DBL_MAX) / x; return ((int32_t)posx) % pBucket->numOfSlots; } else { + + // out of range + if (v < pBucket->range.dMinVal || v > pBucket->range.dMaxVal) { + return -1; + } + // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { @@ -262,7 +305,7 @@ static void resetSlotInfo(tMemBucket* pBucket) { } } -tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) { +tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); if (pBucket == NULL) { return NULL; @@ -278,9 +321,14 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) { pBucket->maxCapacity = 200000; + if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { + uError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); + free(pBucket); + return NULL; + } + pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes; pBucket->comparFn = getKeyComparFunc(pBucket->type); - resetBoundingBox(&pBucket->range, pBucket->type); pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { @@ -395,23 +443,25 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) { /* * in memory bucket, we only accept data array list */ -void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { +int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { assert(pBucket != NULL && data != NULL && size > 0); + pBucket->total += (int32_t)size; int32_t bytes = pBucket->bytes; - for (int32_t i = 0; i < size; ++i) { char *d = (char *) data + i * bytes; - int32_t slotIdx = (pBucket->hashFunc)(pBucket, d); - assert(slotIdx >= 0); + int32_t index = (pBucket->hashFunc)(pBucket, d); + if (index == -1) { // the value is out of range, do not add it into bucket + return -1; + } - tMemBucketSlot *pSlot = &pBucket->pSlots[slotIdx]; + tMemBucketSlot *pSlot = &pBucket->pSlots[index]; tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); // ensure available memory pages to allocate - int32_t groupId = getGroupId(pBucket->numOfSlots, slotIdx, pBucket->times); + int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times); int32_t pageId = -1; if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { @@ -432,10 +482,12 @@ void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data->num += 1; pSlot->info.size += 1; } + + return 0; } //////////////////////////////////////////////////////////////////////////////////////////// -static void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) { +static UNUSED_FUNC void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) { *minVal = DBL_MAX; *maxVal = -DBL_MAX; @@ -681,16 +733,29 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { // find the min/max value, no need to scan all data in bucket if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) { - double minx = 0, maxx = 0; - findMaxMinValue(pMemBucket, &maxx, &minx); + MinMaxEntry* pRange = &pMemBucket->range; - return fabs(percent - 100) < DBL_EPSILON ? maxx : minx; + switch(pMemBucket->type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal; + case TSDB_DATA_TYPE_BIGINT: { + double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal); + return v; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal; + default: + return -1; + } } double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0); - int32_t orderIdx = (int32_t)percentVal; // do put data by using buckets + int32_t orderIdx = (int32_t)percentVal; return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx); } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 509362863c..c195a0b76c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -126,11 +126,26 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - + + int16_t type = pWindowResInfo->type; + + char *key = NULL; + int16_t bytes = -1; + for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); + + // todo refactor + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + key = varDataVal(pResult->key); + bytes = varDataLen(pResult->key); + } else { + key = (char*) &pResult->win.skey; + bytes = tDataTypeDesc[pWindowResInfo->type].nSize; + } + + taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes); } else { break; } @@ -150,15 +165,24 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } pWindowResInfo->size = remain; + for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, - tDataTypeDesc[pWindowResInfo->type].nSize); + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + key = varDataVal(pResult->key); + bytes = varDataLen(pResult->key); + } else { + key = (char*) &pResult->win.skey; + bytes = tDataTypeDesc[pWindowResInfo->type].nSize; + } + + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes); assert(p != NULL); + int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, - (char *)&v, sizeof(int32_t)); + taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t)); } pWindowResInfo->curIndex = -1; @@ -207,20 +231,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; - + int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + TSKEY ekey = pWindowResInfo->pResult[i].win.ekey; while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) { ++i; } } @@ -258,7 +281,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; pWindowRes->closed = false; - pWindowRes->skey = TSKEY_INITIAL_VAL; + pWindowRes->win = TSWINDOW_INITIALIZER; } /** @@ -268,7 +291,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->skey = src->skey; + dst->win = src->win; dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 373e57963c..35e36032d4 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -126,17 +126,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 244 -#define YYNRULE 225 +#define YYNSTATE 246 +#define YYNRULE 227 #define YYNTOKEN 206 -#define YY_MAX_SHIFT 243 -#define YY_MIN_SHIFTREDUCE 403 -#define YY_MAX_SHIFTREDUCE 627 -#define YY_ERROR_ACTION 628 -#define YY_ACCEPT_ACTION 629 -#define YY_NO_ACTION 630 -#define YY_MIN_REDUCE 631 -#define YY_MAX_REDUCE 855 +#define YY_MAX_SHIFT 245 +#define YY_MIN_SHIFTREDUCE 407 +#define YY_MAX_SHIFTREDUCE 633 +#define YY_ERROR_ACTION 634 +#define YY_ACCEPT_ACTION 635 +#define YY_NO_ACTION 636 +#define YY_MIN_REDUCE 637 +#define YY_MAX_REDUCE 863 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -202,121 +202,122 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (549) +#define YY_ACTTAB_COUNT (556) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 731, 444, 221, 729, 730, 629, 243, 510, 732, 445, - /* 10 */ 734, 735, 733, 41, 43, 526, 35, 36, 523, 11, - /* 20 */ 524, 29, 525, 444, 199, 39, 37, 40, 38, 155, - /* 30 */ 241, 445, 748, 34, 33, 219, 218, 32, 31, 30, - /* 40 */ 41, 43, 761, 35, 36, 136, 172, 173, 29, 137, - /* 50 */ 21, 199, 39, 37, 40, 38, 184, 141, 160, 843, - /* 60 */ 34, 33, 839, 772, 32, 31, 30, 404, 405, 406, - /* 70 */ 407, 408, 409, 410, 411, 412, 413, 414, 415, 242, - /* 80 */ 41, 43, 230, 35, 36, 746, 62, 137, 29, 137, - /* 90 */ 21, 199, 39, 37, 40, 38, 159, 843, 27, 842, - /* 100 */ 34, 33, 56, 838, 32, 31, 30, 105, 43, 8, - /* 110 */ 35, 36, 63, 115, 769, 29, 761, 527, 199, 39, - /* 120 */ 37, 40, 38, 168, 539, 747, 583, 34, 33, 18, - /* 130 */ 156, 32, 31, 30, 16, 210, 236, 235, 209, 208, - /* 140 */ 207, 234, 206, 233, 232, 231, 205, 727, 105, 715, - /* 150 */ 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, - /* 160 */ 726, 35, 36, 798, 837, 194, 29, 177, 157, 199, - /* 170 */ 39, 37, 40, 38, 181, 180, 21, 21, 34, 33, - /* 180 */ 444, 12, 32, 31, 30, 164, 596, 750, 445, 587, - /* 190 */ 153, 590, 154, 593, 105, 164, 596, 21, 17, 587, - /* 200 */ 150, 590, 196, 593, 60, 26, 90, 89, 144, 169, - /* 210 */ 217, 747, 747, 16, 149, 236, 235, 161, 162, 167, - /* 220 */ 234, 198, 233, 232, 231, 142, 670, 161, 162, 128, - /* 230 */ 222, 542, 747, 164, 596, 17, 143, 587, 750, 590, - /* 240 */ 105, 593, 26, 39, 37, 40, 38, 100, 170, 145, - /* 250 */ 797, 34, 33, 101, 26, 32, 31, 30, 32, 31, - /* 260 */ 30, 78, 183, 564, 565, 161, 162, 230, 589, 152, - /* 270 */ 592, 76, 80, 85, 88, 79, 240, 239, 97, 34, - /* 280 */ 33, 82, 42, 32, 31, 30, 118, 119, 70, 66, - /* 290 */ 69, 237, 42, 595, 679, 163, 61, 128, 132, 130, - /* 300 */ 93, 92, 91, 595, 671, 187, 585, 128, 594, 588, - /* 310 */ 750, 591, 171, 534, 47, 216, 215, 146, 594, 555, - /* 320 */ 186, 147, 556, 46, 613, 148, 14, 597, 13, 139, - /* 330 */ 42, 13, 50, 48, 3, 135, 75, 74, 140, 516, - /* 340 */ 515, 595, 586, 46, 22, 138, 203, 10, 9, 51, - /* 350 */ 22, 852, 530, 528, 531, 529, 594, 87, 86, 749, - /* 360 */ 808, 807, 165, 804, 803, 166, 771, 741, 220, 776, - /* 370 */ 763, 778, 102, 790, 789, 116, 117, 114, 681, 204, - /* 380 */ 133, 24, 213, 678, 214, 851, 72, 850, 848, 26, - /* 390 */ 120, 699, 25, 23, 185, 95, 134, 668, 81, 551, - /* 400 */ 666, 83, 84, 664, 188, 663, 174, 129, 661, 660, - /* 410 */ 659, 658, 657, 649, 131, 655, 653, 192, 52, 651, - /* 420 */ 760, 57, 49, 58, 791, 44, 197, 195, 193, 191, - /* 430 */ 189, 28, 212, 77, 223, 224, 225, 226, 227, 228, - /* 440 */ 229, 238, 627, 176, 175, 626, 201, 178, 179, 53, - /* 450 */ 625, 618, 182, 536, 64, 151, 186, 67, 552, 55, - /* 460 */ 103, 158, 662, 59, 200, 94, 96, 123, 700, 121, - /* 470 */ 126, 106, 107, 122, 124, 125, 127, 112, 108, 109, - /* 480 */ 113, 745, 110, 656, 111, 1, 2, 190, 5, 557, - /* 490 */ 104, 19, 6, 598, 20, 4, 15, 7, 65, 485, - /* 500 */ 202, 481, 479, 478, 477, 474, 448, 211, 68, 45, - /* 510 */ 71, 73, 22, 512, 511, 509, 54, 469, 467, 459, - /* 520 */ 465, 461, 463, 457, 455, 484, 483, 482, 480, 476, - /* 530 */ 475, 46, 446, 419, 417, 631, 630, 630, 630, 630, - /* 540 */ 630, 630, 630, 630, 630, 630, 630, 98, 99, + /* 0 */ 737, 448, 11, 735, 736, 635, 245, 448, 738, 449, + /* 10 */ 740, 741, 739, 35, 36, 449, 37, 38, 155, 243, + /* 20 */ 165, 29, 137, 136, 200, 41, 39, 43, 40, 105, + /* 30 */ 514, 160, 851, 34, 33, 778, 137, 32, 31, 30, + /* 40 */ 35, 36, 767, 37, 38, 159, 851, 165, 29, 767, + /* 50 */ 105, 200, 41, 39, 43, 40, 185, 157, 221, 220, + /* 60 */ 34, 33, 137, 156, 32, 31, 30, 35, 36, 448, + /* 70 */ 37, 38, 850, 141, 165, 29, 756, 449, 200, 41, + /* 80 */ 39, 43, 40, 197, 78, 60, 775, 34, 33, 232, + /* 90 */ 232, 32, 31, 30, 21, 41, 39, 43, 40, 32, + /* 100 */ 31, 30, 56, 34, 33, 847, 803, 32, 31, 30, + /* 110 */ 21, 21, 105, 408, 409, 410, 411, 412, 413, 414, + /* 120 */ 415, 416, 417, 418, 419, 244, 587, 169, 36, 753, + /* 130 */ 37, 38, 223, 50, 165, 29, 21, 62, 200, 41, + /* 140 */ 39, 43, 40, 170, 219, 753, 753, 34, 33, 27, + /* 150 */ 51, 32, 31, 30, 8, 37, 38, 63, 115, 165, + /* 160 */ 29, 101, 754, 200, 41, 39, 43, 40, 804, 224, + /* 170 */ 195, 753, 34, 33, 168, 846, 32, 31, 30, 16, + /* 180 */ 212, 238, 237, 211, 210, 209, 236, 208, 235, 234, + /* 190 */ 233, 207, 733, 756, 721, 722, 723, 724, 725, 726, + /* 200 */ 727, 728, 729, 730, 731, 732, 164, 600, 12, 239, + /* 210 */ 591, 17, 594, 188, 597, 105, 164, 600, 26, 559, + /* 220 */ 591, 845, 594, 46, 597, 34, 33, 150, 756, 32, + /* 230 */ 31, 30, 21, 90, 89, 144, 568, 569, 161, 162, + /* 240 */ 171, 149, 199, 76, 80, 85, 88, 79, 161, 162, + /* 250 */ 164, 600, 546, 82, 591, 589, 594, 100, 597, 242, + /* 260 */ 241, 97, 17, 16, 26, 238, 237, 752, 201, 26, + /* 270 */ 236, 61, 235, 234, 233, 118, 119, 70, 66, 69, + /* 280 */ 538, 676, 161, 162, 128, 530, 178, 187, 527, 184, + /* 290 */ 528, 590, 529, 182, 181, 593, 152, 596, 132, 130, + /* 300 */ 93, 92, 91, 42, 172, 543, 685, 218, 217, 128, + /* 310 */ 18, 163, 677, 42, 599, 128, 173, 174, 560, 619, + /* 320 */ 153, 47, 14, 13, 599, 592, 601, 595, 520, 598, + /* 330 */ 13, 519, 46, 154, 205, 22, 75, 74, 22, 598, + /* 340 */ 48, 10, 9, 534, 532, 535, 533, 42, 87, 86, + /* 350 */ 3, 139, 860, 140, 755, 142, 143, 603, 599, 147, + /* 360 */ 148, 146, 135, 145, 138, 814, 813, 166, 810, 809, + /* 370 */ 167, 777, 747, 598, 222, 769, 782, 784, 102, 796, + /* 380 */ 114, 116, 795, 117, 687, 206, 133, 531, 186, 26, + /* 390 */ 24, 95, 215, 684, 216, 859, 72, 858, 856, 120, + /* 400 */ 705, 25, 555, 23, 134, 674, 81, 672, 83, 189, + /* 410 */ 84, 670, 193, 669, 175, 52, 129, 667, 49, 666, + /* 420 */ 665, 106, 664, 44, 663, 107, 655, 131, 198, 766, + /* 430 */ 196, 661, 659, 657, 194, 57, 58, 797, 192, 190, + /* 440 */ 28, 214, 77, 225, 226, 227, 228, 229, 230, 203, + /* 450 */ 53, 231, 240, 633, 151, 177, 64, 67, 176, 668, + /* 460 */ 632, 179, 180, 631, 624, 187, 123, 183, 122, 706, + /* 470 */ 121, 125, 124, 94, 127, 662, 126, 96, 1, 2, + /* 480 */ 540, 112, 108, 109, 751, 110, 113, 111, 59, 55, + /* 490 */ 556, 103, 158, 19, 191, 20, 561, 104, 5, 602, + /* 500 */ 6, 4, 604, 15, 202, 7, 204, 65, 489, 485, + /* 510 */ 483, 482, 481, 478, 452, 213, 68, 45, 71, 22, + /* 520 */ 516, 515, 513, 54, 473, 471, 463, 469, 465, 467, + /* 530 */ 73, 461, 459, 488, 487, 486, 484, 480, 479, 46, + /* 540 */ 450, 423, 421, 637, 636, 636, 98, 636, 636, 636, + /* 550 */ 636, 636, 636, 636, 636, 99, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 226, 1, 210, 229, 230, 207, 208, 5, 234, 9, - /* 10 */ 236, 237, 238, 13, 14, 2, 16, 17, 5, 260, - /* 20 */ 7, 21, 9, 1, 24, 25, 26, 27, 28, 209, - /* 30 */ 210, 9, 240, 33, 34, 33, 34, 37, 38, 39, - /* 40 */ 13, 14, 244, 16, 17, 260, 33, 34, 21, 260, - /* 50 */ 210, 24, 25, 26, 27, 28, 258, 260, 269, 270, - /* 60 */ 33, 34, 260, 210, 37, 38, 39, 45, 46, 47, - /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 13, 14, 78, 16, 17, 245, 247, 260, 21, 260, - /* 90 */ 210, 24, 25, 26, 27, 28, 269, 270, 259, 270, - /* 100 */ 33, 34, 102, 260, 37, 38, 39, 210, 14, 98, - /* 110 */ 16, 17, 101, 102, 261, 21, 244, 104, 24, 25, - /* 120 */ 26, 27, 28, 243, 103, 245, 99, 33, 34, 108, - /* 130 */ 258, 37, 38, 39, 85, 86, 87, 88, 89, 90, - /* 140 */ 91, 92, 93, 94, 95, 96, 97, 226, 210, 228, - /* 150 */ 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, - /* 160 */ 239, 16, 17, 266, 260, 268, 21, 126, 227, 24, - /* 170 */ 25, 26, 27, 28, 133, 134, 210, 210, 33, 34, - /* 180 */ 1, 44, 37, 38, 39, 1, 2, 246, 9, 5, - /* 190 */ 260, 7, 260, 9, 210, 1, 2, 210, 98, 5, - /* 200 */ 63, 7, 264, 9, 266, 105, 69, 70, 71, 243, - /* 210 */ 243, 245, 245, 85, 77, 87, 88, 33, 34, 227, - /* 220 */ 92, 37, 94, 95, 96, 260, 214, 33, 34, 217, - /* 230 */ 243, 37, 245, 1, 2, 98, 260, 5, 246, 7, - /* 240 */ 210, 9, 105, 25, 26, 27, 28, 98, 63, 260, - /* 250 */ 266, 33, 34, 210, 105, 37, 38, 39, 37, 38, - /* 260 */ 39, 72, 125, 115, 116, 33, 34, 78, 5, 132, - /* 270 */ 7, 64, 65, 66, 67, 68, 60, 61, 62, 33, - /* 280 */ 34, 74, 98, 37, 38, 39, 64, 65, 66, 67, - /* 290 */ 68, 227, 98, 109, 214, 59, 266, 217, 64, 65, - /* 300 */ 66, 67, 68, 109, 214, 262, 1, 217, 124, 5, - /* 310 */ 246, 7, 127, 99, 103, 130, 131, 260, 124, 99, - /* 320 */ 106, 260, 99, 103, 99, 260, 103, 99, 103, 260, - /* 330 */ 98, 103, 103, 122, 98, 260, 128, 129, 260, 99, - /* 340 */ 99, 109, 37, 103, 103, 260, 99, 128, 129, 120, - /* 350 */ 103, 246, 5, 5, 7, 7, 124, 72, 73, 246, - /* 360 */ 241, 241, 241, 241, 241, 241, 210, 242, 241, 210, - /* 370 */ 244, 210, 210, 267, 267, 210, 210, 248, 210, 210, - /* 380 */ 210, 210, 210, 210, 210, 210, 210, 210, 210, 105, - /* 390 */ 210, 210, 210, 210, 244, 59, 210, 210, 210, 109, - /* 400 */ 210, 210, 210, 210, 263, 210, 210, 210, 210, 210, - /* 410 */ 210, 210, 210, 210, 210, 210, 210, 263, 119, 210, - /* 420 */ 257, 211, 121, 211, 211, 118, 113, 117, 112, 111, - /* 430 */ 110, 123, 75, 84, 83, 49, 80, 82, 53, 81, - /* 440 */ 79, 75, 5, 5, 135, 5, 211, 135, 5, 211, - /* 450 */ 5, 86, 126, 99, 215, 211, 106, 215, 99, 107, - /* 460 */ 98, 1, 211, 103, 100, 212, 212, 219, 225, 224, - /* 470 */ 221, 256, 255, 223, 222, 220, 218, 250, 254, 253, - /* 480 */ 249, 244, 252, 211, 251, 216, 213, 98, 114, 99, - /* 490 */ 98, 103, 114, 99, 103, 98, 98, 98, 72, 9, - /* 500 */ 100, 5, 5, 5, 5, 5, 76, 15, 72, 16, - /* 510 */ 129, 129, 103, 5, 5, 99, 98, 5, 5, 5, - /* 520 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 530 */ 5, 103, 76, 59, 58, 0, 271, 271, 271, 271, - /* 540 */ 271, 271, 271, 271, 271, 271, 271, 21, 21, 271, - /* 550 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + /* 0 */ 226, 1, 260, 229, 230, 207, 208, 1, 234, 9, + /* 10 */ 236, 237, 238, 13, 14, 9, 16, 17, 209, 210, + /* 20 */ 20, 21, 260, 260, 24, 25, 26, 27, 28, 210, + /* 30 */ 5, 269, 270, 33, 34, 210, 260, 37, 38, 39, + /* 40 */ 13, 14, 244, 16, 17, 269, 270, 20, 21, 244, + /* 50 */ 210, 24, 25, 26, 27, 28, 258, 227, 33, 34, + /* 60 */ 33, 34, 260, 258, 37, 38, 39, 13, 14, 1, + /* 70 */ 16, 17, 270, 260, 20, 21, 246, 9, 24, 25, + /* 80 */ 26, 27, 28, 264, 72, 266, 261, 33, 34, 78, + /* 90 */ 78, 37, 38, 39, 210, 25, 26, 27, 28, 37, + /* 100 */ 38, 39, 102, 33, 34, 260, 266, 37, 38, 39, + /* 110 */ 210, 210, 210, 45, 46, 47, 48, 49, 50, 51, + /* 120 */ 52, 53, 54, 55, 56, 57, 99, 243, 14, 245, + /* 130 */ 16, 17, 210, 103, 20, 21, 210, 247, 24, 25, + /* 140 */ 26, 27, 28, 243, 243, 245, 245, 33, 34, 259, + /* 150 */ 120, 37, 38, 39, 98, 16, 17, 101, 102, 20, + /* 160 */ 21, 210, 240, 24, 25, 26, 27, 28, 266, 243, + /* 170 */ 268, 245, 33, 34, 227, 260, 37, 38, 39, 85, + /* 180 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 190 */ 96, 97, 226, 246, 228, 229, 230, 231, 232, 233, + /* 200 */ 234, 235, 236, 237, 238, 239, 1, 2, 44, 227, + /* 210 */ 5, 98, 7, 262, 9, 210, 1, 2, 105, 99, + /* 220 */ 5, 260, 7, 103, 9, 33, 34, 63, 246, 37, + /* 230 */ 38, 39, 210, 69, 70, 71, 115, 116, 33, 34, + /* 240 */ 63, 77, 37, 64, 65, 66, 67, 68, 33, 34, + /* 250 */ 1, 2, 37, 74, 5, 1, 7, 98, 9, 60, + /* 260 */ 61, 62, 98, 85, 105, 87, 88, 245, 15, 105, + /* 270 */ 92, 266, 94, 95, 96, 64, 65, 66, 67, 68, + /* 280 */ 99, 214, 33, 34, 217, 2, 126, 106, 5, 125, + /* 290 */ 7, 37, 9, 133, 134, 5, 132, 7, 64, 65, + /* 300 */ 66, 67, 68, 98, 127, 103, 214, 130, 131, 217, + /* 310 */ 108, 59, 214, 98, 109, 217, 33, 34, 99, 99, + /* 320 */ 260, 103, 103, 103, 109, 5, 99, 7, 99, 124, + /* 330 */ 103, 99, 103, 260, 99, 103, 128, 129, 103, 124, + /* 340 */ 122, 128, 129, 5, 5, 7, 7, 98, 72, 73, + /* 350 */ 98, 260, 246, 260, 246, 260, 260, 104, 109, 260, + /* 360 */ 260, 260, 260, 260, 260, 241, 241, 241, 241, 241, + /* 370 */ 241, 210, 242, 124, 241, 244, 210, 210, 210, 267, + /* 380 */ 248, 210, 267, 210, 210, 210, 210, 104, 244, 105, + /* 390 */ 210, 59, 210, 210, 210, 210, 210, 210, 210, 210, + /* 400 */ 210, 210, 109, 210, 210, 210, 210, 210, 210, 263, + /* 410 */ 210, 210, 263, 210, 210, 119, 210, 210, 121, 210, + /* 420 */ 210, 256, 210, 118, 210, 255, 210, 210, 113, 257, + /* 430 */ 117, 210, 210, 210, 112, 211, 211, 211, 111, 110, + /* 440 */ 123, 75, 84, 83, 49, 80, 82, 53, 81, 211, + /* 450 */ 211, 79, 75, 5, 211, 5, 215, 215, 135, 211, + /* 460 */ 5, 135, 5, 5, 86, 106, 219, 126, 223, 225, + /* 470 */ 224, 220, 222, 212, 218, 211, 221, 212, 216, 213, + /* 480 */ 99, 250, 254, 253, 244, 252, 249, 251, 103, 107, + /* 490 */ 99, 98, 1, 103, 98, 103, 99, 98, 114, 99, + /* 500 */ 114, 98, 104, 98, 100, 98, 100, 72, 9, 5, + /* 510 */ 5, 5, 5, 5, 76, 15, 72, 16, 129, 103, + /* 520 */ 5, 5, 99, 98, 5, 5, 5, 5, 5, 5, + /* 530 */ 129, 5, 5, 5, 5, 5, 5, 5, 5, 103, + /* 540 */ 76, 59, 58, 0, 271, 271, 21, 271, 271, 271, + /* 550 */ 271, 271, 271, 271, 271, 21, 271, 271, 271, 271, /* 560 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 570 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 580 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, @@ -336,83 +337,84 @@ static const YYCODETYPE yy_lookahead[] = { /* 720 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 730 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, /* 740 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - /* 750 */ 271, 271, 271, 271, 271, + /* 750 */ 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + /* 760 */ 271, 271, }; -#define YY_SHIFT_COUNT (243) +#define YY_SHIFT_COUNT (245) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (535) +#define YY_SHIFT_MAX (543) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 137, 49, 128, 184, 232, 179, 179, 179, 179, 179, - /* 10 */ 179, 0, 22, 232, 13, 13, 13, 100, 179, 179, - /* 20 */ 179, 179, 179, 189, 4, 4, 549, 194, 232, 232, - /* 30 */ 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, - /* 40 */ 232, 232, 232, 232, 232, 13, 13, 2, 2, 2, - /* 50 */ 2, 2, 2, 11, 2, 149, 179, 179, 179, 179, - /* 60 */ 148, 148, 21, 179, 179, 179, 179, 179, 179, 179, - /* 70 */ 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, - /* 80 */ 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, - /* 90 */ 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, - /* 100 */ 284, 336, 336, 290, 290, 336, 299, 301, 307, 313, - /* 110 */ 310, 316, 318, 320, 308, 284, 336, 336, 357, 357, - /* 120 */ 336, 349, 351, 386, 356, 355, 385, 358, 361, 336, - /* 130 */ 366, 336, 366, 549, 549, 27, 67, 67, 67, 94, - /* 140 */ 145, 218, 218, 218, 207, 246, 246, 246, 246, 222, - /* 150 */ 234, 185, 41, 221, 221, 216, 214, 220, 223, 225, - /* 160 */ 228, 263, 304, 305, 236, 211, 229, 240, 241, 247, - /* 170 */ 208, 219, 347, 348, 285, 437, 309, 438, 440, 312, - /* 180 */ 443, 445, 365, 326, 350, 354, 352, 360, 359, 362, - /* 190 */ 460, 389, 390, 392, 388, 374, 391, 378, 394, 397, - /* 200 */ 398, 364, 399, 400, 426, 490, 496, 497, 498, 499, - /* 210 */ 500, 430, 492, 436, 493, 381, 382, 409, 508, 509, - /* 220 */ 416, 418, 409, 512, 513, 514, 515, 516, 517, 518, - /* 230 */ 519, 520, 521, 522, 523, 524, 525, 428, 456, 526, - /* 240 */ 527, 474, 476, 535, + /* 0 */ 164, 94, 178, 205, 249, 6, 6, 6, 6, 6, + /* 10 */ 6, 0, 68, 249, 283, 283, 283, 113, 6, 6, + /* 20 */ 6, 6, 6, 12, 11, 11, 556, 215, 249, 249, + /* 30 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, + /* 40 */ 249, 249, 249, 249, 249, 283, 283, 25, 25, 25, + /* 50 */ 25, 25, 25, 56, 25, 159, 6, 6, 6, 6, + /* 60 */ 121, 121, 202, 6, 6, 6, 6, 6, 6, 6, + /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 100 */ 284, 332, 332, 293, 293, 332, 296, 297, 305, 315, + /* 110 */ 313, 322, 327, 329, 317, 284, 332, 332, 366, 366, + /* 120 */ 332, 358, 360, 395, 365, 364, 394, 367, 372, 332, + /* 130 */ 377, 332, 377, 556, 556, 27, 54, 54, 54, 114, + /* 140 */ 139, 70, 70, 70, 179, 192, 192, 192, 192, 211, + /* 150 */ 234, 177, 160, 62, 62, 199, 181, 120, 219, 220, + /* 160 */ 227, 290, 320, 254, 252, 253, 218, 30, 229, 232, + /* 170 */ 235, 208, 213, 338, 339, 276, 448, 323, 450, 455, + /* 180 */ 326, 457, 458, 378, 341, 359, 381, 382, 385, 391, + /* 190 */ 393, 491, 396, 397, 399, 390, 384, 392, 386, 400, + /* 200 */ 403, 398, 405, 404, 407, 406, 435, 499, 504, 505, + /* 210 */ 506, 507, 508, 438, 500, 444, 501, 389, 401, 416, + /* 220 */ 515, 516, 423, 425, 416, 519, 520, 521, 522, 523, + /* 230 */ 524, 526, 527, 528, 529, 530, 531, 532, 533, 436, + /* 240 */ 464, 525, 534, 482, 484, 543, }; #define YY_REDUCE_COUNT (134) -#define YY_REDUCE_MIN (-241) -#define YY_REDUCE_MAX (273) +#define YY_REDUCE_MIN (-258) +#define YY_REDUCE_MAX (266) static const short yy_reduce_ofst[] = { - /* 0 */ -202, -79, -226, -211, -173, -103, -62, -120, -34, -33, - /* 10 */ -13, -147, -180, -171, -59, -8, 64, -128, 43, -16, - /* 20 */ 30, -208, -160, 12, 80, 90, -161, -241, -215, -203, - /* 30 */ -198, -157, -96, -70, -68, -35, -24, -11, 57, 61, - /* 40 */ 65, 69, 75, 78, 85, 105, 113, 119, 120, 121, - /* 50 */ 122, 123, 124, 125, 127, 126, 156, 159, 161, 162, - /* 60 */ 106, 107, 129, 165, 166, 168, 169, 170, 171, 172, - /* 70 */ 173, 174, 175, 176, 177, 178, 180, 181, 182, 183, - /* 80 */ 186, 187, 188, 190, 191, 192, 193, 195, 196, 197, - /* 90 */ 198, 199, 200, 201, 202, 203, 204, 205, 206, 209, - /* 100 */ 150, 210, 212, 141, 154, 213, 163, 215, 217, 224, - /* 110 */ 226, 230, 233, 227, 231, 237, 235, 238, 239, 242, - /* 120 */ 244, 243, 245, 250, 248, 252, 255, 249, 258, 251, - /* 130 */ 253, 272, 254, 269, 273, + /* 0 */ -202, -34, -226, -238, -224, -98, -181, -116, -100, -99, + /* 10 */ -74, -175, -191, -198, -170, -53, -18, -195, -49, -160, + /* 20 */ 5, -78, 22, 67, 92, 98, -110, -258, -237, -187, + /* 30 */ -155, -85, -39, 60, 73, 91, 93, 95, 96, 99, + /* 40 */ 100, 101, 102, 103, 104, 106, 108, 124, 125, 126, + /* 50 */ 127, 128, 129, 130, 133, 131, 161, 166, 167, 168, + /* 60 */ 112, 115, 132, 171, 173, 174, 175, 176, 180, 182, + /* 70 */ 183, 184, 185, 186, 187, 188, 189, 190, 191, 193, + /* 80 */ 194, 195, 196, 197, 198, 200, 201, 203, 204, 206, + /* 90 */ 207, 209, 210, 212, 214, 216, 217, 221, 222, 223, + /* 100 */ 144, 224, 225, 146, 149, 226, 172, 165, 170, 228, + /* 110 */ 230, 233, 236, 231, 237, 240, 238, 239, 241, 242, + /* 120 */ 243, 244, 246, 245, 247, 250, 251, 255, 256, 248, + /* 130 */ 261, 264, 265, 262, 266, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 628, 680, 669, 845, 845, 628, 628, 628, 628, 628, - /* 10 */ 628, 773, 646, 845, 628, 628, 628, 628, 628, 628, - /* 20 */ 628, 628, 628, 682, 682, 682, 768, 628, 628, 628, - /* 30 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 40 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 50 */ 628, 628, 628, 628, 628, 628, 628, 775, 777, 628, - /* 60 */ 794, 794, 766, 628, 628, 628, 628, 628, 628, 628, - /* 70 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 80 */ 628, 667, 628, 665, 628, 628, 628, 628, 628, 628, - /* 90 */ 628, 628, 628, 628, 628, 628, 628, 654, 628, 628, - /* 100 */ 628, 648, 648, 628, 628, 648, 801, 805, 799, 787, - /* 110 */ 795, 786, 782, 781, 809, 628, 648, 648, 677, 677, - /* 120 */ 648, 698, 696, 694, 686, 692, 688, 690, 684, 648, - /* 130 */ 675, 648, 675, 714, 728, 628, 810, 844, 800, 828, - /* 140 */ 827, 840, 834, 833, 628, 832, 831, 830, 829, 628, - /* 150 */ 628, 628, 628, 836, 835, 628, 628, 628, 628, 628, - /* 160 */ 628, 628, 628, 628, 812, 806, 802, 628, 628, 628, - /* 170 */ 628, 628, 628, 628, 628, 628, 628, 628, 628, 628, - /* 180 */ 628, 628, 628, 628, 765, 628, 628, 774, 628, 628, - /* 190 */ 628, 628, 628, 628, 796, 628, 788, 628, 628, 628, - /* 200 */ 628, 628, 628, 742, 628, 628, 628, 628, 628, 628, - /* 210 */ 628, 628, 628, 628, 628, 628, 628, 849, 628, 628, - /* 220 */ 628, 736, 847, 628, 628, 628, 628, 628, 628, 628, - /* 230 */ 628, 628, 628, 628, 628, 628, 628, 701, 628, 652, - /* 240 */ 650, 628, 644, 628, + /* 0 */ 634, 686, 675, 853, 853, 634, 634, 634, 634, 634, + /* 10 */ 634, 779, 652, 853, 634, 634, 634, 634, 634, 634, + /* 20 */ 634, 634, 634, 688, 688, 688, 774, 634, 634, 634, + /* 30 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 40 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 50 */ 634, 634, 634, 634, 634, 634, 634, 781, 783, 634, + /* 60 */ 800, 800, 772, 634, 634, 634, 634, 634, 634, 634, + /* 70 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 80 */ 634, 673, 634, 671, 634, 634, 634, 634, 634, 634, + /* 90 */ 634, 634, 634, 634, 634, 634, 634, 660, 634, 634, + /* 100 */ 634, 654, 654, 634, 634, 654, 807, 811, 805, 793, + /* 110 */ 801, 792, 788, 787, 815, 634, 654, 654, 683, 683, + /* 120 */ 654, 704, 702, 700, 692, 698, 694, 696, 690, 654, + /* 130 */ 681, 654, 681, 720, 734, 634, 816, 852, 806, 842, + /* 140 */ 841, 848, 840, 839, 634, 835, 836, 838, 837, 634, + /* 150 */ 634, 634, 634, 844, 843, 634, 634, 634, 634, 634, + /* 160 */ 634, 634, 634, 634, 818, 634, 812, 808, 634, 634, + /* 170 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 634, + /* 180 */ 634, 634, 634, 634, 634, 771, 634, 634, 780, 634, + /* 190 */ 634, 634, 634, 634, 634, 802, 634, 794, 634, 634, + /* 200 */ 634, 634, 634, 634, 634, 748, 634, 634, 634, 634, + /* 210 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 857, + /* 220 */ 634, 634, 634, 742, 855, 634, 634, 634, 634, 634, + /* 230 */ 634, 634, 634, 634, 634, 634, 634, 634, 634, 707, + /* 240 */ 634, 658, 656, 634, 650, 634, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1198,35 +1200,37 @@ static const char *const yyRuleName[] = { /* 193 */ "expr ::= BOOL", /* 194 */ "expr ::= ID LP exprlist RP", /* 195 */ "expr ::= ID LP STAR RP", - /* 196 */ "expr ::= expr AND expr", - /* 197 */ "expr ::= expr OR expr", + /* 196 */ "expr ::= expr IS NULL", + /* 197 */ "expr ::= expr IS NOT NULL", /* 198 */ "expr ::= expr LT expr", /* 199 */ "expr ::= expr GT expr", /* 200 */ "expr ::= expr LE expr", /* 201 */ "expr ::= expr GE expr", /* 202 */ "expr ::= expr NE expr", /* 203 */ "expr ::= expr EQ expr", - /* 204 */ "expr ::= expr PLUS expr", - /* 205 */ "expr ::= expr MINUS expr", - /* 206 */ "expr ::= expr STAR expr", - /* 207 */ "expr ::= expr SLASH expr", - /* 208 */ "expr ::= expr REM expr", - /* 209 */ "expr ::= expr LIKE expr", - /* 210 */ "expr ::= expr IN LP exprlist RP", - /* 211 */ "exprlist ::= exprlist COMMA expritem", - /* 212 */ "exprlist ::= expritem", - /* 213 */ "expritem ::= expr", - /* 214 */ "expritem ::=", - /* 215 */ "cmd ::= RESET QUERY CACHE", - /* 216 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 217 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 222 */ "cmd ::= KILL CONNECTION INTEGER", - /* 223 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 224 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 204 */ "expr ::= expr AND expr", + /* 205 */ "expr ::= expr OR expr", + /* 206 */ "expr ::= expr PLUS expr", + /* 207 */ "expr ::= expr MINUS expr", + /* 208 */ "expr ::= expr STAR expr", + /* 209 */ "expr ::= expr SLASH expr", + /* 210 */ "expr ::= expr REM expr", + /* 211 */ "expr ::= expr LIKE expr", + /* 212 */ "expr ::= expr IN LP exprlist RP", + /* 213 */ "exprlist ::= exprlist COMMA expritem", + /* 214 */ "exprlist ::= expritem", + /* 215 */ "expritem ::= expr", + /* 216 */ "expritem ::=", + /* 217 */ "cmd ::= RESET QUERY CACHE", + /* 218 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 219 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 220 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 221 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 224 */ "cmd ::= KILL CONNECTION INTEGER", + /* 225 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 226 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1880,35 +1884,37 @@ static const struct { { 260, -1 }, /* (193) expr ::= BOOL */ { 260, -4 }, /* (194) expr ::= ID LP exprlist RP */ { 260, -4 }, /* (195) expr ::= ID LP STAR RP */ - { 260, -3 }, /* (196) expr ::= expr AND expr */ - { 260, -3 }, /* (197) expr ::= expr OR expr */ + { 260, -3 }, /* (196) expr ::= expr IS NULL */ + { 260, -4 }, /* (197) expr ::= expr IS NOT NULL */ { 260, -3 }, /* (198) expr ::= expr LT expr */ { 260, -3 }, /* (199) expr ::= expr GT expr */ { 260, -3 }, /* (200) expr ::= expr LE expr */ { 260, -3 }, /* (201) expr ::= expr GE expr */ { 260, -3 }, /* (202) expr ::= expr NE expr */ { 260, -3 }, /* (203) expr ::= expr EQ expr */ - { 260, -3 }, /* (204) expr ::= expr PLUS expr */ - { 260, -3 }, /* (205) expr ::= expr MINUS expr */ - { 260, -3 }, /* (206) expr ::= expr STAR expr */ - { 260, -3 }, /* (207) expr ::= expr SLASH expr */ - { 260, -3 }, /* (208) expr ::= expr REM expr */ - { 260, -3 }, /* (209) expr ::= expr LIKE expr */ - { 260, -5 }, /* (210) expr ::= expr IN LP exprlist RP */ - { 269, -3 }, /* (211) exprlist ::= exprlist COMMA expritem */ - { 269, -1 }, /* (212) exprlist ::= expritem */ - { 270, -1 }, /* (213) expritem ::= expr */ - { 270, 0 }, /* (214) expritem ::= */ - { 208, -3 }, /* (215) cmd ::= RESET QUERY CACHE */ - { 208, -7 }, /* (216) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 208, -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 208, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 208, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 208, -8 }, /* (220) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 208, -9 }, /* (221) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 208, -3 }, /* (222) cmd ::= KILL CONNECTION INTEGER */ - { 208, -5 }, /* (223) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 208, -5 }, /* (224) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 260, -3 }, /* (204) expr ::= expr AND expr */ + { 260, -3 }, /* (205) expr ::= expr OR expr */ + { 260, -3 }, /* (206) expr ::= expr PLUS expr */ + { 260, -3 }, /* (207) expr ::= expr MINUS expr */ + { 260, -3 }, /* (208) expr ::= expr STAR expr */ + { 260, -3 }, /* (209) expr ::= expr SLASH expr */ + { 260, -3 }, /* (210) expr ::= expr REM expr */ + { 260, -3 }, /* (211) expr ::= expr LIKE expr */ + { 260, -5 }, /* (212) expr ::= expr IN LP exprlist RP */ + { 269, -3 }, /* (213) exprlist ::= exprlist COMMA expritem */ + { 269, -1 }, /* (214) exprlist ::= expritem */ + { 270, -1 }, /* (215) expritem ::= expr */ + { 270, 0 }, /* (216) expritem ::= */ + { 208, -3 }, /* (217) cmd ::= RESET QUERY CACHE */ + { 208, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 208, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 208, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 208, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 208, -8 }, /* (222) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 208, -9 }, /* (223) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 208, -3 }, /* (224) cmd ::= KILL CONNECTION INTEGER */ + { 208, -5 }, /* (225) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 208, -5 }, /* (226) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2570,7 +2576,7 @@ static void yy_reduce( break; case 168: /* having_opt ::= */ case 178: /* where_opt ::= */ yytestcase(yyruleno==178); - case 214: /* expritem ::= */ yytestcase(yyruleno==214); + case 216: /* expritem ::= */ yytestcase(yyruleno==216); {yymsp[1].minor.yy66 = 0;} break; case 169: /* having_opt ::= HAVING expr */ @@ -2643,24 +2649,20 @@ static void yy_reduce( yymsp[0].minor.yy66 = yylhsminor.yy66; break; case 194: /* expr ::= ID LP exprlist RP */ -{ - yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); -} +{ yylhsminor.yy66 = tSQLExprCreateFunction(yymsp[-1].minor.yy224, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy66 = yylhsminor.yy66; break; case 195: /* expr ::= ID LP STAR RP */ -{ - yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); -} +{ yylhsminor.yy66 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy66 = yylhsminor.yy66; break; - case 196: /* expr ::= expr AND expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);} + case 196: /* expr ::= expr IS NULL */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, NULL, TK_ISNULL);} yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 197: /* expr ::= expr OR expr */ -{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); } - yymsp[-2].minor.yy66 = yylhsminor.yy66; + case 197: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-3].minor.yy66, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy66 = yylhsminor.yy66; break; case 198: /* expr ::= expr LT expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LT);} @@ -2686,57 +2688,65 @@ static void yy_reduce( {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_EQ);} yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 204: /* expr ::= expr PLUS expr */ + case 204: /* expr ::= expr AND expr */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_AND);} + yymsp[-2].minor.yy66 = yylhsminor.yy66; + break; + case 205: /* expr ::= expr OR expr */ +{yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_OR); } + yymsp[-2].minor.yy66 = yylhsminor.yy66; + break; + case 206: /* expr ::= expr PLUS expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_PLUS); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 205: /* expr ::= expr MINUS expr */ + case 207: /* expr ::= expr MINUS expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_MINUS); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 206: /* expr ::= expr STAR expr */ + case 208: /* expr ::= expr STAR expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_STAR); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 207: /* expr ::= expr SLASH expr */ + case 209: /* expr ::= expr SLASH expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_DIVIDE);} yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 208: /* expr ::= expr REM expr */ + case 210: /* expr ::= expr REM expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_REM); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 209: /* expr ::= expr LIKE expr */ + case 211: /* expr ::= expr LIKE expr */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-2].minor.yy66, yymsp[0].minor.yy66, TK_LIKE); } yymsp[-2].minor.yy66 = yylhsminor.yy66; break; - case 210: /* expr ::= expr IN LP exprlist RP */ + case 212: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy66 = tSQLExprCreate(yymsp[-4].minor.yy66, (tSQLExpr*)yymsp[-1].minor.yy224, TK_IN); } yymsp[-4].minor.yy66 = yylhsminor.yy66; break; - case 211: /* exprlist ::= exprlist COMMA expritem */ + case 213: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy224 = tSQLExprListAppend(yymsp[-2].minor.yy224,yymsp[0].minor.yy66,0);} yymsp[-2].minor.yy224 = yylhsminor.yy224; break; - case 212: /* exprlist ::= expritem */ + case 214: /* exprlist ::= expritem */ {yylhsminor.yy224 = tSQLExprListAppend(0,yymsp[0].minor.yy66,0);} yymsp[0].minor.yy224 = yylhsminor.yy224; break; - case 213: /* expritem ::= expr */ + case 215: /* expritem ::= expr */ {yylhsminor.yy66 = yymsp[0].minor.yy66;} yymsp[0].minor.yy66 = yylhsminor.yy66; break; - case 215: /* cmd ::= RESET QUERY CACHE */ + case 217: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 216: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 218: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 217: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 219: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2747,14 +2757,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 218: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 220: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy449, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 219: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 221: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2765,7 +2775,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 222: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2779,7 +2789,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 223: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -2791,13 +2801,13 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= KILL CONNECTION INTEGER */ + case 224: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 223: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 225: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 224: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 226: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d829a85754..3a39d5fc49 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -248,6 +248,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); + // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i); @@ -388,9 +389,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", %p", + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pHandle->qinfo); + pCheckInfo->lastKey, pMem->numOfRows, pHandle->qinfo); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -410,9 +411,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", %p", + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pHandle->qinfo); + pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qinfo); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -734,6 +735,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p return TSDB_CODE_SUCCESS; } +static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); @@ -790,9 +792,10 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc * Here the buffer is not enough, so only part of file block can be loaded into memory buffer */ assert(pQueryHandle->outputCapacity >= binfo.rows); + int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); - if ((cur->pos == 0 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->pos == (binfo.rows - 1) && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { + if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { pQueryHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; @@ -808,7 +811,6 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc cur->pos = -1; } } else { // partially copy to dest buffer - int32_t endPos = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.rows - 1): 0; copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); cur->mixBlock = true; } @@ -1203,6 +1205,29 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl cur->win.ekey, cur->rows, pQueryHandle->qinfo); } +int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { + // NOTE: reverse the order to find the end position in data block + int32_t endPos = -1; + int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + + SQueryFilePos* cur = &pQueryHandle->cur; + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + + if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { + endPos = pBlockInfo->rows - 1; + cur->mixBlock = (cur->pos != 0); + } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { + endPos = 0; + cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); + } else { + assert(pCols->numOfRows > 0); + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + cur->mixBlock = true; + } + + return endPos; +} + // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { @@ -1224,19 +1249,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = cur->pos; - - if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { - endPos = blockInfo.rows - 1; - cur->mixBlock = (cur->pos != 0); - } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { - endPos = 0; - cur->mixBlock = (cur->pos != blockInfo.rows - 1); - } else { - assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); - cur->mixBlock = true; - } + int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, %p", @@ -1338,8 +1351,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } cur->blockCompleted = - (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); @@ -2071,13 +2084,17 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { if (keyInfo.pTable != NULL) { totalNumOfTable++; taosArrayPush(pGroup, &keyInfo); + } else { + taosArrayRemove(groupList->pGroupList, j); + numOfGroups -= 1; + j -= 1; } } // window does not being updated, so set the original if (window.skey == INT64_MAX && window.ekey == INT64_MIN) { window = TSWINDOW_INITIALIZER; - assert(totalNumOfTable == 0); + assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0); } groupList->numOfTables = totalNumOfTable; @@ -2398,6 +2415,14 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) { val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); } + if (pInfo->optr == TSDB_RELATION_ISNULL || pInfo->optr == TSDB_RELATION_NOTNULL) { + if (pInfo->optr == TSDB_RELATION_ISNULL) { + return (val == NULL) || isNull(val, pInfo->sch.type); + } else if (pInfo->optr == TSDB_RELATION_NOTNULL) { + return (val != NULL) && (!isNull(val, pInfo->sch.type)); + } + } + int32_t ret = 0; if (val == NULL) { //the val is possible to be null, so check it out carefully ret = -1; // val is missing in table tags value pairs diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index ab489e2e46..4d737ebe66 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -71,7 +71,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const * @param pCacheObj Cache object * @param pNode Cache slot object */ -static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode); +static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode); /** * remove nodes in trash with refCount == 0 in cache @@ -80,7 +80,7 @@ static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode); * @param force force model, if true, remove data in trash without check refcount. * may cause corruption. So, forece model only applys before cache is closed */ -static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force); +static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force); /** * release node @@ -165,7 +165,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - // set free cache node callback function for hash table + // set free cache node callback function pCacheObj->freeFp = fn; pCacheObj->refreshTime = refreshTimeInSeconds * 1000; pCacheObj->extendLifespan = extendLifespan; @@ -222,7 +222,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v taosTFree(p); } else { - taosAddToTrash(pCacheObj, p); + taosAddToTrashcan(pCacheObj, p); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data); } } @@ -322,7 +322,12 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { } void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { - if (pCacheObj == NULL || (*data) == NULL || (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0)) { + if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) { + return; + } + + if ((*data) == NULL) { + uError("cache:%s, NULL data to release", pCacheObj->name); return; } @@ -394,19 +399,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { "others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); assert(p->pTNodeHeader == NULL); - taosAddToTrash(pCacheObj, p); + taosAddToTrashcan(pCacheObj, p); } else { uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - taosAddToTrash(pCacheObj, pNode); + taosAddToTrashcan(pCacheObj, pNode); } else { // ref == 0 atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); - uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes", pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize); if (pCacheObj->freeFp) { @@ -427,6 +432,26 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { char* key = pNode->key; char* p = pNode->data; +// int32_t ref = T_REF_VAL_GET(pNode); +// +// if (ref == 1 && inTrashcan) { +// // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be +// // destroyed by refresh worker if decrease ref count before removing it from linked-list. +// assert(pNode->pTNodeHeader->pData == pNode); +// +// __cache_wr_lock(pCacheObj); +// doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); +// __cache_unlock(pCacheObj); +// +// ref = T_REF_DEC(pNode); +// assert(ref == 0); +// +// doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); +// } else { +// ref = T_REF_DEC(pNode); +// assert(ref >= 0); +// } + int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan); } @@ -447,7 +472,7 @@ static bool travHashTableEmptyFn(void* param, void* data) { if (T_REF_VAL_GET(pNode) == 0) { taosCacheReleaseNode(pCacheObj, pNode); } else { // do add to trashcan - taosAddToTrash(pCacheObj, pNode); + taosAddToTrashcan(pCacheObj, pNode); } // this node should be remove from hash table @@ -458,7 +483,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj) { SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); - taosTrashCanEmpty(pCacheObj, false); + taosTrashcanEmpty(pCacheObj, false); } void taosCacheCleanup(SCacheObj *pCacheObj) { @@ -498,7 +523,7 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * return pNewNode; } -void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { +void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; @@ -520,11 +545,11 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->numOfElemsInTrash++; __cache_unlock(pCacheObj); - uDebug("cache:%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data, + uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); } -void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { +void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { __cache_wr_lock(pCacheObj); if (pCacheObj->numOfElemsInTrash == 0) { @@ -568,7 +593,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); - taosTrashCanEmpty(pCacheObj, true); + taosTrashcanEmpty(pCacheObj, true); __cache_lock_destroy(pCacheObj); @@ -643,7 +668,7 @@ void* taosCacheTimedRefresh(void *handle) { doCacheRefresh(pCacheObj, now, NULL); } - taosTrashCanEmpty(pCacheObj, false); + taosTrashcanEmpty(pCacheObj, false); } return NULL; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index a8587de767..e5afe1b68e 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -433,7 +433,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . va_list argpointer; char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; - int32_t len; + int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; time_t curTime; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 61896a86df..4cf73e6dff 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -16,7 +16,7 @@ #include "os.h" #include "tulog.h" #include "tsocket.h" -#include "tutil.h" +#include "taoserror.h" int taosGetFqdn(char *fqdn) { char hostname[1024]; @@ -56,7 +56,16 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { freeaddrinfo(result); return ip; } else { - uError("failed get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); +#ifdef EAI_SYSTEM + if (ret == EAI_SYSTEM) { + uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + } +#else + uError("failed to get the ip address, fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); +#endif return 0xFFFFFFFF; } } diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 51221e0b35..0a4791f6a9 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -12,65 +12,65 @@ int32_t tsMaxMeterConnections = 200; // test cache TEST(testCase, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; - SCacheObj* tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); + SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); const char* key1 = "test1"; char data1[] = "test11"; - char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data1, strlen(data1)+1, 1); + char* cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1)+1, 1); sleep(REFRESH_TIME_IN_SEC+1); printf("obj is still valid: %s\n", cachedObj); char data2[] = "test22"; - taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj, false); /* the object is cleared by cache clean operation */ - cachedObj = (char*) taosCachePut(tscCacheHandle, key1, strlen(key1), data2, strlen(data2)+1, 20); + cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data2, strlen(data2)+1, 20); printf("after updated: %s\n", cachedObj); printf("start to remove data from cache\n"); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj, false); printf("end of removing data from cache\n"); const char* key3 = "test2"; const char* data3 = "kkkkkkk"; - char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, strlen(key3), data3, strlen(data3) + 1, 1); + char* cachedObj2 = (char*) taosCachePut(tscMetaCache, key3, strlen(key3), data3, strlen(data3) + 1, 1); printf("%s\n", cachedObj2); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); sleep(3); - char* d = (char*) taosCacheAcquireByKey(tscCacheHandle, key3, strlen(key3)); + char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); // assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data5, strlen(data5) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data5, strlen(data5) + 1, 20); const char* data6= "new Data after updated"; - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data6, strlen(data6) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data6, strlen(data6) + 1, 20); printf("%s\n", cachedObj2); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true); const char* data7 = "add call update procedure"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, strlen(key5), data7, strlen(data7) + 1, 20); + cachedObj2 = (char*) taosCachePut(tscMetaCache, key5, strlen(key5), data7, strlen(data7) + 1, 20); printf("%s\n=======================================\n\n", cachedObj2); - char* cc = (char*) taosCacheAcquireByKey(tscCacheHandle, key5, strlen(key5)); + char* cc = (char*) taosCacheAcquireByKey(tscMetaCache, key5, strlen(key5)); - taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); - taosCacheRelease(tscCacheHandle, (void**) &cc, false); + taosCacheRelease(tscMetaCache, (void**) &cachedObj2, true); + taosCacheRelease(tscMetaCache, (void**) &cc, false); const char* data8 = "ttft"; const char* key6 = "key6"; - char* ft = (char*) taosCachePut(tscCacheHandle, key6, strlen(key6), data8, strlen(data8), 20); - taosCacheRelease(tscCacheHandle, (void**) &ft, false); + char* ft = (char*) taosCachePut(tscMetaCache, key6, strlen(key6), data8, strlen(data8), 20); + taosCacheRelease(tscMetaCache, (void**) &ft, false); /** * 140ns @@ -78,14 +78,14 @@ TEST(testCase, client_cache_test) { uint64_t startTime = taosGetTimestampUs(); printf("Cache Performance Test\nstart time:%" PRIu64 "\n", startTime); for(int32_t i=0; i<1000; ++i) { - char* dd = (char*) taosCacheAcquireByKey(tscCacheHandle, key6, strlen(key6)); + char* dd = (char*) taosCacheAcquireByKey(tscMetaCache, key6, strlen(key6)); if (dd != NULL) { // printf("get the data\n"); } else { printf("data has been released\n"); } - taosCacheRelease(tscCacheHandle, (void**) &dd, false); + taosCacheRelease(tscMetaCache, (void**) &dd, false); } uint64_t endTime = taosGetTimestampUs(); @@ -93,7 +93,7 @@ TEST(testCase, client_cache_test) { printf("End of Test, %" PRIu64 "\nTotal Elapsed Time:%" PRIu64 " us.avg:%f us\n", endTime, el, el/1000.0); - taosCacheCleanup(tscCacheHandle); + taosCacheCleanup(tscMetaCache); } TEST(testCase, cache_resize_test) { diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim index 6a005b028a..98d64ab839 100644 --- a/tests/script/general/http/autocreate.sim +++ b/tests/script/general/http/autocreate.sim @@ -24,5 +24,10 @@ print curl 127.0.0.1:7111/rest/sql -----> $system_content # return -1 #endi +sql select * from db.win_cpu_windows_1_processor +print rows: $rows +if $rows != 1 then + return -1 +endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim new file mode 100644 index 0000000000..8673655d96 --- /dev/null +++ b/tests/script/general/http/chunked.sim @@ -0,0 +1,37 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 +restful d1 table_rest 1591172800 10000 +restful d1 table_rest 1591272800 10000 +restful d1 table_rest 1591372800 10000 +restful d1 table_rest 1591472800 10000 +restful d1 table_rest 1591572800 10000 +restful d1 table_rest 1591672800 10000 +restful d1 table_rest 1591772800 10000 +restful d1 table_rest 1591872800 10000 +restful d1 table_rest 1591972800 10000 + +sql select * from table_rest; +print rows: $rows +if $rows != 100000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim new file mode 100644 index 0000000000..0289e337a6 --- /dev/null +++ b/tests/script/general/http/gzip.sim @@ -0,0 +1,27 @@ +system sh/stop_dnodes.sh +sleep 3000 +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ============================ dnode1 start + +print =============== step1 - prepare data +sql create database d1 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 270KB +restful d1 table_rest 1591072800 10000 gzip +sql select * from table_rest; +print rows: $rows +if $rows != 10000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index d91e9f452d..f35362bf07 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,3 +1,5 @@ +run general/http/autocreate.sim +run general/http/chunked.sim run general/http/restful.sim run general/http/restful_insert.sim run general/http/restful_limit.sim diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 255e00ca41..bd0d3c1a12 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -423,6 +423,8 @@ if $data97 != @group_tb0@ then return -1 endi +print ---------------------------------> group by binary|nchar data add cases + #=========================== group by multi tags ====================== sql create table st (ts timestamp, c int) tags (t1 int, t2 int, t3 int, t4 int); diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index ef3245ccaf..254571bda1 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -205,10 +205,12 @@ if $rows != 9 then endi if $data00 != @70-01-01 08:01:40.100@ then + print $data00 return -1 endi if $data10 != @70-01-01 08:01:40.200@ then + print $data10 return -1 endi diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 5fc47ed15d..e9d8ce413d 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -154,6 +154,7 @@ if $rows != 46 then endi print ========>td-1317, empty table last_row query crashed +sql drop table if exists m1; sql create table m1(ts timestamp, k int) tags (a int); sql create table t1 using m1 tags(1); sql create table t2 using m1 tags(2); @@ -172,3 +173,49 @@ sql select last_row(*) from m1 where tbname in ('t1') if $rows != 0 then return -1 endi + +sql insert into t1 values('2019-1-1 1:1:1', 1); +print ===================> last_row query against normal table along with ts/tbname +sql select last_row(*),ts,'k' from t1; +if $rows != 1 then + return -1 +endi + +print ===================> last_row + user-defined column + normal tables +sql select last_row(ts), 'abc', 1234.9384, ts from t1 +if $rows != 1 then + return -1 +endi + +if $data01 != @abc@ then + print expect abc, actual $data02 + return -1 +endi + +if $data02 != 1234.938400000 then + return -1 +endi + +if $data03 != @19-01-01 01:01:01.000@ then + print expect 19-01-01 01:01:01.000, actual:$data03 + return -1 +endi + +print ===================> last_row + stable + ts/tag column + condition + udf +sql select last_row(*), ts, 'abc', 123.981, tbname from m1 +if $rows != 1 then + return -1 +endi + +if $data02 != @19-01-01 01:01:01.000@ then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 123.981000000 then + print expect 123.981000000, actual: $data04 + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 6790564cc7..3dd80b8e38 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -99,6 +99,8 @@ run general/parser/union.sim sleep 2000 run general/parser/constCol.sim sleep 2000 +run general/parser/where.sim +sleep 2000 run general/parser/timestamp.sim sleep 2000 run general/parser/sliding.sim diff --git a/tests/script/general/parser/timestamp_query.sim b/tests/script/general/parser/timestamp_query.sim index 6994b2d295..783c03602b 100644 --- a/tests/script/general/parser/timestamp_query.sim +++ b/tests/script/general/parser/timestamp_query.sim @@ -21,6 +21,10 @@ $tsu = $rowNum * $delta $tsu = $tsu - $delta $tsu = $tsu + $ts0 +print ==================>issue #3481, normal column not allowed, +sql_error select ts,c1,min(c2) from ts_stb0 + + ##### select from supertable $tb = $tbPrefix . 0 sql select first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1) diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 2faee55460..5c575b6163 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 1000 sql connect $dbPrefix = tb_db @@ -25,7 +25,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db cache 16 maxrows 4096 keep 36500 print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) @@ -132,15 +132,15 @@ sleep 5000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 3000 +sleep 1000 sql select count(*) from t1.test where ts>10000 and ts<90000 interval(5000a) if $rows != 3 then return -1 endi -print =========>td-1308 -sql create database db; +print ==============>td-1308 +sql create database db keep 36500; sql use db; sql create table stb (ts timestamp, c1 int, c2 binary(10)) tags(t1 binary(10)); @@ -158,4 +158,59 @@ if $rows != 1 then return -1 endi +print =======================>td-1446 +sql create table t(ts timestamp, k int) +$ts = 6000 +while $ts < 7000 + sql insert into t values ( $ts , $ts ) + $ts = $ts + 1 +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 1000 +sql use db; + +$ts = 1000 +while $ts < 5096 + sql insert into t values ( $ts , $ts ) + $ts = $ts + 1 +endw + +sql select * from t where ts < 6500 +if $rows != 4596 then + print expect 4596, actual: $rows + return -1 +endi + +sql select * from t where ts < 7000 +if $rows != 5096 then + return -1 +endi + +sql select * from t where ts <= 6000 +if $rows != 4097 then + return -1 +endi + +sql select * from t where ts <= 6001 +if $rows != 4098 then + return -1 +endi + +print ======================>td-1454 +sql select count(*)/10, count(*)+99 from t +if $rows != 1 then + return -1 +endi + +if $data00 != 509.600000000 then + return -1 +endi + +if $data01 != 5195.000000000 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index dd3b11c2dc..fb15fb6dbe 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -2,6 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 3000 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 063d11bd9d..adb22aa265 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -77,6 +77,8 @@ cd ../../../debug; make ./test.sh -f general/field/smallint.sim ./test.sh -f general/field/tinyint.sim +./test.sh -f general/http/autocreate.sim +./test.sh -f general/http/chunked.sim ./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 5339203ec2..ffab39d41c 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -39,5 +39,8 @@ IF (TD_LINUX) add_executable(httpTest httpTest.c) target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) + + add_executable(cacheTest cacheTest.c) + target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4) ENDIF() diff --git a/tests/test/c/cacheTest.c b/tests/test/c/cacheTest.c new file mode 100644 index 0000000000..54aca0038e --- /dev/null +++ b/tests/test/c/cacheTest.c @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include +#include "os.h" +#include "taos.h" +#include "tcache.h" +#include "tulog.h" +#include "tutil.h" + +#define MAX_REFRESH_TIME_SEC 2 +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +int32_t tsKeepTimeInSec = 3; +int32_t tsNumOfRows = 1000000; +int32_t tsSizeOfRow = 64 * 1024; +void * tsCacheObj = NULL; +int32_t destroyTimes = 0; + +typedef int64_t CacheTestKey; +typedef struct CacheTestRow { + int32_t index; + void ** ppRow; + void * data; +} CacheTestRow; + +CacheTestRow *initRow(int32_t index) { + CacheTestRow *row = calloc(sizeof(CacheTestRow), 1); + row->index = index; + row->data = malloc(tsSizeOfRow * sizeof(int8_t)); + return row; +} + +void detroyRow(void *data) { + CacheTestRow *row = *(CacheTestRow **)data; + free(row->data); + free(row); + destroyTimes++; + if (destroyTimes % 50000 == 0) { + pPrint("%s ===> destroyTimes:%d %s", GREEN, destroyTimes, NC); + } +} + +void initCache() { + tsCacheObj = taosCacheInit(TSDB_DATA_TYPE_BIGINT, MAX_REFRESH_TIME_SEC, true, detroyRow, "cachetest"); +} + +void putRowInCache() { + for (int index = 0; index < tsNumOfRows; ++index) { + CacheTestRow *row = initRow(index); + uint64_t key = (uint64_t)row; + void **ppRow = taosCachePut(tsCacheObj, &key, sizeof(int64_t), &row, sizeof(int64_t), tsKeepTimeInSec * 1000); + row->ppRow = ppRow; + taosCacheRelease(tsCacheObj, (void **)&ppRow, false); + } +} + +void cleanupCache() { + taosCacheCleanup(tsCacheObj); +} + +void initGetMemory() { + osInit(); + taos_init(); +} + +float getProcMemory() { + float procMemoryUsedMB = 0; + taosGetProcMemory(&procMemoryUsedMB); + return procMemoryUsedMB; +} + +void doTest() { + initCache(); + pPrint("%s initialize procMemory %f MB %s", GREEN, getProcMemory(), NC); + + putRowInCache(); + pPrint("%s insert %d rows, procMemory %f MB %s", GREEN, tsNumOfRows, getProcMemory(), NC); + + int32_t sleepMs = (MAX_REFRESH_TIME_SEC * 3) * 1000 + tsKeepTimeInSec * 1000; + taosMsleep(sleepMs); + pPrint("%s after sleep %d ms, procMemory %f MB %s", GREEN, sleepMs, getProcMemory(), NC); + + cleanupCache(); + taosMsleep(sleepMs); + pPrint("%s after cleanup cache, procMemory %f MB %s", GREEN, getProcMemory(), NC); + + malloc_trim(0); + taosMsleep(sleepMs); + pPrint("%s after malloc_trim, procMemory %f MB %s", GREEN, getProcMemory(), NC); +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance of cache\n"); + + printf("%s%s\n", indent, "-k"); + printf("%s%s%s%d\n", indent, indent, "KeepTimeInSec, default is ", tsKeepTimeInSec); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "NumOfRows, default is ", tsNumOfRows); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "SizeOfData, default is ", tsSizeOfRow); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-k") == 0) { + tsKeepTimeInSec = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + tsNumOfRows = atoi(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + tsSizeOfRow = atoi(argv[++i]); + } else { + } + } + + pPrint("%s KeepTimeInSec:%d %s", GREEN, tsKeepTimeInSec, NC); + pPrint("%s NumOfRows:%d %s", GREEN, tsNumOfRows, NC); + pPrint("%s SizeOfData:%d %s", GREEN, tsSizeOfRow, NC); +} + +int main(int argc, char *argv[]) { + initGetMemory(); + parseArgument(argc, argv); + doTest(); +} diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 6f3bc7099d..58e58a442c 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -84,6 +84,7 @@ enum { SIM_CMD_SQL, SIM_CMD_SQL_ERROR, SIM_CMD_SQL_SLOW, + SIM_CMD_RESTFUL, SIM_CMD_TEST, SIM_CMD_RETURN, SIM_CMD_END @@ -172,6 +173,7 @@ bool simExecuteReturnCmd(SScript *script, char *option); bool simExecuteSqlCmd(SScript *script, char *option); bool simExecuteSqlErrorCmd(SScript *script, char *rest); bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index adc2fd0b9d..463dc33c7c 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -915,6 +915,47 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { return simExecuteSqlImpCmd(script, rest, isSlow); } +bool simExecuteRestfulCmd(SScript *script, char *rest) { + FILE *fp = NULL; + char filename[256]; + sprintf(filename, "%s/tmp.sql", tsScriptDir); + fp = fopen(filename, "w"); + if (fp == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return false; + } + + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; + int32_t ts; + int32_t times; + sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); + + fprintf(fp, "insert into %s.%s values ", db, tb); + for (int i = 0; i < times; ++i) { + fprintf(fp, "(%d000, %d)", ts + i, ts); + } + fprintf(fp, " \n"); + fflush(fp); + fclose(fp); + + char cmd[1024] = {0}; + if (strcmp(gzip, "gzip") == 0) { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } else { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } + + return simExecuteSystemCmd(script, cmd); +} + bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 8dcf83806f..2e6121304f 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -721,6 +721,12 @@ bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; + return true; +} + bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { int expLen; @@ -1020,6 +1026,14 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); + cmdno = SIM_CMD_RESTFUL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "restful"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRestfulCmd; + simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + /* test is only an internal command */ cmdno = SIM_CMD_TEST; simCmdList[cmdno].cmdno = cmdno;