diff --git a/.travis.yml b/.travis.yml index db6ce79703..39fddc20c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: memError=`grep -m 1 'ERROR SUMMARY' mem-error-out.txt | awk '{print $4}'` if [ -n "$memError" ]; then - if [ "$memError" -gt 16 ] && [ "$defiMemError" -gt 0 ]; then + if [ "$memError" -gt 16 ] || [ "$defiMemError" -gt 0 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $memError diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index c16945bbcf..d46c32d73d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -196,14 +196,14 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); -void tscFreeSubqueryInfo(SSqlCmd* pCmd); +void tscFreeQueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 9cc6bcb364..d6fb9279c7 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -298,6 +298,7 @@ typedef struct STscObj { char sversion[TSDB_VERSION_LEN]; char writeAuth : 1; char superAuth : 1; + void* pMgmtConn; struct SSqlObj * pSql; struct SSqlObj * pHb; struct SSqlObj * sqlList; @@ -359,7 +360,7 @@ typedef struct SSqlStream { struct SSqlStream *prev, *next; } SSqlStream; -int32_t tscInitRpc(const char *user, const char *secret); +int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn); void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); @@ -427,9 +428,7 @@ int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column); extern void * pVnodeConn; -extern void * pTscMgmtConn; extern void * tscCacheHandle; -extern int slaveIndex; extern void * tscTmr; extern void * tscQhandle; extern int tscKeepConn[]; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 99d20de48b..b954db0734 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -46,7 +46,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = 1; + pSql->maxRetry = TSDB_REPLICA_MAX_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index df47afd0dc..8d6d9baca3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -733,7 +733,7 @@ int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SS */ if (size > 0) { if (strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearMeterMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo, false); } } else { assert(pTableMetaInfo->pTableMeta == NULL); @@ -2477,6 +2477,10 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return TSDB_CODE_SUCCESS; } + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + pQueryInfo->groupbyExpr.numOfGroupCols = pList->nExpr; if (pList->nExpr > TSDB_MAX_TAGS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -3545,7 +3549,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1; + SStringBuilder sb1 = { 0 }; taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; @@ -4915,7 +4919,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { list.num = 1; list.ids[0] = colIndex; - insertResultField(pQueryInfo, size - 1, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); + insertResultField(pQueryInfo, size, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1); pInfo->visible = false; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 15abcebe58..21f6f9d390 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -190,6 +190,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { } int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; SSqlCmd* pCmd = &pSql->cmd; char *pMsg = rpcMallocCont(pCmd->payloadLen); @@ -223,7 +224,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = pSql, .code = 0 }; - rpcSendRequest(pTscMgmtConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg); } return TSDB_CODE_SUCCESS; @@ -696,7 +697,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->numOfTags = htonl(numOfTags); - + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); pQueryMsg->queryType = htons(pQueryInfo->type); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); @@ -915,6 +916,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } + if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { + *pMsg = 0; + pMsg++; + } else { + strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); + pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + } + // tbname in/like query expression should be sent to mgmt node msgLen = pMsg - pStart; @@ -1847,7 +1856,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = TSDB_MSG_TYPE_CM_HEARTBEAT; assert(msgLen + minMsgSize() <= size); - return msgLen; + return TSDB_CODE_SUCCESS; } int tscProcessTableMetaRsp(SSqlObj *pSql) { @@ -2599,7 +2608,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { // if (pSql->fp != NULL && pSql->pStream == NULL) { // pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); -// tscFreeSubqueryInfo(pCmd); +// tscFreeQueryInfo(pCmd); // } tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f70beab166..4320e8f81b 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -66,7 +66,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } - if (tscInitRpc(user, pass) != 0) { + void* pMgmtConn = NULL; + if (tscInitRpc(user, pass, &pMgmtConn) != 0) { terrno = TSDB_CODE_NETWORK_UNAVAIL; return NULL; } @@ -118,6 +119,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con strtolower(pObj->db, tmp); } + pObj->pMgmtConn = pMgmtConn; pthread_mutex_init(&pObj->mutex, NULL); SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9f7d4887d1..d690681729 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -147,7 +147,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - tscClearMeterMetaInfo(pTableMetaInfo, true); + tscClearTableMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; @@ -177,7 +177,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); tscError("%p stream:%p, retrieve data failed, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); - tscClearMeterMetaInfo(pTableMetaInfo, true); + tscClearTableMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; @@ -259,7 +259,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf pStream->numOfRes); // release the metric/meter meta information reference, so data in cache can be updated - tscClearMeterMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo, false); tscSetNextLaunchTimer(pStream, pSql); } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index fac771234a..f1b6065f1c 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -33,9 +33,6 @@ // global, not configurable void * pVnodeConn; -void * pVMeterConn; -void * pTscMgmtConn; -void * pSlaveConn; void * tscCacheHandle; int slaveIndex; void * tscTmr; @@ -54,7 +51,7 @@ void tscCheckDiskUsage(void *para, void *unused) { taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } -int32_t tscInitRpc(const char *user, const char *secret) { +int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) { SRpcInit rpcInit; char secretEncrypt[32] = {0}; taosEncryptPass((uint8_t *)secret, strlen(secret), secretEncrypt); @@ -80,7 +77,7 @@ int32_t tscInitRpc(const char *user, const char *secret) { } } - if (pTscMgmtConn == NULL) { + if (*pMgmtConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; rpcInit.localPort = 0; @@ -96,8 +93,8 @@ int32_t tscInitRpc(const char *user, const char *secret) { rpcInit.spi = 1; rpcInit.secret = secretEncrypt; - pTscMgmtConn = rpcOpen(&rpcInit); - if (pTscMgmtConn == NULL) { + *pMgmtConn = rpcOpen(&rpcInit); + if (*pMgmtConn == NULL) { tscError("failed to init connection to mgmt"); return -1; } @@ -211,11 +208,6 @@ void taos_cleanup() { pVnodeConn = NULL; } - if (pTscMgmtConn != NULL) { - rpcClose(pTscMgmtConn); - pTscMgmtConn = NULL; - } - taosTmrCleanUp(tscTmr); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index abdab2d9a1..e9ff558b92 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -337,7 +337,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->pTableList= NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeSubqueryInfo(pCmd); + tscFreeQueryInfo(pCmd); } /* @@ -761,6 +761,8 @@ void tscCloseTscObj(STscObj* pObj) { tscFreeSqlObj(pSql); sem_destroy(&pSql->rspSem); + rpcClose(pObj->pMgmtConn); + pthread_mutex_destroy(&pObj->mutex); tscTrace("%p DB connection is closed", pObj); @@ -1459,7 +1461,7 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) { void tscCleanSqlCmd(SSqlCmd* pCmd) { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeSubqueryInfo(pCmd); + tscFreeQueryInfo(pCmd); uint32_t allocSize = pCmd->allocSize; char* allocPtr = pCmd->payload; @@ -1601,7 +1603,7 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { return TSDB_CODE_SUCCESS; } -static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { +static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); tscFieldInfoClear(&pQueryInfo->fieldsInfo); @@ -1611,6 +1613,11 @@ static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { tscColumnListDestroy(pQueryInfo->colList); memset(&pQueryInfo->colList, 0, sizeof(pQueryInfo->colList)); + if (pQueryInfo->groupbyExpr.columnInfo != NULL) { + taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); + pQueryInfo->groupbyExpr.columnInfo = NULL; + } + pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); tfree(pQueryInfo->defaultVal); @@ -1619,11 +1626,11 @@ static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { void tscClearSubqueryInfo(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->numOfClause; ++i) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); - doClearSubqueryInfo(pQueryInfo); + freeQueryInfoImpl(pQueryInfo); } } -void tscFreeSubqueryInfo(SSqlCmd* pCmd) { +void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } @@ -1632,7 +1639,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); - doClearSubqueryInfo(pQueryInfo); + freeQueryInfoImpl(pQueryInfo); tscClearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); tfree(pQueryInfo); } @@ -1691,7 +1698,7 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index); - tscClearMeterMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); free(pTableMetaInfo); int32_t after = pQueryInfo->numOfTables - index - 1; @@ -1713,13 +1720,18 @@ void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool tfree(pQueryInfo->pTableMetaInfo); } -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { if (pTableMetaInfo == NULL) { return; } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); tfree(pTableMetaInfo->vgroupList); + + if (pTableMetaInfo->tagColList != NULL) { + taosArrayDestroy(pTableMetaInfo->tagColList); + pTableMetaInfo->tagColList = NULL; + } } void tscResetForNextRetrieve(SSqlRes* pRes) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 216de90598..043e58cb35 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -32,6 +32,9 @@ extern "C" { #define TSKEY int64_t #endif +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) + // Bytes for each type. extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below @@ -141,16 +144,17 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_IN 8 -#define TSDB_RELATION_AND 8 -#define TSDB_RELATION_OR 9 -#define TSDB_RELATION_NOT 10 +#define TSDB_RELATION_AND 9 +#define TSDB_RELATION_OR 10 +#define TSDB_RELATION_NOT 11 -#define TSDB_BINARY_OP_ADD 11 -#define TSDB_BINARY_OP_SUBTRACT 12 -#define TSDB_BINARY_OP_MULTIPLY 13 -#define TSDB_BINARY_OP_DIVIDE 14 -#define TSDB_BINARY_OP_REMAINDER 15 +#define TSDB_BINARY_OP_ADD 12 +#define TSDB_BINARY_OP_SUBTRACT 13 +#define TSDB_BINARY_OP_MULTIPLY 14 +#define TSDB_BINARY_OP_DIVIDE 15 +#define TSDB_BINARY_OP_REMAINDER 16 #define TSDB_USERID_LEN 9 #define TS_PATH_DELIMITER_LEN 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 5f3ea93afb..16e147c5af 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -469,6 +469,7 @@ typedef struct { int64_t offset; uint16_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers + int16_t tagNameRelType; // relation of tag criteria and tbname criteria int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 481e05c102..e5c884c819 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -279,8 +279,17 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * @param pTagCond. tag query condition * */ -int32_t tsdbQueryByTagsCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, STableGroupInfo *pGroupList, - SColIndex *pColIndex, int32_t numOfCols); +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupList, + SColIndex *pColIndex, + int32_t numOfCols + ); int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index f3484509f8..7de1e5be66 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -91,7 +91,8 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken); SBuffer exprTreeToBinary(tExprNode* pExprTree); -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode); +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +tExprNode* exprTreeFromTableName(const char* tbnameCond); #ifdef __cplusplus } diff --git a/src/query/inc/tvariant.h b/src/query/inc/tvariant.h index 7ed5aa8dbe..9e11e6c294 100644 --- a/src/query/inc/tvariant.h +++ b/src/query/inc/tvariant.h @@ -17,6 +17,7 @@ #define TDENGINE_TVARIANT_H #include "tstoken.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -31,6 +32,7 @@ typedef struct tVariant { double dKey; char * pz; wchar_t *wpz; + SArray *arr; // only for 'in' query to hold value list, not value for a field }; } tVariant; diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 14f340b3d3..b262429e5d 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -30,6 +30,7 @@ #include "tarray.h" #include "tskiplist.h" #include "queryLog.h" +#include "tsdbMain.h" /* * @@ -521,38 +522,48 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { pCond->start = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->start->v, &queryColInfo->q); pCond->start->optr = queryColInfo->optr; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { pCond->end = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->end->v, &queryColInfo->q); pCond->end->optr = queryColInfo->optr; + + } else if (optr == TSDB_RELATION_IN) { + printf("relation is in\n"); + + } else if (optr == TSDB_RELATION_LIKE) { + printf("relation is like\n"); + } return TSDB_CODE_SUCCESS; } -static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t type, SArray* result) { +static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = NULL; - - if (pCond->start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->start->v.i64Key, type, TSDB_ORDER_ASC); + int32_t type = pQueryInfo->q.nType; + + SQueryCond cond = { 0 }; + setQueryCond(pQueryInfo, &cond); + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v.i64Key, type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->end->v.i64Key, type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v.i64Key, type, TSDB_ORDER_DESC); } __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type, 0); - if (pCond->start != NULL) { - int32_t optr = pCond->start->optr; + if (cond.start != NULL) { + int32_t optr = cond.start->optr; if (optr == TSDB_RELATION_EQUAL) { while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); if (ret == 0) { taosArrayPush(result, SL_GET_NODE_DATA(pNode)); } else { @@ -567,7 +578,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); assert(ret >= 0); } @@ -584,7 +595,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty assert(0); } } else { - int32_t optr = pCond->end->optr; + int32_t optr = cond.end->optr; if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { bool comp = true; @@ -594,7 +605,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->end->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.end->v.i64Key); assert(ret <= 0); } @@ -620,6 +631,8 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty // DEFAULT_COMP(p1, p2); //} +// develop_old mgmtSTableQuery for merge & intersect + int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { // assert(pFinalRes->pRes == 0); // @@ -770,19 +783,55 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilte taosArrayCopy(pResult, array); } -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, - SBinaryFilterSupp *param) { + +static void tSQLBinaryTraverseOnSkipList( + tExprNode *pExpr, + SArray *pResult, + SSkipList *pSkipList, + SBinaryFilterSupp *param +) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); } } + tSkipListDestroyIter(iter); } + + +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + STable* table = *(STable**) SL_GET_NODE_DATA(pNode); + + if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.arr); + } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(table->name, pQueryInfo->q.pz); + } + } else { + // TODO: other columns + } + + if (addToResult) { + taosArrayPush(result, (void*)&table); + } + } + + tSkipListDestroyIter(iter); +} + + + // post-root order traverse syntax tree void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { @@ -792,95 +841,99 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; - // recursive traverse left child branch - if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; - - if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { - /** - * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. - * Since no index presented, the filter operation is done by scan all elements in the result set. - * - * if the query is a high selectivity filter, only small portion of meters are retrieved. - */ - exprTreeTraverseImpl(pExpr, result, param); - } else if (weight == 0) { - /** - * apply the hierarchical expression to every node in skiplist for find the qualified nodes - */ - assert(taosArrayGetSize(result) == 0); - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } else if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { - SArray* rLeft = taosArrayInit(10, POINTER_BYTES); - SArray* rRight = taosArrayInit(10, POINTER_BYTES); - - tExprTreeTraverse(pLeft, pSkipList, rLeft, param); - tExprTreeTraverse(pRight, pSkipList, rRight, param); - - if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS - intersect(rLeft, rRight, result); - } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - merge(rLeft, rRight, result); - } else { - assert(false); - } - - taosArrayDestroy(rLeft); - taosArrayDestroy(rRight); - } else { - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } - - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); - } - } else { // column project + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { tSQLListTraverseOnResult(pExpr, param->fp, result); - } else { - tQueryInfo *pQueryInfo = pExpr->_node.info; - - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { - SQueryCond cond = {0}; - /*int32_t ret = */ setQueryCond(pQueryInfo, &cond); - tQueryOnSkipList(pSkipList, &cond, pQueryInfo->q.nType, result); - } else { - /* Brutal force scan the whole skip list to find the appropriate result, - * since the filter is not applied to indexed column. - */ - assert(0); -// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); - } + return; } + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + tQueryIndexColumn(pSkipList, pQueryInfo, result); + } else { + tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + } + + return; } + + // recursive traverse left child branch + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + + if (weight == 0 ) { + if (taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ + exprTreeTraverseImpl(pExpr, result, param); + } else { + /** + * apply the hierarchical expression to every node in skiplist for find the qualified nodes + */ + assert(taosArrayGetSize(result) == 0); + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); + } + return; + } + + if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { + SArray* rLeft = taosArrayInit(10, POINTER_BYTES); + SArray* rRight = taosArrayInit(10, POINTER_BYTES); + + tExprTreeTraverse(pLeft, pSkipList, rLeft, param); + tExprTreeTraverse(pRight, pSkipList, rRight, param); + + if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS + intersect(rLeft, rRight, result); + } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + merge(rLeft, rRight, result); + } else { + assert(false); + } + + taosArrayDestroy(rLeft); + taosArrayDestroy(rRight); + return; + } + + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->_node.optr == TSDB_RELATION_AND); + + tExprNode *pFirst = NULL; + tExprNode *pSecond = NULL; + if (pLeft->_node.hasPK == 1) { + pFirst = pLeft; + pSecond = pRight; + } else { + pFirst = pRight; + pSecond = pLeft; + } + + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + + // we filter the result based on the skiplist index in the first place + tExprTreeTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skip list index as a parameter + */ + tExprTreeTraverse(pSecond, NULL, result, param); } + void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, char *, int32_t)) { if (pExprs == NULL) { @@ -1032,49 +1085,143 @@ SBuffer exprTreeToBinary(tExprNode* pExprTree) { return buf; } -static void exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { +static tExprNode* exprTreeFromBinaryImpl(SBuffer* pBuf) { tExprNode* pExpr = calloc(1, sizeof(tExprNode)); - tbufReadToBuffer(pBuf, &pExpr->nodeType, sizeof(pExpr->nodeType)); + pExpr->nodeType = tbufReadUint8(pBuf); if (pExpr->nodeType == TSQL_NODE_VALUE) { tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + pExpr->pVal = pVal; - tbufReadToBuffer(pBuf, &pVal->nType, sizeof(pVal->nType)); + pVal->nType = tbufReadUint32(pBuf); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen)); pVal->pz = calloc(1, pVal->nLen + 1); tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen); } else { - tbufReadToBuffer(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + pVal->i64Key = tbufReadInt64(pBuf); } - pExpr->pVal = pVal; } else if (pExpr->nodeType == TSQL_NODE_COL) { SSchema* pSchema = calloc(1, sizeof(SSchema)); - tbufReadToBuffer(pBuf, &pSchema->colId, sizeof(pSchema->colId)); - tbufReadToBuffer(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); - tbufReadToBuffer(pBuf, &pSchema->type, sizeof(pSchema->type)); - + if (pSchema == NULL) { + // TODO: + } + pExpr->pSchema = pSchema; + + pSchema->colId = tbufReadInt16(pBuf); + pSchema->bytes = tbufReadInt16(pBuf); + pSchema->type = tbufReadUint8(pBuf); tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN); - pExpr->pSchema = pSchema; } else if (pExpr->nodeType == TSQL_NODE_EXPR) { - tbufReadToBuffer(pBuf, &pExpr->_node.optr, sizeof(pExpr->_node.optr)); - tbufReadToBuffer(pBuf, &pExpr->_node.hasPK, sizeof(pExpr->_node.hasPK)); - - exprTreeFromBinaryImpl(&pExpr->_node.pLeft, pBuf); - exprTreeFromBinaryImpl(&pExpr->_node.pRight, pBuf); + pExpr->_node.optr = tbufReadUint8(pBuf); + pExpr->_node.hasPK = tbufReadUint8(pBuf); + pExpr->_node.pLeft = exprTreeFromBinaryImpl(pBuf); + pExpr->_node.pRight = exprTreeFromBinaryImpl(pBuf); assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); } - *pExprTree = pExpr; + return pExpr; } -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode) { +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { + if (size == 0) { + return NULL; + } SBuffer rbuf = {0}; - /*int32_t code =*/ tbufBeginRead(&rbuf, pBuf, size); - exprTreeFromBinaryImpl(pExprNode, &rbuf); - return TSDB_CODE_SUCCESS; + tbufBeginRead(&rbuf, pBuf, size); + return exprTreeFromBinaryImpl(&rbuf); } +tExprNode* exprTreeFromTableName(const char* tbnameCond) { + if (!tbnameCond) { + return NULL; + } + + tExprNode* expr = calloc(1, sizeof(tExprNode)); + if (expr == NULL) { + // TODO: + } + expr->nodeType = TSQL_NODE_EXPR; + + tExprNode* left = calloc(1, sizeof(tExprNode)); + if (left == NULL) { + // TODO: + } + expr->_node.pLeft = left; + + left->nodeType = TSQL_NODE_COL; + SSchema* pSchema = calloc(1, sizeof(SSchema)); + if (pSchema == NULL) { + // TODO: + } + left->pSchema = pSchema; + + pSchema->type = TSDB_DATA_TYPE_BINARY; + pSchema->bytes = TSDB_TABLE_NAME_LEN; + strcpy(pSchema->name, TSQL_TBNAME_L); + pSchema->colId = -1; + + tExprNode* right = calloc(1, sizeof(tExprNode)); + if (right == NULL) { + // TODO + } + expr->_node.pRight = right; + + if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; + expr->_node.optr = TSDB_RELATION_LIKE; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_BINARY; + size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; + pVal->pz = malloc(len); + if (pVal->pz == NULL) { + // TODO: + } + memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); + pVal->nLen = (int32_t)len; + + } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; + expr->_node.optr = TSDB_RELATION_IN; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_ARRAY; + pVal->arr = taosArrayInit(2, sizeof(char*)); + + const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; + for (const char *e = cond; *e != 0; e++) { + if (*e == TS_PATH_DELIMITER[0]) { + cond = e + 1; + } else if (*e == ',') { + size_t len = e - cond + 1; + char* p = malloc( len ); + memcpy(p, cond, len); + p[len - 1] = 0; + cond += len; + taosArrayPush(pVal->arr, &p); + } + } + + if (*cond != 0) { + char* p = strdup( cond ); + taosArrayPush(pVal->arr, &p); + } + + taosArraySortString(pVal->arr); + } + + return expr; +} \ No newline at end of file diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e2f3b2f655..9b410ce1e5 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -990,9 +990,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int16_t *type, int16_t *bytes) { - char *groupbyColumnData = NULL; - +static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, SArray* pDataBlock) { SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { @@ -1015,12 +1013,22 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 *type = pQuery->colList[colIndex].type; *bytes = pQuery->colList[colIndex].bytes; - - // groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); - break; + /* + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare + * stage, the remain meter may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding meter schema is reinforced. + */ + int32_t numOfCols = taosArrayGetSize(pDataBlock); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData *p = taosArrayGet(pDataBlock, i); + if (pColIndex->colId == p->info.colId) { + return p->pData; + } + } } - - return groupbyColumnData; + + return NULL; } static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { @@ -1091,8 +1099,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS char *groupbyColumnData = NULL; if (groupbyStateValue) { - assert(0); - // groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); + groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); } for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { @@ -5273,7 +5280,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, - char **tagCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5286,6 +5293,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); pQueryMsg->queryType = htons(pQueryMsg->queryType); + pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); @@ -5429,13 +5437,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } - // the tag query condition expression string is located at the end of query msg - if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen); - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); - pMsg += pQueryMsg->tagCondLen; - } - if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { @@ -5447,9 +5448,24 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pTagCol->numOfFilters = 0; (*tagCols)[i] = *pTagCol; + pMsg += sizeof(SColumnInfo); } } + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryMsg->tagCondLen); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; + } + + if (*pMsg != 0) { + size_t len = strlen(pMsg) + 1; + *tbnameCond = malloc(len); + strcpy(*tbnameCond, pMsg); + pMsg += len; + } + qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 @@ -6047,13 +6063,13 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL; + char * tagCond = NULL, *tbnameCond = NULL; SArray * pTableIdList = NULL; SSqlFuncMsg **pExprMsg = NULL; SColIndex * pGroupColIndex = NULL; SColumnInfo* pTagColumnInfo = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex, &pTagColumnInfo)) != + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != TSDB_CODE_SUCCESS) { return code; } @@ -6088,9 +6104,16 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) STableId *id = taosArrayGet(pTableIdList, 0); id->uid = -1; // todo fix me - - /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, - pQueryMsg->numOfGroupCols); + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + // todo handle the error + /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, + numOfGroupByCols); if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; goto _query_over; @@ -6112,6 +6135,8 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); _query_over: + tfree(tagCond); + tfree(tbnameCond); taosArrayDestroy(pTableIdList); // if failed to add ref for all meters in this query, abort current query diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index f0addb733b..e90cc3a44d 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -130,6 +130,17 @@ void tVariantDestroy(tVariant *pVar) { tfree(pVar->pz); pVar->nLen = 0; } + + // NOTE: this is only for string array + if (pVar->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pVar->arr); + for(size_t i = 0; i < num; i++) { + void* p = taosArrayGetP(pVar->arr, i); + free(p); + } + taosArrayDestroy(pVar->arr); + pVar->arr = NULL; + } } void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { @@ -145,6 +156,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { pDst->pz = calloc(1, len); memcpy(pDst->pz, pSrc->pz, len); + return; + } + + // this is only for string array + if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(char*)); + for(size_t i = 0; i < num; i++) { + char* p = (char*)taosArrayGetP(pSrc->arr, i); + char* n = strdup(p); + taosArrayPush(pDst->arr, &n); + } } } diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index 2e70f26269..41156ce8ff 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -58,7 +58,7 @@ TEST(testCase, patternMatchTest) { EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 1, &info); + ret = patternMatch("abc%f_", str, 1, &info); // pattern string is longe than the size EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); str = "abcdefgabcdeju"; @@ -72,4 +72,8 @@ TEST(testCase, patternMatchTest) { str = "abcdefgabcdeju"; ret = patternMatch("a__", str, 2, &info); EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "carzero"; + ret = patternMatch("%o", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2357cc892f..eb35be5383 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1213,7 +1213,9 @@ void filterPrepare(void* expr, void* param) { pInfo->compare = getComparFunc(pSchema->type, pCond->nType, pInfo->optr); tVariantAssign(&pInfo->q, pCond); - tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + if (pInfo->optr != TSDB_RELATION_IN) { + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + } } int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { @@ -1327,12 +1329,9 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - size_t num = taosArrayGetSize(pTableList); - - SArray* sa = taosArrayInit(num, sizeof(SPair)); - for(int32_t i = 0; i < num; ++i) { + SArray* sa = taosArrayInit(size, sizeof(SPair)); + for(int32_t i = 0; i < size; ++i) { STable* pTable = taosArrayGetP(pTableList, i); - SPair p = {.first = pTable}; taosArrayPush(sa, &p); } @@ -1358,16 +1357,26 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + char* val = NULL; int8_t type = pInfo->sch.type; + + if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + val = pTable->name; + type = TSDB_DATA_TYPE_BINARY; + } else { + val = dataRowTuple(pTable->tagVal); // todo not only the first column + } int32_t ret = 0; - if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { - ret = pInfo->compare(val, pInfo->q.pz); + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (pInfo->optr == TSDB_RELATION_IN) { + ret = pInfo->compare(val, pInfo->q.arr); + } else { + ret = pInfo->compare(val, pInfo->q.pz); + } } else { tVariant t = {0}; tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); - ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); } @@ -1393,6 +1402,9 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_LIKE: { return ret == 0; } + case TSDB_RELATION_IN: { + return ret == 1; + } default: assert(false); @@ -1400,6 +1412,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return true; } + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression STSchema* pSchema = pSTable->tagSchema; @@ -1422,12 +1435,23 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) tExprTreeDestroy(&pExpr, destroyHelper); convertQueryResult(pRes, pTableList); + taosArrayDestroy(pTableList); + free(schema); return TSDB_CODE_SUCCESS; } -int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols) { - + +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupInfo, + SColIndex *pColIndex, + int32_t numOfCols +) { STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pSTable == NULL) { uError("failed to get stable, uid:%" PRIu64, uid); @@ -1437,31 +1461,35 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, SArray* res = taosArrayInit(8, POINTER_BYTES); STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pSTable); - if (pTagCond == NULL || len == 0) { // no tags condition, all tables created according to this stable are involved + // no tags and tbname condition, all child tables of this stable are involved + if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { int32_t ret = getAllTableIdList(tsdb, uid, res); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(res); - return ret; + if (ret == TSDB_CODE_SUCCESS) { + pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); } - - pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); taosArrayDestroy(res); return ret; } - tExprNode* pExprNode = NULL; int32_t ret = TSDB_CODE_SUCCESS; - // failed to build expression, no result, return immediately - if ((ret = exprTreeFromBinary(pTagCond, len, &pExprNode) != TSDB_CODE_SUCCESS) || (pExprNode == NULL)) { - uError("stable:%" PRIu64 ", failed to deserialize expression tree, error exists", uid); - taosArrayDestroy(res); - return ret; + tExprNode* expr = exprTreeFromTableName(tbnameCond); + tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); + if (tagExpr != NULL) { + if (expr == NULL) { + expr = tagExpr; + } else { + tExprNode* tbnameExpr = expr; + expr = calloc(1, sizeof(tExprNode)); + expr->nodeType = TSQL_NODE_EXPR; + expr->_node.optr = tagNameRelType; + expr->_node.pLeft = tagExpr; + expr->_node.pRight = tbnameExpr; + } } - doQueryTableList(pSTable, res, pExprNode); - + doQueryTableList(pSTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 6fab24d51a..dbb74edf07 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -53,7 +53,7 @@ void* taosArrayPush(SArray* pArray, void* pData); * * @param pArray */ -void taosArrayPop(SArray* pArray); +void* taosArrayPop(SArray* pArray); /** * get the data from array @@ -112,6 +112,34 @@ SArray* taosArrayClone(SArray* pSrc); */ void taosArrayDestroy(SArray* pArray); +/** + * sort the array + * @param pArray + * @param compar + */ +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); + +/** + * sort string array + * @param pArray + */ +void taosArraySortString(SArray* pArray); + +/** + * search the array + * @param pArray + * @param compar + * @param key + */ +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key); + +/** + * search the array + * @param pArray + * @param key + */ +char* taosArraySearchString(const SArray* pArray, const char* key); + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index d52031fc6a..9bc0fd9f43 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -120,7 +120,7 @@ void tbufWriteString(SBuffer* buf, const char* str); TBUFFER_DEFINE_FUNCTION(bool, Bool) TBUFFER_DEFINE_FUNCTION(char, Char) TBUFFER_DEFINE_FUNCTION(int8_t, Int8) -TBUFFER_DEFINE_FUNCTION(uint8_t, Unt8) +TBUFFER_DEFINE_FUNCTION(uint8_t, Uint8) TBUFFER_DEFINE_FUNCTION(int16_t, Int16) TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16) TBUFFER_DEFINE_FUNCTION(int32_t, Int32) diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index c025958438..51684e767c 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -27,7 +27,7 @@ void* taosArrayInit(size_t size, size_t elemSize) { return NULL; } - pArray->pData = calloc(size, elemSize * size); + pArray->pData = calloc(size, elemSize); if (pArray->pData == NULL) { free(pArray); return NULL; @@ -76,12 +76,14 @@ void* taosArrayPush(SArray* pArray, void* pData) { return dst; } -void taosArrayPop(SArray* pArray) { - if (pArray == NULL || pArray->size == 0) { - return; - } +void* taosArrayPop(SArray* pArray) { + assert( pArray != NULL ); + if (pArray->size == 0) { + return NULL; + } pArray->size -= 1; + return TARRAY_GET_ELEM(pArray, pArray->size); } void* taosArrayGet(const SArray* pArray, size_t index) { @@ -183,3 +185,40 @@ void taosArrayDestroy(SArray* pArray) { free(pArray->pData); free(pArray); } + +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { + assert(pArray != NULL); + assert(compar != NULL); + + qsort(pArray->pData, pArray->size, pArray->elemSize, compar); +} + +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) { + assert(pArray != NULL); + assert(compar != NULL); + assert(key != NULL); + + return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar); +} + +static int taosArrayCompareString(const void* a, const void* b) { + const char* x = *(const char**)a; + const char* y = *(const char**)b; + return strcmp(x, y); +} + +void taosArraySortString(SArray* pArray) { + assert(pArray != NULL); + qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); +} + +char* taosArraySearchString(const SArray* pArray, const char* key) { + assert(pArray != NULL); + assert(key != NULL); + + void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); + if (p == NULL) { + return NULL; + } + return *(char**)p; +} \ No newline at end of file diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 4505ea533a..4d78a8cf0a 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,42 +1,34 @@ #include "taosdef.h" #include "tcompare.h" - +#include #include "tutil.h" int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int64_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { @@ -69,12 +61,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + return (int32_t)strcmp(pLeft, pRight); } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -228,6 +215,11 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } +static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { + const SArray* arr = (const SArray*)pRight; + return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1; +} + static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; @@ -250,7 +242,6 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { // assert(type == filterDataType); - if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { comparFn = compareInt64Val; } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { @@ -259,6 +250,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) break; } + case TSDB_DATA_TYPE_BOOL: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { comparFn = compareInt32Val; @@ -267,6 +259,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { @@ -276,12 +269,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_BINARY: { - assert(filterDataType == TSDB_DATA_TYPE_BINARY); - if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrPatternComp; + + } else if (optr == TSDB_RELATION_IN) { + assert(filterDataType == TSDB_DATA_TYPE_ARRAY); + comparFn = compareFindStrInArray; + } else { /* normal relational comparFn */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrVal; } diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 9847300024..e5a70740f1 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -75,7 +75,7 @@ int main(int argc, char *argv[]) { doQuery(taos, "create database if not exists test"); doQuery(taos, "use test"); - doQuery(taos, "insert into tm99 values('2020-01-01 1:1:1', 99);"); + doQuery(taos, "select count(*),k,sum(k) from m1 group by k"); // doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);"); @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) { // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:7', 7);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:8', 8);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:9', 9);"); - doQuery(taos, "select sum(k),count(*) from m1 group by a"); +// doQuery(taos, "select sum(k),count(*) from m1 group by a"); taos_close(taos); return 0; diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py new file mode 100644 index 0000000000..7c4474f7ad --- /dev/null +++ b/tests/pytest/insert/float.py @@ -0,0 +1,147 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import datetime + +import taos + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + cmd = 'create table tb (ts timestamp, speed float)' + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = 'insert into tb values (now, -3.40E+38)' + tdLog.info(cmd) + tdSql.execute(cmd) + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1a, 3.40E+308)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'select * from tb order by ts desc' + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(1) + + tdLog.info("=============== step3") + cmd = "insert into tb values (now+2a, 2.85)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(2) + + if ((abs(tdSql.getData(0, 1) - 2.850000)) > 1.0e-7): + tdLog.exit("data is not 2.850000") + + tdLog.info("=============== step4") + cmd = "insert into tb values (now+3a, 3.4)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(3) + if (abs(tdSql.getData(0, 1) - 3.400000) > 1.0e-7): + tdLog.exit("data is not 3.400000") + + tdLog.info("=============== step5") + cmd = "insert into tb values (now+4a, a2)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit("This test failed: \ + insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+4a, 0)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(4) + if (abs(tdSql.getData(0, 1) - 0.000000) != 0): + tdLog.exit("data is not 0.000000") + + tdLog.info("=============== step6") + cmd = "insert into tb values (now+5a, 2a)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+5a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(5) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.info("data is not 2.000000") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6a, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+6a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.exit("data is not 2.000000") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py new file mode 100644 index 0000000000..8d38337576 --- /dev/null +++ b/tests/pytest/insert/int.py @@ -0,0 +1,184 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import datetime + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + tdSql.execute('create table tb (ts timestamp, speed int)') + + cmd = 'insert into tb values (now, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(1) + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1m, -2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+1m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(2) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step3") + cmd = 'insert into tb values (now+2m, 2147483647)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(3) + if(tdSql.getData(0, 1) != 2147483647): + tdLog.exit("data is not 2147483647") + + tdLog.info("=============== step4") + cmd = 'insert into tb values (now+3m, 2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+3m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(4) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step5") + cmd = 'insert into tb values (now+4m, a2)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+4m, 0)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(5) + + if(tdSql.getData(0, 1) != 0): + tdLog.exit("data is not 0") + + tdLog.info("=============== step6") + cmd = 'insert into tb values (now+5m, 2a)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+5m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(6) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6m, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+6m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(7) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step8") + cmd = 'insert into tb values (now+8m, "null")' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(8) + + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step9") + cmd = "insert into tb values (now+9m, 'null')" + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(9) + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step10") + cmd = 'insert into tb values (now+10m, -123)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(10) + + if (tdSql.getData(0, 1) != -123): + tdLog.exit("data is not -123") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/__init__.py b/tests/pytest/query/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/query/tbname.py b/tests/pytest/query/tbname.py new file mode 100644 index 0000000000..d2799efa25 --- /dev/null +++ b/tests/pytest/query/tbname.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute('create table cars (ts timestamp, speed int) tags(id int)') + tdSql.execute("create table carzero using cars tags(0)") + tdSql.execute("create table carone using cars tags(1)") + tdSql.execute("create table cartwo using cars tags(2)") + + tdSql.execute("insert into carzero values(now, 100) carone values(now, 110)") + + tdSql.query("select * from cars where tbname in ('carzero', 'carone')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 or tbname in ('carzero', 'cartwo')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where id=1 and tbname in ('carzero', 'cartwo')") + tdSql.checkRows(0) + + tdSql.query("select * from cars where id=0 and tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + """ + tdSql.query("select * from cars where tbname like 'car%'") + tdSql.checkRows(2) + + tdSql.cursor.execute("use db") + tdSql.query("select * from cars where tbname like '%o'") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 and tbname like 'car%') + tdSql.checkRows(1) + + tdSql.query("select * from cars where id = 1 and tbname like '%o') + tdSql.checkRows(0) + + tdSql.query("select * from cars where id = 1 or tbname like '%o') + tdSql.checkRows(2) + """ + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index b77c1aa142..58345e13ee 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1,3 +1,9 @@ #!/bin/bash python3 ./test.py -f insert/basic.py $1 python3 ./test.py -s $1 +sleep 1 +python3 ./test.py -f insert/int.py $1 +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py -f insert/float.py $1 +python3 ./test.py -s $1 diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 55ec46e526..479406a00b 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -100,7 +100,7 @@ if __name__ == "__main__": tdDnodes.deploy(1) tdDnodes.start(1) conn = taos.connect( - host='192.168.0.1', + host='127.0.0.1', config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 4bd1b1e9d1..1771c23620 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -67,12 +67,16 @@ class TDCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() case.init(conn) - case.run() + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.notice("%s failed: %s" % (__file__, fileName)) case.stop() runNum += 1 continue - tdLog.notice("total %d Linux test case(s) executed" % (runNum)) + tdLog.success("total %d Linux test case(s) executed" % (runNum)) def runAllWindows(self, conn): # TODO: load all Windows cases here diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 6a9c2607e6..51eed3d25a 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -19,12 +19,19 @@ from util.log import * class TDSimClient: + def __init__(self): + self.testCluster = False + def init(self, path): + self.__init__() self.path = path def getCfgDir(self): return self.cfgDir + def setTestCluster(self, value): + self.testCluster = value + def cfg(self, option, value): cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: @@ -55,8 +62,9 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") self.cfg("numOfThreadsPerCore", "2.0") @@ -128,11 +136,12 @@ class TDDnode: if self.testCluster: self.startIP() - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) self.cfg("dataDir", self.dataDir) self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") @@ -291,10 +300,6 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - self.sim = TDSimClient() - self.sim.init(self.path) - self.sim.deploy() - def setTestCluster(self, value): self.testCluster = value @@ -302,6 +307,11 @@ class TDDnodes: self.valgrind = value def deploy(self, index): + self.sim = TDSimClient() + self.sim.init(self.path) + self.sim.setTestCluster(self.testCluster) + self.sim.deploy() + self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b4ac845bc8..0008468130 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -63,7 +63,7 @@ class TDSql: def checkRows(self, expectRows): if self.queryRows != expectRows: tdLog.exit( - "sql:%.40s, queryRows:%d != expect:%d" % + "failed: sql:%.40s, queryRows:%d != expect:%d" % (self.sql, self.queryRows, expectRows)) tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) diff --git a/tests/test-all.sh b/tests/test-all.sh index dee89b9dc5..6943dd47a7 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,15 +25,17 @@ if [ "$totalFailed" -ne "0" ]; then fi cd ../pytest -./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt +./simpletest.sh 2>&1 | tee pytest-out.txt totalPySuccess=`grep 'successfully executed' pytest-out.txt | wc -l` if [ "$totalPySuccess" -gt "0" ]; then + grep 'successfully executed' pytest-out.txt echo -e "${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" fi -totalPyFailed=`grep 'failed' pytest-out.txt | wc -l` +totalPyFailed=`grep 'failed\|fault' pytest-out.txt | wc -l` if [ "$totalPyFailed" -ne "0" ]; then + cat pytest-out.txt echo -e "${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" exit $totalPyFailed fi