diff --git a/Jenkinsfile b/Jenkinsfile index 0ccf302ba6..780f37ca2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,9 @@ properties([pipelineTriggers([githubPush()])]) node { git url: 'https://github.com/taosdata/TDengine.git' } + def skipbuild=0 + def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -146,6 +148,7 @@ pipeline { skipbuild='2' skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) println skipbuild + } sh''' rm -rf ${WORKSPACE}.tes @@ -160,6 +163,7 @@ pipeline { changeRequest() expression{ return skipbuild.trim() == '2' + } } } diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b3674a7bf5..f2d25c1e84 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -61,6 +61,7 @@ typedef struct SJoinSupporter { uint64_t uid; // query table uid SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution SArray* exprList; + SArray* colCond; SFieldInfo fieldsInfo; STagCond tagCond; SGroupbyExpr groupInfo; // group by info @@ -244,8 +245,9 @@ SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw); int32_t tscTagCondCopy(STagCond* dest, const STagCond* src); +int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx); void tscTagCondRelease(STagCond* pCond); - +void tscColCondRelease(SArray** pCond); void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); bool tscShouldBeFreed(SSqlObj* pSql); @@ -340,7 +342,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromSTableMeta(STableMeta** pChild, const char* name, size_t *tableMetaCapacity); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo); @@ -355,6 +357,7 @@ char* strdup_throw(const char* str); bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); +STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx); void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ac5adcbbb4..8d579b375a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -84,9 +84,14 @@ typedef struct SParamInfo { } SParamInfo; typedef struct SBoundColumn { - bool hasVal; // denote if current column has bound or not - int32_t offset; // all column offset value + int32_t offset; // all column offset value + int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future + uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val) } SBoundColumn; +typedef enum { + VAL_STAT_HAS = 0x0, // 0 means has val + VAL_STAT_NONE = 0x01, // 1 means no val +} EValStat; typedef struct { uint16_t schemaColIdx; @@ -99,32 +104,106 @@ typedef enum _COL_ORDER_STATUS { ORDER_STATUS_ORDERED = 1, ORDER_STATUS_DISORDERED = 2, } EOrderStatus; - typedef struct SParsedDataColInfo { int16_t numOfCols; int16_t numOfBound; - int32_t * boundedColumns; // bounded column idx according to schema + uint16_t flen; // TODO: get from STSchema + uint16_t allNullLen; // TODO: get from STSchema + uint16_t extendedVarLen; + int32_t * boundedColumns; // bound column idx according to schema SBoundColumn * cols; SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bounded columns: + int8_t orderStatus; // bound columns } SParsedDataColInfo; -#define IS_DATA_COL_ORDERED(s) ((s) == (int8_t)ORDER_STATUS_ORDERED) +#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) typedef struct { - SSchema * pSchema; - int16_t sversion; - int32_t flen; - uint16_t nCols; - void * buf; - void * pDataBlock; - SSubmitBlk *pSubmitBlk; + int32_t dataLen; // len of SDataRow + int32_t kvLen; // len of SKVRow +} SMemRowInfo; +typedef struct { + uint8_t memRowType; + uint8_t compareStat; // 0 unknown, 1 need compare, 2 no need + TDRowTLenT dataRowInitLen; + TDRowTLenT kvRowInitLen; + SMemRowInfo *rowInfo; } SMemRowBuilder; -typedef struct { - TDRowLenT allNullLen; -} SMemRowHelper; +typedef enum { + ROW_COMPARE_UNKNOWN = 0, + ROW_COMPARE_NEED = 1, + ROW_COMPARE_NO_NEED = 2, +} ERowCompareStat; +int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec); + +int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, + int32_t allNullLen); +void destroyMemRowBuilder(SMemRowBuilder *pBuilder); + +/** + * @brief + * + * @param memRowType + * @param spd + * @param idx the absolute bound index of columns + * @return FORCE_INLINE + */ +static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, + int32_t idx, int32_t *toffset, int16_t *colId) { + int32_t schemaIdx = 0; + if (IS_DATA_COL_ORDERED(spd)) { + schemaIdx = spd->boundedColumns[idx]; + if (isDataRowT(memRowType)) { + *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart + } else { + *toffset = idx * sizeof(SColIdx); // the offset of SColIdx + } + } else { + ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); + schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; + if (isDataRowT(memRowType)) { + *toffset = (spd->cols + schemaIdx)->toffset; + } else { + *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); + } + } + *colId = pSchema[schemaIdx].colId; +} + +/** + * @brief Applicable to consume by multi-columns + * + * @param row + * @param value + * @param isCopyVarData In some scenario, the varVal is copied to row directly before calling tdAppend***ColVal() + * @param colId + * @param colType + * @param idx index in SSchema + * @param pBuilder + * @param spd + * @return FORCE_INLINE + */ +static FORCE_INLINE void tscAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, + int8_t colType, int32_t toffset, SMemRowBuilder *pBuilder, + int32_t rowNum) { + tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset); + if (pBuilder->compareStat == ROW_COMPARE_NEED) { + SMemRowInfo *pRowInfo = pBuilder->rowInfo + rowNum; + tdGetColAppendDeltaLen(value, colType, &pRowInfo->dataLen, &pRowInfo->kvLen); + } +} + +// Applicable to consume by one row +static FORCE_INLINE void tscAppendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, + int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen, + uint8_t compareStat) { + tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset); + if (compareStat == ROW_COMPARE_NEED) { + tdGetColAppendDeltaLen(value, colType, dataLen, kvLen); + } +} typedef struct STableDataBlocks { SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client @@ -146,7 +225,7 @@ typedef struct STableDataBlocks { uint32_t numOfAllocedParams; uint32_t numOfParams; SParamInfo * params; - SMemRowHelper rowHelper; + SMemRowBuilder rowBuilder; } STableDataBlocks; typedef struct { @@ -435,8 +514,398 @@ int16_t getNewResColId(SSqlCmd* pCmd); int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); -int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen); -int32_t getExtendedRowSize(STableComInfo *tinfo); +static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { + ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize); + return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen; +} + +static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) { + if (isDataRow(row)) { + if (kvLen < (dataLen * KVRatioConvert)) { + memRowSetConvert(row); + } + } else if (kvLen > dataLen) { + memRowSetConvert(row); + } +} + +static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) { + memRowSetType(row, memRowType); + if (isDataRowT(memRowType)) { + dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion); + dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen)); + } else { + ASSERT(nBoundCols > 0); + memRowSetKvVersion(row, pBlock->pTableMeta->sversion); + kvRowSetNCols(memRowKvBody(row), nBoundCols); + kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); + } +} +/** + * TODO: Move to tdataformat.h and refactor when STSchema available. + * - fetch flen and toffset from STSChema and remove param spd + */ +static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, + SParsedDataColInfo *spd) { + ASSERT(isKvRow(src)); + SKVRow kvRow = memRowKvBody(src); + SDataRow dataRow = memRowDataBody(dest); + + memRowSetType(dest, SMEM_ROW_DATA); + dataRowSetVersion(dataRow, memRowKvVersion(src)); + dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen)); + + int32_t kvIdx = 0; + for (int i = 0; i < nCols; ++i) { + SSchema *schema = pSchema + i; + void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx); + tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type, + (spd->cols + i)->toffset); + } +} + +// TODO: Move to tdataformat.h and refactor when STSchema available. +static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, + SParsedDataColInfo *spd) { + ASSERT(isDataRow(src)); + + SDataRow dataRow = memRowDataBody(src); + SKVRow kvRow = memRowKvBody(dest); + + memRowSetType(dest, SMEM_ROW_KV); + memRowSetKvVersion(kvRow, dataRowVersion(dataRow)); + kvRowSetNCols(kvRow, nBoundCols); + kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); + + int32_t toffset = 0, kvOffset = 0; + for (int i = 0; i < nCols; ++i) { + if ((spd->cols + i)->valStat == VAL_STAT_HAS) { + SSchema *schema = pSchema + i; + toffset = (spd->cols + i)->toffset; + void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE); + tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset); + kvOffset += sizeof(SColIdx); + } + } +} + +// TODO: Move to tdataformat.h and refactor when STSchema available. +static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) { + STableMeta * pTableMeta = pBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + SSchema * pSchema = tscGetTableSchema(pTableMeta); + SParsedDataColInfo *spd = &pBlock->boundColumnInfo; + + ASSERT(dest != src); + + if (isDataRow(src)) { + // TODO: Can we use pBlock -> numOfParam directly? + ASSERT(spd->numOfBound > 0); + convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd); + } else { + convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd); + } +} + +static bool isNullStr(SStrToken *pToken) { + return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); +} + +static FORCE_INLINE int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { + errno = 0; + *value = strtold(pToken->z, endPtr); + + // not a valid integer number, return error + if ((*endPtr - pToken->z) != pToken->n) { + return TK_ILLEGAL; + } + + return pToken->type; +} + +static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; +static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; + +static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str, + bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId, + int32_t *dataLen, int32_t *kvLen, uint8_t compareStat) { + int64_t iv; + int32_t ret; + char * endptr = NULL; + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); + } + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { // bool + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + tscAppendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + tscAppendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + tscAppendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, + dataLen, kvLen, compareStat); + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + tscAppendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, + dataLen, kvLen, compareStat); + } else { + return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); + } + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return tscInvalidOperationMsg(msg, "data overflow", pToken->z); + } + + uint8_t tmpVal = (uint8_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_UTINYINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); + } + + uint8_t tmpVal = (uint8_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_SMALLINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); + } + + int16_t tmpVal = (int16_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_USMALLINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); + } + + uint16_t tmpVal = (uint16_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_INT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); + } + + int32_t tmpVal = (int32_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_UINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); + } + + uint32_t tmpVal = (uint32_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_BIGINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); + } + + tscAppendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_UBIGINT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); + } + + uint64_t tmpVal = (uint64_t)iv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_FLOAT: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); + } + + float tmpVal = (float)dv; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_DOUBLE: + if (isNullStr(pToken)) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + double dv; + if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { + return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); + } + + tscAppendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_BINARY: + // binary data cannot be null-terminated char string, otherwise the last char of the string is lost + if (pToken->type == TK_NULL) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); + } + // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); + char *rowEnd = memRowEnd(row); + STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n); + tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_NCHAR: + if (pToken->type == TK_NULL) { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + char * rowEnd = memRowEnd(row); + if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE, + &output)) { + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return tscInvalidOperationMsg(msg, buf, pToken->z); + } + varDataSetLen(rowEnd, output); + tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_TIMESTAMP: { + if (pToken->type == TK_NULL) { + if (primaryKey) { + // When building SKVRow primaryKey, we should not skip even with NULL value. + int64_t tmpVal = 0; + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } else { + tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } + } else { + int64_t tmpVal; + if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { + return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); + } + tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + } + } + + return TSDB_CODE_SUCCESS; +} #ifdef __cplusplus } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 73e4f898c8..b7a2320b07 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -38,43 +38,60 @@ enum { TSDB_USE_CLI_TS = 1, }; -static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; -static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; - static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema, char *str, char **end); - -int32_t getExtendedRowSize(STableComInfo *tinfo) { - return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns; -} -int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) { - pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta - if (pHelper->allNullLen == 0) { - for (uint16_t i = 0; i < nCols; ++i) { - uint8_t type = pSSchema[i].type; - int32_t typeLen = TYPE_BYTES[type]; - pHelper->allNullLen += typeLen; - if (TSDB_DATA_TYPE_BINARY == type) { - pHelper->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - } else if (TSDB_DATA_TYPE_NCHAR == type) { - int len = VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE; - pHelper->allNullLen += len; - } +int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, + int32_t allNullLen) { + ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols)); + if (nRows > 0) { + // already init(bind multiple rows by single column) + if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) { + return TSDB_CODE_SUCCESS; } } - return 0; -} -static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { - errno = 0; - *value = strtold(pToken->z, endPtr); - - // not a valid integer number, return error - if ((*endPtr - pToken->z) != pToken->n) { - return TK_ILLEGAL; + + if (nBoundCols == 0) { // file input + pBuilder->memRowType = SMEM_ROW_DATA; + pBuilder->compareStat = ROW_COMPARE_NO_NEED; + return TSDB_CODE_SUCCESS; + } else { + float boundRatio = ((float)nBoundCols / (float)nCols); + + if (boundRatio < KVRatioKV) { + pBuilder->memRowType = SMEM_ROW_KV; + pBuilder->compareStat = ROW_COMPARE_NO_NEED; + return TSDB_CODE_SUCCESS; + } else if (boundRatio > KVRatioData) { + pBuilder->memRowType = SMEM_ROW_DATA; + pBuilder->compareStat = ROW_COMPARE_NO_NEED; + return TSDB_CODE_SUCCESS; + } + pBuilder->compareStat = ROW_COMPARE_NEED; + + if (boundRatio < KVRatioPredict) { + pBuilder->memRowType = SMEM_ROW_KV; + } else { + pBuilder->memRowType = SMEM_ROW_DATA; + } } - return pToken->type; + pBuilder->dataRowInitLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen; + pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx); + + if (nRows > 0) { + pBuilder->rowInfo = tcalloc(nRows, sizeof(SMemRowInfo)); + if (pBuilder->rowInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int i = 0; i < nRows; ++i) { + (pBuilder->rowInfo + i)->dataLen = pBuilder->dataRowInitLen; + (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen; + } + } + + return TSDB_CODE_SUCCESS; } int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { @@ -146,10 +163,6 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -static bool isNullStr(SStrToken* pToken) { - return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && - (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); -} int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; @@ -400,342 +413,6 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha return TSDB_CODE_SUCCESS; } -static FORCE_INLINE TDRowLenT tsSetPayloadColValue(char *payloadStart, char *payload, int16_t columnId, - uint8_t columnType, const void *value, uint16_t valueLen, TDRowTLenT tOffset) { - payloadColSetId(payload, columnId); - payloadColSetType(payload, columnType); - memcpy(POINTER_SHIFT(payloadStart,tOffset), value, valueLen); - return valueLen; -} - -static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payloadStart, char *primaryKeyStart, - char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec, - TDRowTLenT tOffset, TDRowLenT *sizeAppend, TDRowLenT *dataRowColDeltaLen, - TDRowLenT *kvRowColLen) { - int64_t iv; - int32_t ret; - char * endptr = NULL; - - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z); - } - - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { // bool - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &TRUE_VALUE, - TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &FALSE_VALUE, - TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - } else { - return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]); - } else { - return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z); - } - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return tscInvalidOperationMsg(msg, "data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]); - } - - break; - - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]); - } - - break; - - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z); - } - - int16_t tmpVal = (int16_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]); - } - - break; - - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - *sizeAppend = - tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z); - } - - uint16_t tmpVal = (uint16_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]); - } - - break; - - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return tscInvalidOperationMsg(msg, "int data overflow", pToken->z); - } - - int32_t tmpVal = (int32_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]); - } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z); - } - - uint32_t tmpVal = (uint32_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]); - } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z); - } - - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &iv, - TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]); - } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); - } else { - ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z); - } - - uint64_t tmpVal = (uint64_t)iv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]); - } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_FLOAT), TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal float data", pToken->z); - } - - float tmpVal = (float)dv; - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]); - } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); - } else { - double dv; - if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return tscInvalidOperationMsg(msg, "illegal double data", pToken->z); - } - - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &dv, - TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]); - } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - payloadColSetId(payload, pSchema->colId); - payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart, tOffset), getNullValue(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES); - *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor - return tscInvalidOperationMsg(msg, "string data overflow", pToken->z); - } - // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - - payloadColSetId(payload, pSchema->colId); - payloadColSetType(payload, pSchema->type); - varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), pToken->n); - memcpy(varDataVal(POINTER_SHIFT(payloadStart,tOffset)), pToken->z, pToken->n); - *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + pToken->n); - *dataRowColDeltaLen += (TDRowLenT)(pToken->n - CHAR_BYTES); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + pToken->n); - } - - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - payloadColSetId(payload, pSchema->colId); - payloadColSetType(payload, pSchema->type); - memcpy(POINTER_SHIFT(payloadStart,tOffset), getNullValue(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - payloadColSetId(payload, pSchema->colId); - payloadColSetType(payload, pSchema->type); - if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(POINTER_SHIFT(payloadStart,tOffset)), - pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return tscInvalidOperationMsg(msg, buf, pToken->z); - } - - varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), output); - - *sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + output); - *dataRowColDeltaLen += (TDRowLenT)(output - sizeof(uint32_t)); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + output); - } - break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - // When building SKVRow primaryKey, we should not skip even with NULL value. - int64_t tmpVal = 0; - *sizeAppend = tsSetPayloadColValue(payloadStart, primaryKeyStart, pSchema->colId, pSchema->type, &tmpVal, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); - } else { - *sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, - getNullValue(TSDB_DATA_TYPE_TIMESTAMP), - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); - } - } else { - int64_t tmpVal; - if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) { - return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z); - } - - *sizeAppend = tsSetPayloadColValue(payloadStart, primaryKey ? primaryKeyStart : payload, pSchema->colId, - pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset); - *kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]); - } - - break; - } - } - - return TSDB_CODE_SUCCESS; -} - /* * The server time/client time should not be mixed up in one sql string * Do not employ sort operation is not involved if server time is used. @@ -777,31 +454,24 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i int32_t index = 0; SStrToken sToken = {0}; - SMemRowHelper *pHelper = &pDataBlocks->rowHelper; - char * payload = pDataBlocks->pData + pDataBlocks->size; + char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; - SSchema * schema = tscGetTableSchema(pDataBlocks->pTableMeta); + STableMeta * pTableMeta = pDataBlocks->pTableMeta; + SSchema * schema = tscGetTableSchema(pTableMeta); + SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; + int32_t dataLen = pBuilder->dataRowInitLen; + int32_t kvLen = pBuilder->kvRowInitLen; + bool isParseBindParam = false; - TDRowTLenT dataRowLen = pHelper->allNullLen; - TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE; - TDRowTLenT payloadValOffset = 0; - TDRowLenT colValOffset = 0; - ASSERT(dataRowLen > 0); - - payloadSetNCols(payload, spd->numOfBound); - payloadValOffset = payloadValuesOffset(payload); // rely on payloadNCols - // payloadSetTLen(payload, payloadValOffset); - - char *kvPrimaryKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple - char *kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey + initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql int32_t colIndex = spd->boundedColumns[i]; - char *start = payload + spd->cols[colIndex].offset; + char *start = row + spd->cols[colIndex].offset; SSchema *pSchema = &schema[colIndex]; // get colId here @@ -810,6 +480,9 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i *str += index; if (sToken.type == TK_QUESTION) { + if (!isParseBindParam) { + isParseBindParam = true; + } if (pInsertParam->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { return tscSQLSyntaxErrMsg(pInsertParam->msg, "? only allowed in binding insertion", *str); } @@ -860,54 +533,45 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i sToken.n -= 2 + cnt; } - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns. - TDRowLenT kvRowColLen = 0; - TDRowLenT colValAppended = 0; + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); + int32_t toffset = -1; + int16_t colId = -1; + tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); - if (!IS_DATA_COL_ORDERED(spd->orderStatus)) { - ASSERT(spd->colIdxInfo != NULL); - if(!isPrimaryKey) { - kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN); - } else { - ASSERT(spd->colIdxInfo[i].finalIdx == 0); - } - } - // the primary key locates in 1st column - int32_t ret = tsParseOneColumnKV(pSchema, &sToken, payload, kvPrimaryKeyStart, kvStart, pInsertParam->msg, str, - isPrimaryKey, timePrec, payloadValOffset + colValOffset, &colValAppended, - &dataRowDeltaColLen, &kvRowColLen); + int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, + colId, &dataLen, &kvLen, pBuilder->compareStat); if (ret != TSDB_CODE_SUCCESS) { return ret; } if (isPrimaryKey) { - if (tsCheckTimestamp(pDataBlocks, payloadValues(payload)) != TSDB_CODE_SUCCESS) { + TSKEY tsKey = memRowKey(row); + if (tsCheckTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - payloadColSetOffset(kvPrimaryKeyStart, colValOffset); - } else { - payloadColSetOffset(kvStart, colValOffset); - if (IS_DATA_COL_ORDERED(spd->orderStatus)) { - kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column + } + } + + if (!isParseBindParam) { + // 2. check and set convert flag + if (pBuilder->compareStat == ROW_COMPARE_NEED) { + checkAndConvertMemRow(row, dataLen, kvLen); + } + + // 3. set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) { + SDataRow dataRow = memRowDataBody(row); + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { + tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset); + } } } - - colValOffset += colValAppended; - kvRowLen += kvRowColLen; - dataRowLen += dataRowDeltaColLen; } - if (kvRowLen < dataRowLen) { - payloadSetType(payload, SMEM_ROW_KV); - } else { - payloadSetType(payload, SMEM_ROW_DATA); - } + *len = getExtendedRowSize(pDataBlocks); - *len = (int32_t)(payloadValOffset + colValOffset); - payloadSetTLen(payload, *len); - return TSDB_CODE_SUCCESS; } @@ -957,11 +621,13 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn int32_t precision = tinfo.precision; - int32_t extendedRowSize = getExtendedRowSize(&tinfo); - - initSMemRowHelper(&pDataBlock->rowHelper, tscGetTableSchema(pDataBlock->pTableMeta), - tscGetNumOfColumns(pDataBlock->pTableMeta), 0); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + if (TSDB_CODE_SUCCESS != + (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, + pDataBlock->boundColumnInfo.allNullLen))) { + return code; + } while (1) { index = 0; sToken = tStrGetToken(*str, &index, false); @@ -1012,19 +678,37 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; - pColInfo->orderStatus = ORDER_STATUS_ORDERED; + pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); pColInfo->colIdxInfo = NULL; + pColInfo->flen = 0; + pColInfo->allNullLen = 0; + int32_t nVar = 0; for (int32_t i = 0; i < pColInfo->numOfCols; ++i) { + uint8_t type = pSchema[i].type; if (i > 0) { pColInfo->cols[i].offset = pSchema[i - 1].bytes + pColInfo->cols[i - 1].offset; + pColInfo->cols[i].toffset = pColInfo->flen; + } + pColInfo->flen += TYPE_BYTES[type]; + switch (type) { + case TSDB_DATA_TYPE_BINARY: + pColInfo->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); + ++nVar; + break; + case TSDB_DATA_TYPE_NCHAR: + pColInfo->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + ++nVar; + break; + default: + break; } - - pColInfo->cols[i].hasVal = true; pColInfo->boundedColumns[i] = i; } + pColInfo->allNullLen += pColInfo->flen; + pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); } int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { @@ -1124,35 +808,29 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { assert(dataBuf->ordered); } - // allocate memory + // allocate memory size_t nAlloc = nRows * sizeof(SBlockKeyTuple); if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { size_t nRealAlloc = nAlloc + 10 * sizeof(SBlockKeyTuple); char * tmp = trealloc(pBlkKeyInfo->pKeyTuple, nRealAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nRealAlloc; } memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); + int32_t extendedRowSize = getExtendedRowSize(dataBuf); SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; char * pBlockData = pBlocks->data; - TDRowTLenT totolPayloadTLen = 0; - TDRowTLenT payloadTLen = 0; int n = 0; while (n < nRows) { - pBlkKeyTuple->skey = payloadTSKey(pBlockData); + pBlkKeyTuple->skey = memRowKey(pBlockData); pBlkKeyTuple->payloadAddr = pBlockData; - payloadTLen = payloadTLen(pBlockData); -#if 0 - ASSERT(payloadNCols(pBlockData) <= 4096); - ASSERT(payloadTLen(pBlockData) < 65536); -#endif - totolPayloadTLen += payloadTLen; + // next loop - pBlockData += payloadTLen; + pBlockData += extendedRowSize; ++pBlkKeyTuple; ++n; } @@ -1169,7 +847,6 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk TSKEY tj = (pBlkKeyTuple + j)->skey; if (ti == tj) { - totolPayloadTLen -= payloadTLen(pBlkKeyTuple + j); ++j; continue; } @@ -1185,17 +862,15 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk pBlocks->numOfRows = i + 1; } - dataBuf->size = sizeof(SSubmitBlk) + totolPayloadTLen; + dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; dataBuf->prevTS = INT64_MIN; return 0; } -static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { - STableComInfo tinfo = tscGetTableInfo(dataBuf->pTableMeta); - +static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { int32_t maxNumOfRows; - int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(&tinfo), &maxNumOfRows); + int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows); if (TSDB_CODE_SUCCESS != code) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1533,7 +1208,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->numOfBound = 0; memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { - pColInfo->cols[i].hasVal = false; + pColInfo->cols[i].valStat = VAL_STAT_NONE; } int32_t code = TSDB_CODE_SUCCESS; @@ -1572,12 +1247,12 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int32_t nScanned = 0, t = lastColIdx + 1; while (t < nCols) { if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - if (pColInfo->cols[t].hasVal == true) { + if (pColInfo->cols[t].valStat == VAL_STAT_HAS) { code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); goto _clean; } - pColInfo->cols[t].hasVal = true; + pColInfo->cols[t].valStat = VAL_STAT_HAS; pColInfo->boundedColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; findColumnIndex = true; @@ -1595,12 +1270,12 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int32_t nRemain = nCols - nScanned; while (t < nRemain) { if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - if (pColInfo->cols[t].hasVal == true) { + if (pColInfo->cols[t].valStat == VAL_STAT_HAS) { code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z); goto _clean; } - pColInfo->cols[t].hasVal = true; + pColInfo->cols[t].valStat = VAL_STAT_HAS; pColInfo->boundedColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; findColumnIndex = true; @@ -1835,7 +1510,7 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - if (dataBuf->boundColumnInfo.cols[0].hasVal == false) { + if (dataBuf->boundColumnInfo.cols[0].valStat == VAL_STAT_NONE) { code = tscInvalidOperationMsg(pInsertParam->msg, "primary timestamp column can not be null", NULL); goto _clean; } @@ -2046,15 +1721,18 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow goto _error; } - tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(&tinfo), &maxRows); + tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(pTableDataBlock), &maxRows); tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); if (tokenBuf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } - initSMemRowHelper(&pTableDataBlock->rowHelper, tscGetTableSchema(pTableDataBlock->pTableMeta), - tscGetNumOfColumns(pTableDataBlock->pTableMeta), 0); + if (TSDB_CODE_SUCCESS != + (ret = initMemRowBuilder(&pTableDataBlock->rowBuilder, 0, tinfo.numOfColumns, pTableDataBlock->numOfParams, + pTableDataBlock->boundColumnInfo.allNullLen))) { + goto _error; + } while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 2c2a299549..40664241c1 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -299,7 +299,7 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) { SSchema *schema = (SSchema*)pBlock->pTableMeta->schema; for (int32_t i = 0; i < spd->numOfCols; ++i) { - if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null + if (spd->cols[i].valStat == VAL_STAT_NONE) { // current column do not have any value to insert, set it to null for (int32_t n = 0; n < rowNum; ++n) { char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 173c709545..ad91209f18 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -39,6 +39,7 @@ #include "ttokendef.h" #include "qScript.h" #include "ttype.h" +#include "qFilter.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -140,6 +141,7 @@ static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlE static bool validateDebugFlag(int32_t v); static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo); +static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex); static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) { return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0; @@ -183,6 +185,9 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, if (var->nType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(var->nType)) { break; } + if (colType == TSDB_DATA_TYPE_BOOL && (var->i64 > 1 ||var->i64 < 0)) { + break; + } tbufWriteInt64(&bw, var->i64); } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { @@ -3217,7 +3222,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { pCmd->command = TSDB_SQL_SHOW; const char* msg1 = "invalid name"; - const char* msg2 = "pattern filter string too long"; + const char* msg2 = "wildcard string should be less than %d characters"; const char* msg3 = "database name too long"; const char* msg5 = "database name is empty"; const char* msg6 = "pattern string is empty"; @@ -3260,8 +3265,10 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (!tscValidateTableNameLength(pCmd->payloadLen)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + if (pPattern->n > tsMaxWildCardsLen){ + char tmp[64] = {0}; + sprintf(tmp, msg2, tsMaxWildCardsLen); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tmp); } } } else if (showType == TSDB_MGMT_TABLE_VNODES) { @@ -3732,8 +3739,8 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); } else if (colType == TSDB_DATA_TYPE_NCHAR) { - // pRight->value.nLen + 1 is larger than the actual nchar string length - pColumnFilter->pz = (int64_t)calloc(1, bufLen * TSDB_NCHAR_SIZE); + // bufLen + 1 is larger than the actual nchar string length + pColumnFilter->pz = (int64_t)calloc(1, (bufLen + 1) * TSDB_NCHAR_SIZE); retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); size_t len = twcslen((wchar_t*)pColumnFilter->pz); pColumnFilter->len = len * TSDB_NCHAR_SIZE; @@ -3839,51 +3846,40 @@ static int32_t tablenameCondToString(tSqlExpr* pExpr, SStringBuilder* sb) { } enum { - TSQL_EXPR_TS = 0, - TSQL_EXPR_TAG = 1, - TSQL_EXPR_COLUMN = 2, - TSQL_EXPR_TBNAME = 3, + TSQL_EXPR_TS = 1, + TSQL_EXPR_TAG = 2, + TSQL_EXPR_COLUMN = 4, + TSQL_EXPR_TBNAME = 8, + TSQL_EXPR_JOIN = 16, }; -static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) { +#define GET_MIXED_TYPE(t) (((t) >= TSQL_EXPR_JOIN) || ((t) > TSQL_EXPR_COLUMN && (t) < TSQL_EXPR_TBNAME) || ((t) == (TSQL_EXPR_TS|TSQL_EXPR_TAG))) + +static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - + int32_t ret = 0; const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; const char* msg3 = "bool column not support this operator"; const char* msg4 = "primary key not support this operator"; SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema); - SColumnFilterInfo* pColFilter = NULL; + pColumn->info.flist.numOfFilters++; + /* * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together * the already existed condition. */ - if (sqlOptr == TK_AND) { - // this is a new filter condition on this column - if (pColumn->info.flist.numOfFilters == 0) { - pColFilter = addColumnFilterInfo(&pColumn->info.flist); - } else { // update the existed column filter information, find the filter info here - pColFilter = &pColumn->info.flist.filterInfo[0]; - } - - if (pColFilter == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } else if (sqlOptr == TK_OR) { - // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" - pColFilter = addColumnFilterInfo(&pColumn->info.flist); - if (pColFilter == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } else { // error; + if (sqlOptr != TK_AND && sqlOptr != TK_OR) { return TSDB_CODE_TSC_INVALID_OPERATION; } + SColumnFilterInfo* pColFilter = calloc(1, sizeof(SColumnFilterInfo)); + pColFilter->filterstr = ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); @@ -3894,17 +3890,20 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC && pExpr->tokenId != TK_NOTNULL && pExpr->tokenId != TK_LIKE && pExpr->tokenId != TK_IN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + goto _err_ret; } } else { if (pExpr->tokenId == TK_LIKE) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + goto _err_ret; } if (pSchema->type == TSDB_DATA_TYPE_BOOL) { int32_t t = pExpr->tokenId; if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL && t != TK_IN) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + goto _err_ret; } } } @@ -3916,7 +3915,12 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } STableComInfo tinfo = tscGetTableInfo(pTableMeta); - return doExtractColumnFilterInfo(pCmd, pQueryInfo, tinfo.precision, pColFilter, pSchema->type, pExpr); + ret = doExtractColumnFilterInfo(pCmd, pQueryInfo, tinfo.precision, pColFilter, pSchema->type, pExpr); + +_err_ret: + freeColumnFilterInfo(pColFilter, 1); + + return ret; } static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pTableCond, SStringBuilder* sb) { @@ -3953,7 +3957,61 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* return ret; } -static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) { +static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) { + int32_t ret = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i); + if (p1 == NULL) { // no query condition on this table + continue; + } + + tExprNode* p = NULL; + + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); + taosArrayDestroy(colList); + + SBufferWriter bw = tbufInitWriter(NULL, false); + + TRY(0) { + exprTreeToBinary(&bw, p); + } CATCH(code) { + tbufCloseWriter(&bw); + UNUSED(code); + // TODO: more error handling + } END_TRY + + // add to required table column list + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + int64_t uid = pTableMetaInfo->pTableMeta->id.uid; + + STblCond cond = { + .uid = uid, + .idx = i, + .len = (int32_t)(tbufTell(&bw)), + .cond = tbufGetData(&bw, true) + }; + + if (pQueryInfo->colCond == NULL) { + pQueryInfo->colCond = taosArrayInit(2, sizeof(SCond)); + } + + taosArrayPush(pQueryInfo->colCond, &cond); + + tSqlExprDestroy(p1); + tExprTreeDestroy(p, NULL); + + if (ret) { + break; + } + } + + return ret; +} + + +static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) { if (pExpr == NULL) { pQueryInfo->onlyHasTagCond &= true; return TSDB_CODE_SUCCESS; @@ -3961,19 +4019,19 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSq pQueryInfo->onlyHasTagCond &= false; if (!tSqlExprIsParentOfLeaf(pExpr)) { // internal node - int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId); + int32_t ret = checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } - return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); + return checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); } else { // handle leaf node SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - return extractColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr); + return checkColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr); } } @@ -4240,6 +4298,10 @@ static bool isValidExpr(tSqlExpr* pLeft, tSqlExpr* pRight, int32_t optr) { return false; } + if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && (optr == TK_NOTNULL || optr == TK_ISNULL)) { + return false; + } + return true; } @@ -4359,7 +4421,18 @@ static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg return TSDB_CODE_SUCCESS; } -static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) { +static int32_t setNormalExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, int32_t parentOptr) { + if (*parent != NULL) { + *parent = tSqlExprCreate((*parent), pExpr, parentOptr); + } else { + *parent = pExpr; + } + + return TSDB_CODE_SUCCESS; +} + + +static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { const char* msg = "only support is [not] null"; tSqlExpr* pRight = pExpr->pRight; @@ -4367,13 +4440,34 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) { return invalidOperationMsg(msgBuf, msg); } + if (pRight->tokenId == TK_STRING) { + SSchema* pSchema = tscGetTableSchema(pTableMeta); + if (IS_VAR_DATA_TYPE(pSchema[index].type)) { + return TSDB_CODE_SUCCESS; + } + + char *v = strndup(pRight->exprToken.z, pRight->exprToken.n); + int32_t len = strRmquote(v, pRight->exprToken.n); + if (len > 0) { + uint32_t type = 0; + tGetToken(v, &type); + + if (type == TK_NULL) { + free(v); + return invalidOperationMsg(msgBuf, msg); + } + } + + free(v); + } + return TSDB_CODE_SUCCESS; } // check for like expression static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { const char* msg1 = "wildcard string should be less than %d characters"; - const char* msg2 = "illegal column name"; + const char* msg2 = "illegal column type for like"; tSqlExpr* pLeft = pExpr->pLeft; tSqlExpr* pRight = pExpr->pRight; @@ -4394,11 +4488,23 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t return TSDB_CODE_SUCCESS; } +int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) { + tSqlExpr* left = tSqlExprClone(expr); + tSqlExpr* right = expr; + + left->tokenId = TK_LT; + right->tokenId = TK_GT; + + *rexpr = tSqlExprCreate(left, right, TK_OR); + + return TSDB_CODE_SUCCESS; +} + + static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t parentOptr) { + int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) { const char* msg1 = "table query cannot use tags filter"; const char* msg2 = "illegal column name"; - const char* msg3 = "only one query time range allowed"; const char* msg4 = "too many join tables"; const char* msg5 = "not support ordinary column join"; const char* msg6 = "only one query condition on tbname allowed"; @@ -4414,13 +4520,15 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } + *tbIdx = index.tableIndex; + assert(tSqlExprIsParentOfLeaf(*pExpr)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; // validate the null expression - int32_t code = validateNullExpr(*pExpr, tscGetErrorMsgPayload(pCmd)); + int32_t code = validateNullExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4487,12 +4595,24 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql * since this expression is used to set the join query type */ tSqlExprDestroy(*pExpr); + if (type) { + *type |= TSQL_EXPR_JOIN; + } } else { - ret = setExprToCond(&pCondExpr->pTimewindow, *pExpr, msg3, parentOptr, pQueryInfo->msg); + tSqlExpr *rexpr = NULL; + if ((*pExpr)->tokenId == TK_NE) { + handleNeOptr(&rexpr, *pExpr); + } else { + rexpr = *pExpr; + } + + ret = setNormalExprToCond(tsExpr, rexpr, parentOptr); + if (type) { + *type |= TSQL_EXPR_TS; + } } *pExpr = NULL; // remove this expression - *type = TSQL_EXPR_TS; } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // query on tags, check for tag query condition if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { @@ -4517,7 +4637,9 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - *type = TSQL_EXPR_TBNAME; + if (type) { + *type |= TSQL_EXPR_TAG; + } *pExpr = NULL; } else { if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query @@ -4528,23 +4650,41 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; ret = setExprToCond(&pCondExpr->pJoinExpr, *pExpr, NULL, parentOptr, pQueryInfo->msg); *pExpr = NULL; + if (type) { + *type |= TSQL_EXPR_JOIN; + } } else { // do nothing // ret = setExprToCond(pCmd, &pCondExpr->pTagCond, // *pExpr, NULL, parentOptr); + tSqlExpr *rexpr = NULL; + if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) { + handleNeOptr(&rexpr, *pExpr); + *pExpr = rexpr; + } + + if (type) { + *type |= TSQL_EXPR_TAG; + } } - - *type = TSQL_EXPR_TAG; } - } else { // query on other columns - *type = TSQL_EXPR_COLUMN; - + if (type) { + *type |= TSQL_EXPR_COLUMN; + } + if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } - ret = setExprToCond(&pCondExpr->pColumnCond, *pExpr, NULL, parentOptr, pQueryInfo->msg); + tSqlExpr *rexpr = NULL; + if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) { + handleNeOptr(&rexpr, *pExpr); + } else { + rexpr = *pExpr; + } + + ret = setNormalExprToCond(columnExpr, rexpr, parentOptr); *pExpr = NULL; // remove it from expr tree } @@ -4552,12 +4692,20 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, - int32_t* type, int32_t parentOptr) { + int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - const char* msg1 = "query condition between different columns must use 'AND'"; + tSqlExpr *columnLeft = NULL; + tSqlExpr *columnRight = NULL; + tSqlExpr *tsLeft = NULL; + tSqlExpr *tsRight = NULL; + + int32_t ret = 0; + + const char* msg1 = "query condition between columns/tags/timestamp/join fields must use 'AND'"; + const char* msg2 = "query condition between tables must use 'AND'"; if ((*pExpr)->flags & (1 << EXPR_FLAG_TS_ERROR)) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -4570,48 +4718,94 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr return TSDB_CODE_TSC_INVALID_OPERATION; } - int32_t leftType = -1; - int32_t rightType = -1; + int32_t leftType = 0; + int32_t rightType = 0; + int32_t leftTbIdx = 0; + int32_t rightTbIdx = 0; if (!tSqlExprIsParentOfLeaf(*pExpr)) { - int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId); + ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, type ? &leftType : NULL, &leftTbIdx, (*pExpr)->tokenId, &columnLeft, &tsLeft); if (ret != TSDB_CODE_SUCCESS) { - return ret; + goto err_ret; } - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId); + ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, type ? &rightType : NULL, &rightTbIdx, (*pExpr)->tokenId, &columnRight, &tsRight); if (ret != TSDB_CODE_SUCCESS) { - return ret; + goto err_ret; } /* * if left child and right child do not belong to the same group, the sub * expression is not valid for parent node, it must be TK_AND operator. */ - if (leftType != rightType) { - if ((*pExpr)->tokenId == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + if (type != NULL && ((leftType != rightType) || GET_MIXED_TYPE(leftType)) && ((*pExpr)->tokenId == TK_OR)) { + ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); + goto err_ret; } - *type = rightType; + if (((leftTbIdx != rightTbIdx) || (leftTbIdx == -1 || rightTbIdx == -1)) && ((*pExpr)->tokenId == TK_OR)) { + ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + goto err_ret; + } + + if (columnLeft && columnRight) { + setNormalExprToCond(&columnLeft, columnRight, (*pExpr)->tokenId); + + *columnExpr = columnLeft; + } else { + *columnExpr = columnLeft ? columnLeft : columnRight; + } + + if (tsLeft && tsRight) { + setNormalExprToCond(&tsLeft, tsRight, (*pExpr)->tokenId); + + *tsExpr = tsLeft; + } else { + *tsExpr = tsLeft ? tsLeft : tsRight; + } + + if (type) { + *type = leftType|rightType; + } + *tbIdx = (leftTbIdx == rightTbIdx) ? leftTbIdx : -1; + return TSDB_CODE_SUCCESS; } exchangeExpr(*pExpr); if (pLeft->tokenId == TK_ID && pRight->tokenId == TK_TIMESTAMP && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) { - return TSDB_CODE_TSC_INVALID_OPERATION; + ret = TSDB_CODE_TSC_INVALID_OPERATION; + goto err_ret; } if ((pLeft->flags & (1 << EXPR_FLAG_TS_ERROR)) || (pRight->flags & (1 << EXPR_FLAG_TS_ERROR))) { - return TSDB_CODE_TSC_INVALID_OPERATION; + ret = TSDB_CODE_TSC_INVALID_OPERATION; + goto err_ret; } - return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr); + ret = handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, tbIdx, parentOptr, columnExpr, tsExpr); + if (ret) { + goto err_ret; + } + + return TSDB_CODE_SUCCESS; + +err_ret: + + tSqlExprDestroy(columnLeft); + tSqlExprDestroy(columnRight); + tSqlExprDestroy(tsLeft); + tSqlExprDestroy(tsRight); + return ret; } static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) { + if (*pExpr == NULL) { + *pOut = NULL; + return; + } + if (tSqlExprIsParentOfLeaf(*pExpr)) { tSqlExpr* pLeft = (*pExpr)->pLeft; @@ -4733,38 +4927,73 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, return TSDB_CODE_SUCCESS; } -static bool validateFilterExpr(SQueryInfo* pQueryInfo) { - SArray* pColList = pQueryInfo->colList; +int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) { + const char* msg0 = "only one time stamp window allowed"; + +#define SET_EMPTY_RANGE(w) do { (w)->skey = INT64_MAX; (w)->ekey = INT64_MIN; } while (0) +#define IS_EMPTY_RANGE(w) ((w)->skey == INT64_MAX && (w)->ekey == INT64_MIN) - size_t num = taosArrayGetSize(pColList); - - for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(pColList, i); - - for (int32_t j = 0; j < pCol->info.flist.numOfFilters; ++j) { - SColumnFilterInfo* pColFilter = &pCol->info.flist.filterInfo[j]; - int32_t lowerOptr = pColFilter->lowerRelOptr; - int32_t upperOptr = pColFilter->upperRelOptr; - - if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) && - (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { - continue; - } - - // there must be at least two range, not support yet. - if (lowerOptr * upperOptr != TSDB_RELATION_INVALID) { - return false; - } + if (optr == TSDB_RELATION_AND) { + if (res->skey > win->ekey || win->skey > res->ekey) { + SET_EMPTY_RANGE(res); + return TSDB_CODE_SUCCESS; } + + if (res->skey < win->skey) { + res->skey = win->skey; + } + + if (res->ekey > win->ekey) { + res->ekey = win->ekey; + } + + return TSDB_CODE_SUCCESS; } - return true; + if (res->skey > win->ekey || win->skey > res->ekey) { + if (IS_EMPTY_RANGE(res)) { + res->skey = win->skey; + res->ekey = win->ekey; + return TSDB_CODE_SUCCESS; + } + + if (IS_EMPTY_RANGE(win)) { + return TSDB_CODE_SUCCESS; + } + + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); + } + + if (res->skey > win->skey) { + res->skey = win->skey; + } + + if (res->ekey < win->ekey) { + res->ekey = win->ekey; + } + + return TSDB_CODE_SUCCESS; } -static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { - const char* msg0 = "invalid timestamp"; - const char* msg1 = "only one time stamp window allowed"; +static int32_t createTimeRangeExpr(tSqlExpr** pExpr, STimeWindow* win, uint32_t tokenId) { + *pExpr = calloc(1, sizeof(tSqlExpr)); + + (*pExpr)->type = SQL_NODE_VALUE; + (*pExpr)->tokenId = tokenId; + (*pExpr)->value.nType = TSDB_DATA_TYPE_VALUE_ARRAY; + (*pExpr)->value.nLen = 2; + (*pExpr)->value.arr = taosArrayInit(2, sizeof(int64_t)); + + taosArrayPush((*pExpr)->value.arr, &win->skey); + taosArrayPush((*pExpr)->value.arr, &win->ekey); + + return TSDB_CODE_SUCCESS; +} + +static int32_t convertTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { + const char* msg0 = "invalid timestamp or operator for timestamp"; int32_t code = 0; + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; if (pExpr == NULL) { pQueryInfo->onlyHasTagCond &= true; @@ -4774,16 +5003,15 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE if (!tSqlExprIsParentOfLeaf(pExpr)) { - if (pExpr->tokenId == TK_OR) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); + if (code) { + return code; + } - code = getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); - if (code) { - return code; - } - - return getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight); + code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight); + if (code) { + return code; + } } else { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { @@ -4795,19 +5023,13 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE tSqlExpr* pRight = pExpr->pRight; - STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0); } - // update the timestamp query range - if (pQueryInfo->window.skey < win.skey) { - pQueryInfo->window.skey = win.skey; - } + createTimeRangeExpr(&pExpr->pRight, &win, pRight->tokenId); - if (pQueryInfo->window.ekey > win.ekey) { - pQueryInfo->window.ekey = win.ekey; - } + tSqlExprDestroy(pRight); } return TSDB_CODE_SUCCESS; @@ -4973,7 +5195,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { return TSDB_CODE_SUCCESS; } -static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSqlExpr** pExpr) { +static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { int32_t ret = TSDB_CODE_SUCCESS; if (pCondExpr->pTagCond == NULL) { @@ -4981,7 +5203,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE } for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i); + tSqlExpr* p1 = extractExprForSTable(pCmd, &pCondExpr->pTagCond, pQueryInfo, i); if (p1 == NULL) { // no query condition on this table continue; } @@ -4990,6 +5212,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE SArray* colList = taosArrayInit(10, sizeof(SColIndex)); ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); + //if (ret == TSDB_CODE_SUCCESS) { + // ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList)); + //} + SBufferWriter bw = tbufInitWriter(NULL, false); TRY(0) { @@ -5016,14 +5242,14 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE } tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); - tSqlExprCompact(pExpr); + tSqlExprCompact(&pCondExpr->pTagCond); if (ret == TSDB_CODE_SUCCESS) { ret = validateTagCondExpr(pCmd, p); } tSqlExprDestroy(p1); - tExprTreeDestroy(p, NULL); + tExprTreeDestroy(p, NULL); //TODO taosArrayDestroy(colList); if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -5035,7 +5261,6 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE } } - pCondExpr->pTagCond = NULL; return ret; } @@ -5134,6 +5359,46 @@ int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } +static int32_t getQueryTimeRange(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) { + int32_t ret = TSDB_CODE_SUCCESS; + + if (*pExpr == NULL) { + return ret; + } + + //multiple tables's query time range mixed together + + tExprNode* p = NULL; + SFilterInfo *filter = NULL; + + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + ret = exprTreeFromSqlExpr(pCmd, &p, *pExpr, pQueryInfo, colList, NULL); + taosArrayDestroy(colList); + + if (ret != TSDB_CODE_SUCCESS) { + goto _ret; + } + + ret = filterInitFromTree(p, &filter, FI_OPTION_NO_REWRITE|FI_OPTION_TIMESTAMP); + if (ret != TSDB_CODE_SUCCESS) { + goto _ret; + } + + ret = filterGetTimeRange(filter, &pQueryInfo->window); + + filterFreeInfo(filter); + +_ret: + tExprTreeDestroy(p, NULL); + + if (ret) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tstrerror(ret)); + } + + return ret; +} + + int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) { if (pExpr == NULL) { @@ -5141,7 +5406,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq } const char* msg1 = "invalid expression"; - const char* msg2 = "invalid filter expression"; +// const char* msg2 = "invalid filter expression"; int32_t ret = TSDB_CODE_SUCCESS; @@ -5154,14 +5419,30 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq } int32_t type = 0; - if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId)) != TSDB_CODE_SUCCESS) { + int32_t tbIdx = 0; + int32_t *etype = &type; + +#if 0 + //DISABLE PARENT CONDITION GROUP TYPE CHECK + if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) { + etype = NULL; + } +#endif + + if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, etype, &tbIdx, (*pExpr)->tokenId, &condExpr.pColumnCond, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { goto PARSE_WHERE_EXIT; } + if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 && condExpr.pTimewindow != NULL) { + setNormalExprToCond(&condExpr.pColumnCond, condExpr.pTimewindow, TK_AND); + condExpr.pTimewindow = NULL; + } + tSqlExprCompact(pExpr); // after expression compact, the expression tree is only include tag query condition condExpr.pTagCond = (*pExpr); + *pExpr = NULL; // 1. check if it is a join query if ((ret = validateJoinExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { @@ -5169,14 +5450,18 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq } // 2. get the query time range - if ((ret = getTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { - return ret; + if ((ret = convertTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { + goto PARSE_WHERE_EXIT; + } + + if ((ret = getQueryTimeRange(&pSql->cmd, pQueryInfo, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { + goto PARSE_WHERE_EXIT; } // 3. get the tag query condition - if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr, pExpr)) != TSDB_CODE_SUCCESS) { - return ret; + if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) { + goto PARSE_WHERE_EXIT; } // 4. get the table name query condition @@ -5185,16 +5470,15 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq } // 5. other column query condition - if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { + if ((ret = checkColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { goto PARSE_WHERE_EXIT; } - if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 ) { - if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pTimewindow, TK_AND)) != TSDB_CODE_SUCCESS) { - goto PARSE_WHERE_EXIT; - } + if ((ret = getColQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr.pColumnCond)) != TSDB_CODE_SUCCESS) { + goto PARSE_WHERE_EXIT; } + // 6. join condition if ((ret = getJoinCondInfo(&pSql->cmd, pQueryInfo, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { goto PARSE_WHERE_EXIT; @@ -5209,10 +5493,10 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq goto PARSE_WHERE_EXIT; } - if (!validateFilterExpr(pQueryInfo)) { - ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); - goto PARSE_WHERE_EXIT; - } + //if (!validateFilterExpr(pQueryInfo)) { + // ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + // goto PARSE_WHERE_EXIT; + //} //doAddJoinTagsColumnsIntoTagList(&pSql->cmd, pQueryInfo, &condExpr); if (condExpr.tsJoin) { @@ -5243,7 +5527,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t * filter primary ts filter expression like: * where ts in ('2015-12-12 4:8:12') */ - if (pRight->tokenId == TK_SET || optr == TK_IN) { + if (pRight->tokenId == TK_SET || optr == TK_IN || optr == TK_NE) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -8115,19 +8399,13 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name[TSDB_TABLE_FNAME_LEN] = {0}; - //if (!pSql->pBuf) { - // if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { - // code = TSDB_CODE_TSC_OUT_OF_MEMORY; - // goto _end; - // } - //} - plist = taosArrayInit(4, POINTER_BYTES); pVgroupList = taosArrayInit(4, POINTER_BYTES); taosArraySort(tableNameList, tnameComparFn); taosArrayRemoveDuplicate(tableNameList, tnameComparFn, NULL); + STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); size_t numOfTables = taosArrayGetSize(tableNameList); for (int32_t i = 0; i < numOfTables; ++i) { SName* pname = taosArrayGet(tableNameList, i); @@ -8143,7 +8421,8 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { // avoid mem leak, may should update pTableMeta void* pVgroupIdList = NULL; if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity); + code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity, (STableMeta **)(&pSTMeta)); + pSql->pBuf = (void *)pSTMeta; // create the child table meta from super table failed, try load it from mnode if (code != TSDB_CODE_SUCCESS) { @@ -8671,12 +8950,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0); if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) { - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryInfo->colList); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCols = taosArrayGetP(pQueryInfo->colList, i); - if (pCols->info.flist.numOfFilters > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - } + if ((pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) || IS_TSWINDOW_SPECIFIED(pQueryInfo->window)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } } } @@ -8778,6 +9053,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pSqlExpr->pLeft != NULL) { int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid); @@ -8845,13 +9121,13 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS } } } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression - SColumnIndex index = COLUMN_INDEX_INITIALIZER; int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)); if (ret != TSDB_CODE_SUCCESS) { return ret; } - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + pQueryInfo->curTableIdx = index.tableIndex; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); *pExpr = calloc(1, sizeof(tExprNode)); @@ -8874,12 +9150,16 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { int32_t colType = -1; - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - if (pCols != NULL && taosArrayGetSize(pCols) > 0) { - SColIndex* idx = taosArrayGet(pCols, taosArrayGetSize(pCols) - 1); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); - if (pSchema != NULL) { - colType = pSchema->type; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx)->pTableMeta; + if (pCols != NULL) { + size_t colSize = taosArrayGetSize(pCols); + + if (colSize > 0) { + SColIndex* idx = taosArrayGet(pCols, colSize - 1); + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); + if (pSchema != NULL) { + colType = pSchema->type; + } } } tVariant *pVal; @@ -8888,7 +9168,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { colType = TSDB_DATA_TYPE_DOUBLE; } - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); if (serializeExprListToVariant(pSqlExpr->Expr.paramList, &pVal, colType, tinfo.precision) == false) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); @@ -8927,7 +9207,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS // NOTE: binary|nchar data allows the >|< type filter if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) { + if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) { return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -8982,3 +9262,4 @@ void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) { #endif + diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 36cc212868..f0ee180bbe 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -684,7 +684,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo); + int32_t srcColFilterSize = 0; int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); size_t numOfExprs = tscNumOfExprs(pQueryInfo); @@ -695,6 +695,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { int32_t tableSerialize = 0; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pVgroupTables != NULL) { size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); @@ -707,8 +708,15 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { tableSerialize = totalTables * sizeof(STableIdInfo); } - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + - exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + srcColFilterSize = pCond->len; + } + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + exprSize + tsBufSize + + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, @@ -966,10 +974,21 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tableCols[i].colId = htons(pCol->colId); pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); pQueryMsg->tableCols[i].type = htons(pCol->type); - pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); + //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); + pQueryMsg->tableCols[i].flist.numOfFilters = 0; // append the filter information after the basic column information - serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); + //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->colCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } } for (int32_t i = 0; i < query.numOfOutput; ++i) { @@ -1044,7 +1063,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htonl(pCond->len); + pQueryMsg->tagCondLen = htons(pCond->len); memcpy(pMsg, pCond->cond, pCond->len); pMsg += pCond->len; @@ -2853,18 +2872,19 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool tNameExtractFullName(&pTableMetaInfo->name, name); size_t len = strlen(name); - if (pTableMetaInfo->tableMetaCapacity != 0) { - if (pTableMetaInfo->pTableMeta != NULL) { - memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); - } + // just make runtime happy + if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { + memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); } taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity); - - STableMeta* pMeta = pTableMetaInfo->pTableMeta; + + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); if (pMeta && pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity); + int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); + pSql->pBuf = (void *)(pSTMeta); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8c0d642ca6..3554b43ff6 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -796,6 +796,7 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* STimeWindow window = pQueryInfo->window; tscInitQueryInfo(pQueryInfo); + pQueryInfo->colCond = pSupporter->colCond; pQueryInfo->window = window; TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); @@ -1883,6 +1884,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag SColumnIndex colIndex = {0}; + pSupporter->colCond = pNewQueryInfo->colCond; + pNewQueryInfo->colCond = NULL; + STagCond* pTagCond = &pSupporter->tagCond; assert(pTagCond->joinInfo.hasJoin); @@ -2319,6 +2323,11 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { goto _error; } + if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, 0) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + pNewQueryInfo->window = pQueryInfo->window; pNewQueryInfo->interval = pQueryInfo->interval; pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5a724af7bf..19a816faeb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -62,11 +62,11 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le break; case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%f", GET_FLOAT_VAL(buf)); + n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); break; case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%f", GET_DOUBLE_VAL(buf)); + n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); break; case TSDB_DATA_TYPE_BINARY: @@ -82,6 +82,22 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le n = bufSize + 2; break; + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%d", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%d", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + default: tscError("unsupported type:%d", type); return TSDB_CODE_TSC_INVALID_VALUE; @@ -118,6 +134,24 @@ SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { return NULL; } +STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx) { + if (filters == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(filters); + for (int32_t i = 0; i < size; ++i) { + STblCond* cond = taosArrayGet(filters, i); + + if (uid == cond->uid && (idx >= 0 && cond->idx == idx)) { + return cond; + } + } + + return NULL; +} + + void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) { if (tbufTell(bw) == 0) { return; @@ -753,8 +787,7 @@ typedef struct SDummyInputInfo { SSDataBlock *block; STableQueryInfo *pTableQueryInfo; SSqlObj *pSql; // refactor: remove it - int32_t numOfFilterCols; - SSingleColumnFilterInfo *pFilterInfo; + SFilterInfo *pFilterInfo; } SDummyInputInfo; typedef struct SJoinStatus { @@ -770,38 +803,7 @@ typedef struct SJoinOperatorInfo { SRspResultInfo resultInfo; // todo refactor, add this info for each operator } SJoinOperatorInfo; -static void converNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t rows, bool *gotNchar) { - for (int32_t i = 0; i < numOfFilterCols; ++i) { - if (pFilterInfo[i].info.type == TSDB_DATA_TYPE_NCHAR) { - pFilterInfo[i].pData2 = pFilterInfo[i].pData; - pFilterInfo[i].pData = malloc(rows * pFilterInfo[i].info.bytes); - int32_t bufSize = pFilterInfo[i].info.bytes - VARSTR_HEADER_SIZE; - for (int32_t j = 0; j < rows; ++j) { - char* dst = (char *)pFilterInfo[i].pData + j * pFilterInfo[i].info.bytes; - char* src = (char *)pFilterInfo[i].pData2 + j * pFilterInfo[i].info.bytes; - int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); - varDataLen(dst) = len; - } - *gotNchar = true; - } - } -} - -static void freeNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { - for (int32_t i = 0; i < numOfFilterCols; ++i) { - if (pFilterInfo[i].info.type == TSDB_DATA_TYPE_NCHAR) { - if (pFilterInfo[i].pData2) { - tfree(pFilterInfo[i].pData); - pFilterInfo[i].pData = pFilterInfo[i].pData2; - pFilterInfo[i].pData2 = NULL; - } - } - } -} - - -static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { +static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* pFilterInfo) { int32_t offset = 0; char* pData = pRes->data; @@ -817,14 +819,16 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SSingleColumnF } // filter data if needed - if (numOfFilterCols > 0) { - doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); + if (pFilterInfo) { + //doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); + doSetFilterColInfo(pFilterInfo, pBlock); bool gotNchar = false; - converNcharFilterColumn(pFilterInfo, numOfFilterCols, pBlock->info.rows, &gotNchar); + filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar); int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t)); - bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p); + //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p); + bool all = filterExecute(pFilterInfo, pBlock->info.rows, p); if (gotNchar) { - freeNcharFilterColumn(pFilterInfo, numOfFilterCols); + filterFreeNcharColumns(pFilterInfo); } if (!all) { doCompactSDataBlock(pBlock, pBlock->info.rows, p); @@ -862,7 +866,7 @@ SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { pBlock->info.rows = pRes->numOfRows; if (pRes->numOfRows != 0) { - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols); + doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo); *newgroup = false; return pBlock; } @@ -877,7 +881,7 @@ SSDataBlock* doGetDataBlock(void* param, bool* newgroup) { } pBlock->info.rows = pRes->numOfRows; - doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols); + doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo); *newgroup = false; return pBlock; } @@ -920,25 +924,40 @@ SSDataBlock* doDataBlockJoin(void* param, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return pJoinInfo->pRes; } - + SJoinStatus* st0 = &pJoinInfo->status[0]; SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0); int64_t* ts0 = (int64_t*) p0->pData; + if (st0->index >= st0->pBlock->info.rows) { + continue; + } + bool prefixEqual = true; while(1) { prefixEqual = true; for (int32_t i = 1; i < pJoinInfo->numOfUpstream; ++i) { SJoinStatus* st = &pJoinInfo->status[i]; + ts0 = (int64_t*) p0->pData; SColumnInfoData* p = taosArrayGet(st->pBlock->pDataBlock, 0); int64_t* ts = (int64_t*)p->pData; + if (st->index >= st->pBlock->info.rows || st0->index >= st0->pBlock->info.rows) { + fetchNextBlockIfCompleted(pOperator, newgroup); + if (pOperator->status == OP_EXEC_DONE) { + return pJoinInfo->pRes; + } + + prefixEqual = false; + break; + } + if (ts[st->index] < ts0[st0->index]) { // less than the first prefixEqual = false; - if ((++(st->index)) >= st->pBlock->info.rows) { + if ((++(st->index)) >= st->pBlock->info.rows) { fetchNextBlockIfCompleted(pOperator, newgroup); if (pOperator->status == OP_EXEC_DONE) { return pJoinInfo->pRes; @@ -1053,22 +1072,21 @@ static void destroyDummyInputOperator(void* param, int32_t numOfOutput) { pInfo->block = destroyOutputBuf(pInfo->block); pInfo->pSql = NULL; - doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); + filterFreeInfo(pInfo->pFilterInfo); cleanupResultRowInfo(&pInfo->pTableQueryInfo->resInfo); tfree(pInfo->pTableQueryInfo); } // todo this operator servers as the adapter for Operator tree and SqlRes result, remove it later -SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { +SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SFilterInfo* pFilters) { assert(numOfCols > 0); STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; SDummyInputInfo* pInfo = calloc(1, sizeof(SDummyInputInfo)); pInfo->pSql = pSql; - pInfo->pFilterInfo = pFilterInfo; - pInfo->numOfFilterCols = numOfFilterCols; + pInfo->pFilterInfo = pFilters; pInfo->pTableQueryInfo = createTmpTableQueryInfo(win); pInfo->block = calloc(numOfCols, sizeof(SSDataBlock)); @@ -1156,6 +1174,7 @@ void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, b pRes->completed = (pRes->numOfRows == 0); } +/* static void createInputDataFilterInfo(SQueryInfo* px, int32_t numOfCol1, int32_t* numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo) { SColumnInfo* tableCols = calloc(numOfCol1, sizeof(SColumnInfo)); for(int32_t i = 0; i < numOfCol1; ++i) { @@ -1173,6 +1192,7 @@ static void createInputDataFilterInfo(SQueryInfo* px, int32_t numOfCol1, int32_t tfree(tableCols); } +*/ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pSql) { SSqlRes* pOutput = &pSql->res; @@ -1201,11 +1221,17 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue // if it is a join query, create join operator here int32_t numOfCol1 = pTableMeta->tableInfo.numOfColumns; - int32_t numOfFilterCols = 0; - SSingleColumnFilterInfo* pFilterInfo = NULL; - createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols, &pFilterInfo); + SFilterInfo *pFilters = NULL; + STblCond *pCond = NULL; + if (px->colCond) { + pCond = tsGetTableFilter(px->colCond, pTableMeta->id.uid, 0); + if (pCond && pCond->cond) { + createQueryFilter(pCond->cond, pCond->len, &pFilters); + } + //createInputDataFlterInfo(px, numOfCol1, &numOfFilterCols, &pFilterInfo); + } - SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols); + SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilters); pOutput->precision = pSqlObjList[0]->res.precision; @@ -1222,15 +1248,21 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue for(int32_t i = 1; i < px->numOfTables; ++i) { STableMeta* pTableMeta1 = tscGetMetaInfo(px, i)->pTableMeta; + numOfCol1 = pTableMeta1->tableInfo.numOfColumns; + SFilterInfo *pFilters1 = NULL; SSchema* pSchema1 = tscGetTableSchema(pTableMeta1); int32_t n = pTableMeta1->tableInfo.numOfColumns; - int32_t numOfFilterCols1 = 0; - SSingleColumnFilterInfo* pFilterInfo1 = NULL; - createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols1, &pFilterInfo1); + if (px->colCond) { + pCond = tsGetTableFilter(px->colCond, pTableMeta1->id.uid, i); + if (pCond && pCond->cond) { + createQueryFilter(pCond->cond, pCond->len, &pFilters1); + } + //createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols1, &pFilterInfo1); + } - p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilterInfo1, numOfFilterCols1); + p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilters1); memcpy(&schema[offset], pSchema1, n * sizeof(SSchema)); offset += n; } @@ -1776,101 +1808,6 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i return TSDB_CODE_SUCCESS; } -static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) { - SSchema* pSchema = pBuilder->pSchema; - char* p = (char*)pBuilder->buf; - int toffset = 0; - uint16_t nCols = pBuilder->nCols; - - uint8_t memRowType = payloadType(p); - uint16_t nColsBound = payloadNCols(p); - if (pBuilder->nCols <= 0 || nColsBound <= 0) { - return NULL; - } - char* pVals = POINTER_SHIFT(p, payloadValuesOffset(p)); - SMemRow* memRow = (SMemRow)pBuilder->pDataBlock; - memRowSetType(memRow, memRowType); - - // ----------------- Raw payload structure for row: - /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| - * | |<----------------- flen ------------->|<--- value part --->| - * |SMemRowType| dataTLen | nCols | colId | colType | offset | ... | value |...|...|... | - * +-----------+----------+----------+--------------------------------------|--------------------| - * | uint8_t | uint32_t | uint16_t | int16_t | uint8_t | uint16_t | ... |.......|...|...|... | - * +-----------+----------+----------+--------------------------------------+--------------------| - * 1. offset in column data tuple starts from the value part in case of uint16_t overflow. - * 2. dataTLen: total length including the header and body. - */ - - if (memRowType == SMEM_ROW_DATA) { - SDataRow trow = (SDataRow)memRowDataBody(memRow); - dataRowSetLen(trow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen)); - dataRowSetVersion(trow, pBuilder->sversion); - - p = (char*)payloadBody(pBuilder->buf); - uint16_t i = 0, j = 0; - while (j < nCols) { - if (i >= nColsBound) { - break; - } - int16_t colId = payloadColId(p); - if (colId == pSchema[j].colId) { - // ASSERT(payloadColType(p) == pSchema[j].type); - tdAppendColVal(trow, POINTER_SHIFT(pVals, payloadColOffset(p)), pSchema[j].type, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - p = payloadNextCol(p); - ++i; - ++j; - } else if (colId < pSchema[j].colId) { - p = payloadNextCol(p); - ++i; - } else { - tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - ++j; - } - } - - while (j < nCols) { - tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset); - toffset += TYPE_BYTES[pSchema[j].type]; - ++j; - } - - #if 0 // no need anymore - while (i < nColsBound) { - p = payloadNextCol(p); - ++i; - } - #endif - - } else if (memRowType == SMEM_ROW_KV) { - SKVRow kvRow = (SKVRow)memRowKvBody(memRow); - kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound)); - kvRowSetNCols(kvRow, nColsBound); - memRowSetKvVersion(memRow, pBuilder->sversion); - - p = (char*)payloadBody(pBuilder->buf); - int i = 0; - while (i < nColsBound) { - int16_t colId = payloadColId(p); - uint8_t colType = payloadColType(p); - tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, &toffset); - //toffset += sizeof(SColIdx); - p = payloadNextCol(p); - ++i; - } - - } else { - ASSERT(0); - } - int32_t rowTLen = memRowTLen(memRow); - pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + rowTLen; // next row - pBuilder->pSubmitBlk->dataLen += rowTLen; - - return memRow; -} - // Erase the empty space reserved for binary data static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam, SBlockKeyTuple* blkKeyTuple) { @@ -1902,10 +1839,11 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI int32_t schemaSize = sizeof(STColumn) * numOfCols; pBlock->schemaLen = schemaSize; } else { - for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { - flen += TYPE_BYTES[pSchema[j].type]; + if (IS_RAW_PAYLOAD(insertParam->payloadType)) { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { + flen += TYPE_BYTES[pSchema[j].type]; + } } - pBlock->schemaLen = 0; } @@ -1932,18 +1870,19 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI pBlock->dataLen += memRowTLen(memRow); } } else { - SMemRowBuilder rowBuilder; - rowBuilder.pSchema = pSchema; - rowBuilder.sversion = pTableMeta->sversion; - rowBuilder.flen = flen; - rowBuilder.nCols = tinfo.numOfColumns; - rowBuilder.pDataBlock = pDataBlock; - rowBuilder.pSubmitBlk = pBlock; - rowBuilder.buf = p; - for (int32_t i = 0; i < numOfRows; ++i) { - rowBuilder.buf = (blkKeyTuple + i)->payloadAddr; - tdGenMemRowFromBuilder(&rowBuilder); + char* payload = (blkKeyTuple + i)->payloadAddr; + if (isNeedConvertRow(payload)) { + convertSMemRow(pDataBlock, payload, pTableDataBlock); + TDRowTLenT rowTLen = memRowTLen(pDataBlock); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; + } else { + TDRowTLenT rowTLen = memRowTLen(payload); + memcpy(pDataBlock, payload, rowTLen); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; + } } } @@ -1956,9 +1895,9 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI static int32_t getRowExpandSize(STableMeta* pTableMeta) { int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; - int32_t columns = tscGetNumOfColumns(pTableMeta); + int32_t columns = tscGetNumOfColumns(pTableMeta); SSchema* pSchema = tscGetTableSchema(pTableMeta); - for(int32_t i = 0; i < columns; i++) { + for (int32_t i = 0; i < columns; i++) { if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; } @@ -2004,7 +1943,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; STableDataBlocks* dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, @@ -2017,7 +1956,8 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl return ret; } - int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); + int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); @@ -2061,7 +2001,9 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl pBlocks->numOfRows, pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey); } - int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); + int32_t len = pBlocks->numOfRows * + (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); pBlocks->tid = htonl(pBlocks->tid); pBlocks->uid = htobe64(pBlocks->uid); @@ -2268,6 +2210,11 @@ int32_t tscGetResRowLength(SArray* pExprList) { } static void destroyFilterInfo(SColumnFilterList* pFilterList) { + if (pFilterList->filterInfo == NULL) { + pFilterList->numOfFilters = 0; + return; + } + for(int32_t i = 0; i < pFilterList->numOfFilters; ++i) { if (pFilterList->filterInfo[i].filterstr) { tfree(pFilterList->filterInfo[i].pz); @@ -2970,6 +2917,64 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { return 0; } +int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx) { + if (src == NULL) { + return 0; + } + + size_t s = taosArrayGetSize(src); + *dest = taosArrayInit(s, sizeof(SCond)); + + for (int32_t i = 0; i < s; ++i) { + STblCond* pCond = taosArrayGet(src, i); + STblCond c = {0}; + + if (tidx > 0) { + if (!(pCond->uid == uid && pCond->idx == tidx)) { + continue; + } + + c.idx = 0; + } else { + c.idx = pCond->idx; + } + + c.len = pCond->len; + c.uid = pCond->uid; + + if (pCond->len > 0) { + assert(pCond->cond != NULL); + c.cond = malloc(c.len); + if (c.cond == NULL) { + return -1; + } + + memcpy(c.cond, pCond->cond, c.len); + } + + taosArrayPush(*dest, &c); + } + + return 0; +} + +void tscColCondRelease(SArray** pCond) { + if (*pCond == NULL) { + return; + } + + size_t s = taosArrayGetSize(*pCond); + for (int32_t i = 0; i < s; ++i) { + STblCond* p = taosArrayGet(*pCond, i); + tfree(p->cond); + } + + taosArrayDestroy(*pCond); + + *pCond = NULL; +} + + void tscTagCondRelease(STagCond* pTagCond) { free(pTagCond->tbnameCond.cond); @@ -3162,6 +3167,7 @@ int32_t tscAddQueryInfo(SSqlCmd* pCmd) { static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); + tscColCondRelease(&pQueryInfo->colCond); tscFieldInfoClear(&pQueryInfo->fieldsInfo); tscExprDestroy(pQueryInfo->exprList); @@ -3252,6 +3258,11 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) { goto _error; } + if (tscColCondCopy(&pQueryInfo->colCond, pSrc->colCond, 0, -1) != 0) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + if (pSrc->fillType != TSDB_FILL_NONE) { pQueryInfo->fillVal = calloc(1, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); if (pQueryInfo->fillVal == NULL) { @@ -3647,6 +3658,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t goto _error; } + if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, tableIndex) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + if (pQueryInfo->fillType != TSDB_FILL_NONE) { //just make memory memory sanitizer happy //refactor later @@ -4448,14 +4464,16 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { return cMeta; } -int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta**ppSTable) { assert(*ppChild != NULL); - - STableMeta* p = NULL; - size_t sz = 0; + STableMeta* p = *ppSTable; STableMeta* pChild = *ppChild; - + size_t sz = (p != NULL) ? tscGetTableMetaSize(p) : 0; //ppSTableBuf actually capacity may larger than sz, dont care + if (p != NULL && sz != 0) { + memset((char *)p, 0, sz); + } taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz); + *ppSTable = p; // tableMeta exists, build child table meta according to the super table meta // the uid need to be checked in addition to the general name of the super table. @@ -4474,10 +4492,8 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, memcpy(pChild->schema, p->schema, totalBytes); *ppChild = pChild; - tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here - tfree(p); taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); return -1; } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index fb6bab0cf2..46259c8488 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -186,6 +186,7 @@ typedef void *SDataRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) #define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 +#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) #define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) @@ -201,14 +202,18 @@ void tdFreeDataRow(SDataRow row); void tdInitDataRow(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); + // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { +static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, + int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; if (IS_VAR_DATA_TYPE(type)) { *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value)); + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value)); + } dataRowLen(row) += varDataTLen(value); } else { if (offset == 0) { @@ -223,6 +228,12 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t return 0; } + +// offset here not include dataRow header length +static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { + return tdAppendDataColVal(row, value, true, type, offset); +} + // NOTE: offset here including the header size static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { if (IS_VAR_DATA_TYPE(type)) { @@ -328,11 +339,10 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } int tdAllocMemForCol(SDataCol *pCol, int maxPoints); void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); +int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); -void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { @@ -357,13 +367,11 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { } typedef struct { - int maxRowSize; - int maxCols; // max number of columns - int maxPoints; // max number of points - - int numOfRows; - int numOfCols; // Total number of cols - int sversion; // TODO: set sversion + int maxCols; // max number of columns + int maxPoints; // max number of points + int numOfRows; + int numOfCols; // Total number of cols + int sversion; // TODO: set sversion SDataCol *cols; } SDataCols; @@ -407,7 +415,7 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) { } } -SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); +SDataCols *tdNewDataCols(int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); @@ -475,9 +483,10 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { } // offset here not include kvRow header length -static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t *offset) { +static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, + int32_t offset) { ASSERT(value != NULL); - int32_t toffset = *offset + TD_KV_ROW_HEAD_SIZE; + int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); @@ -485,10 +494,12 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE if (IS_VAR_DATA_TYPE(type)) { - memcpy(ptr, value, varDataTLen(value)); + if (isCopyValData) { + memcpy(ptr, value, varDataTLen(value)); + } kvRowLen(row) += varDataTLen(value); } else { - if (*offset == 0) { + if (offset == 0) { ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); @@ -497,7 +508,6 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t } kvRowLen(row) += TYPE_BYTES[type]; } - *offset += sizeof(SColIdx); return 0; } @@ -592,12 +602,24 @@ typedef void *SMemRow; #define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) -#define SMEM_ROW_DATA 0U // SDataRow -#define SMEM_ROW_KV 1U // SKVRow +#define SMEM_ROW_DATA 0x0U // SDataRow +#define SMEM_ROW_KV 0x01U // SKVRow +#define SMEM_ROW_CONVERT 0x80U // SMemRow convert flag -#define memRowType(r) (*(uint8_t *)(r)) +#define KVRatioKV (0.2f) // all bool +#define KVRatioPredict (0.4f) +#define KVRatioData (0.75f) // all bigint +#define KVRatioConvert (0.9f) + +#define memRowType(r) ((*(uint8_t *)(r)) & 0x01) + +#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory +#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT)) // highest bit +#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) +#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) #define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) +#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT) #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag #define memRowKvBody(r) \ @@ -614,6 +636,14 @@ typedef void *SMemRow; #define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen +static FORCE_INLINE char *memRowEnd(SMemRow row) { + if (isDataRow(row)) { + return (char *)dataRowEnd(memRowDataBody(row)); + } else { + return (char *)kvRowEnd(memRowKvBody(row)); + } +} + #define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version @@ -631,7 +661,6 @@ typedef void *SMemRow; } \ } while (0) -#define memRowSetType(r, t) (memRowType(r) = (t)) #define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) #define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) #define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) @@ -664,12 +693,12 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_ } } -static FORCE_INLINE int tdAppendMemColVal(SMemRow row, const void *value, int16_t colId, int8_t type, int32_t offset, - int32_t *kvOffset) { +static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, + int8_t type, int32_t offset) { if (isDataRow(row)) { - tdAppendColVal(memRowDataBody(row), value, type, offset); + tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset); } else { - tdAppendKvColVal(memRowKvBody(row), value, colId, type, kvOffset); + tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset); } return 0; } @@ -691,6 +720,30 @@ static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value return len; } +/** + * 1. calculate the delta of AllNullLen for SDataRow. + * 2. calculate the real len for SKVRow. + */ +static FORCE_INLINE void tdGetColAppendDeltaLen(const void *value, int8_t colType, int32_t *dataLen, int32_t *kvLen) { + switch (colType) { + case TSDB_DATA_TYPE_BINARY: { + int32_t varLen = varDataLen(value); + *dataLen += (varLen - CHAR_BYTES); + *kvLen += (varLen + sizeof(SColIdx)); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + int32_t varLen = varDataLen(value); + *dataLen += (varLen - TSDB_NCHAR_SIZE); + *kvLen += (varLen + sizeof(SColIdx)); + break; + } + default: { + *kvLen += (TYPE_BYTES[colType] + sizeof(SColIdx)); + break; + } + } +} typedef struct { int16_t colId; @@ -706,7 +759,7 @@ static FORCE_INLINE void setSColInfo(SColInfo* colInfo, int16_t colId, uint8_t c SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2); - +#if 0 // ----------------- Raw payload structure for row: /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->| * | |<----------------- flen ------------->|<--- value part --->| @@ -752,6 +805,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); } +#endif + #ifdef __cplusplus } #endif diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 21b7fd8223..c69a662846 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -53,6 +53,8 @@ int32_t tVariantToString(tVariant *pVar, char *dst); int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); +int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo); + int32_t tVariantTypeSetType(tVariant *pVariant, char type); #ifdef __cplusplus diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 3f0ab7f93e..a3a6c0fed4 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -19,10 +19,10 @@ #include "wchar.h" #include "tarray.h" +static void dataColSetNEleNull(SDataCol *pCol, int nEle); static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows, bool forceSetNull); -//TODO: change caller to use return val int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; if(IS_VAR_DATA_TYPE(pCol->type)) { @@ -31,7 +31,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { if(pCol->spaceSize < spaceNeeded) { void* ptr = realloc(pCol->pData, spaceNeeded); if(ptr == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize, + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; } else { @@ -239,20 +239,19 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->len = 0; } // value from timestamp should be TKEY here instead of TSKEY -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { +int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); if (isAllRowsNull(pCol)) { if (isNull(value, pCol->type)) { // all null value yet, just return - return; + return 0; } + if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1; if (numOfRows > 0) { // Find the first not null value, fill all previouse values as NULL - dataColSetNEleNull(pCol, numOfRows, maxPoints); - } else { - tdAllocMemForCol(pCol, maxPoints); + dataColSetNEleNull(pCol, numOfRows); } } @@ -268,12 +267,21 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } + return 0; +} + +static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); + } } bool isNEleNull(SDataCol *pCol, int nEle) { if(isAllRowsNull(pCol)) return true; for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; + if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } return true; } @@ -290,9 +298,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { } } -void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { - tdAllocMemForCol(pCol, maxPoints); - +static void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; for (int i = 0; i < nEle; i++) { @@ -318,7 +324,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) { } } -SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { +SDataCols *tdNewDataCols(int maxCols, int maxRows) { SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); if (pCols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); @@ -326,6 +332,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { } pCols->maxPoints = maxRows; + pCols->maxCols = maxCols; + pCols->numOfRows = 0; + pCols->numOfCols = 0; if (maxCols > 0) { pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); @@ -342,13 +351,8 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; } - - pCols->maxCols = maxCols; } - pCols->maxRowSize = maxRowSize; - - return pCols; } @@ -357,8 +361,9 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); - if (pCols->cols == NULL) return -1; + void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + if (ptr == NULL) return -1; + pCols->cols = ptr; for(i = oldMaxCols; i < pCols->maxCols; i++) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; @@ -366,10 +371,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { } } - if (schemaTLen(pSchema) > pCols->maxRowSize) { - pCols->maxRowSize = schemaTLen(pSchema); - } - tdResetDataCols(pCols); pCols->numOfCols = schemaNCols(pSchema); @@ -398,7 +399,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { } SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { - SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints); + SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); if (pRet == NULL) return NULL; pRet->numOfCols = pDataCols->numOfCols; @@ -413,7 +414,10 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (keepData) { if (pDataCols->cols[i].len > 0) { - tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints); + if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + tdFreeDataCols(pRet); + return NULL; + } pRet->cols[i].len = pDataCols->cols[i].len; memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { @@ -584,9 +588,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0 && (forceSetNull || (!forceSetNull && !isNull(src2->cols[i].pData, src2->cols[i].type)))) { + if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) { dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, target->maxPoints); + } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, + target->maxPoints); } } target->numOfRows++; @@ -844,7 +851,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch int16_t k; for (k = 0; k < nKvNCols; ++k) { SColInfo *pColInfo = taosArrayGet(stashRow, k); - tdAppendKvColVal(kvRow, pColInfo->colVal, pColInfo->colId, pColInfo->colType, &toffset); + tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset); + toffset += sizeof(SColIdx); } ASSERT(kvLen == memRowTLen(tRow)); } diff --git a/src/common/src/texpr.c b/src/common/src/texpr.c index 973b88fef9..ebdb33fd5b 100644 --- a/src/common/src/texpr.c +++ b/src/common/src/texpr.c @@ -118,7 +118,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { } else if (pNode->nodeType == TSQL_NODE_VALUE) { tVariantDestroy(pNode->pVal); } else if (pNode->nodeType == TSQL_NODE_COL) { - free(pNode->pSchema); + tfree(pNode->pSchema); } free(pNode); @@ -435,7 +435,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { expr->_node.optr = TSDB_RELATION_IN; tVariant* pVal = exception_calloc(1, sizeof(tVariant)); right->pVal = pVal; - pVal->nType = TSDB_DATA_TYPE_ARRAY; + pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY; pVal->arr = taosArrayInit(2, POINTER_BYTES); const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; @@ -502,6 +502,183 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) { *q = (void *)pObj; } +void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { + SBufferReader br = tbufInitReader(buf, len, false); + uint32_t sType = tbufReadUint32(&br); + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); + + int dummy = -1; + tVariant tmpVar = {0}; + size_t t = 0; + int32_t sz = tbufReadInt32(&br); + void *pvar = NULL; + int64_t val = 0; + int32_t bufLen = 0; + if (IS_NUMERIC_TYPE(sType)) { + bufLen = 60; // The maximum length of string that a number is converted to. + } else { + bufLen = 128; + } + + char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + + for (int32_t i = 0; i < sz; i++) { + switch (sType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + *(double *)&val = tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(float *)&val = (float)tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_BINARY: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + default: + taosHashCleanup(pObj); + *q = NULL; + return; + } + + tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); + + if (bufLen < t) { + tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + bufLen = (int32_t)t; + } + + switch (tType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto err_ret; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto err_ret; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto err_ret; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto err_ret; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto err_ret; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto err_ret; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_BINARY: { + if (tVariantDump(&tmpVar, tmp, tType, true)) { + goto err_ret; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + if (tVariantDump(&tmpVar, tmp, tType, true)) { + goto err_ret; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + default: + goto err_ret; + } + + taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + } + + *q = (void *)pObj; + pObj = NULL; + +err_ret: + tVariantDestroy(&tmpVar); + taosHashCleanup(pObj); + tfree(tmp); +} + + tExprNode* exprdup(tExprNode* pNode) { if (pNode == NULL) { return NULL; diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 5da48b2e9a..e92169f097 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -61,7 +61,7 @@ bool tscValidateTableNameLength(size_t len) { // TODO refactor SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) { - if (numOfFilters == 0) { + if (numOfFilters == 0 || src == NULL) { assert(src == NULL); return NULL; } diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index eeffe49adc..13c4160e7f 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -372,21 +372,21 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i } tDataTypeDescriptor tDataTypes[15] = { - {TSDB_DATA_TYPE_NULL, 6,1, "NOTYPE", NULL, NULL, NULL}, - {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_bool}, - {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, - {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, - {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32}, - {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint, getStatics_i64}, - {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f}, - {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d}, - {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, getStatics_bin}, - {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, getStatics_nchr}, - {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, - {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, - {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", tsCompressInt, tsDecompressInt, getStatics_u32}, - {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", tsCompressBigint, tsDecompressBigint, getStatics_u64}, + {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, + {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, + {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, + {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, }; char tTokenTypeSwitcher[13] = { @@ -405,6 +405,32 @@ char tTokenTypeSwitcher[13] = { TSDB_DATA_TYPE_NCHAR, // TK_NCHAR }; +float floatMin = -FLT_MAX, floatMax = FLT_MAX; +double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; + +FORCE_INLINE void* getDataMin(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + return &floatMin; + case TSDB_DATA_TYPE_DOUBLE: + return &doubleMin; + default: + return &tDataTypes[type].minValue; + } +} + +FORCE_INLINE void* getDataMax(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + return &floatMax; + case TSDB_DATA_TYPE_DOUBLE: + return &doubleMax; + default: + return &tDataTypes[type].maxValue; + } +} + + bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; } @@ -566,6 +592,53 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } } +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { + if (optr == TSDB_BINARY_OP_ADD) { + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); + break; + case TSDB_DATA_TYPE_UTINYINT: + *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2); + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)dst) = GET_UINT16_VAL(s1) + GET_UINT16_VAL(s2); + break; + case TSDB_DATA_TYPE_INT: + *((int32_t *)dst) = GET_INT32_VAL(s1) + GET_INT32_VAL(s2); + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)dst) = GET_UINT32_VAL(s1) + GET_UINT32_VAL(s2); + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2); + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)dst) = GET_UINT64_VAL(s1) + GET_UINT64_VAL(s2); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2); + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_VAL(dst, GET_FLOAT_VAL(s1) + GET_FLOAT_VAL(s2)); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2)); + break; + default: { + assert(0); + break; + } + } + } else { + assert(0); + } +} + + void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 9e0f7ffc74..a491df6f98 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -23,6 +23,13 @@ #include "tutil.h" #include "tvariant.h" +#define SET_EXT_INFO(converted, res, minv, maxv, exti) do { \ + if (converted == NULL || exti == NULL || *converted == false) { break; } \ + if ((res) < (minv)) { *exti = -1; break; } \ + if ((res) > (maxv)) { *exti = 1; break; } \ + assert(0); \ + } while (0) + void tVariantCreate(tVariant *pVar, SStrToken *token) { int32_t ret = 0; int32_t type = token->type; @@ -184,7 +191,7 @@ void tVariantDestroy(tVariant *pVar) { } // NOTE: this is only for string array - if (pVar->nType == TSDB_DATA_TYPE_ARRAY) { + if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { size_t num = taosArrayGetSize(pVar->arr); for(size_t i = 0; i < num; i++) { void* p = taosArrayGetP(pVar->arr, i); @@ -192,6 +199,9 @@ void tVariantDestroy(tVariant *pVar) { } taosArrayDestroy(pVar->arr); pVar->arr = NULL; + } else if (pVar->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { + taosArrayDestroy(pVar->arr); + pVar->arr = NULL; } } @@ -220,7 +230,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) { pDst->i64 = pSrc->i64; - } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array + } else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(char*)); for(size_t i = 0; i < num; i++) { @@ -228,9 +238,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } + } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(int64_t)); + pDst->nLen = pSrc->nLen; + assert(pSrc->nLen == num); + for(size_t i = 0; i < num; i++) { + int64_t *p = taosArrayGet(pSrc->arr, i); + taosArrayPush(pDst->arr, p); + } } - if (pDst->nType != TSDB_DATA_TYPE_ARRAY) { + if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) { pDst->nLen = tDataTypes[pDst->nType].bytes; } } @@ -450,7 +469,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val return 0; } -static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr) { +static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { setNull((char *)result, type, tDataTypes[type].bytes); return 0; @@ -540,6 +559,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result } } + if (converted) { + *converted = true; + } + bool code = false; uint64_t ui = 0; @@ -602,6 +625,18 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) { * to column type defined in schema */ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) { + return tVariantDumpEx(pVariant, payload, type, includeLengthPrefix, NULL, NULL); +} + +/* + * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType + * to column type defined in schema + */ +int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) { + if (converted) { + *converted = false; + } + if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) { return -1; } @@ -620,7 +655,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_TINYINT: { - if (convertToInteger(pVariant, &result, type, true, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { + SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo); return -1; } *((int8_t *)payload) = (int8_t) result; @@ -628,7 +664,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_UTINYINT: { - if (convertToInteger(pVariant, &result, type, false, false) < 0) { + if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { + SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo); return -1; } *((uint8_t *)payload) = (uint8_t) result; @@ -636,7 +673,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_SMALLINT: { - if (convertToInteger(pVariant, &result, type, true, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { + SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo); return -1; } *((int16_t *)payload) = (int16_t)result; @@ -644,7 +682,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_USMALLINT: { - if (convertToInteger(pVariant, &result, type, false, false) < 0) { + if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { + SET_EXT_INFO(converted, result, 0, UINT16_MAX - 1, extInfo); return -1; } *((uint16_t *)payload) = (uint16_t)result; @@ -652,7 +691,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_INT: { - if (convertToInteger(pVariant, &result, type, true, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { + SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo); return -1; } *((int32_t *)payload) = (int32_t)result; @@ -660,7 +700,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_UINT: { - if (convertToInteger(pVariant, &result, type, false, false) < 0) { + if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { + SET_EXT_INFO(converted, result, 0, UINT32_MAX - 1, extInfo); return -1; } *((uint32_t *)payload) = (uint32_t)result; @@ -668,7 +709,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_BIGINT: { - if (convertToInteger(pVariant, &result, type, true, false) < 0) { + if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { + SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo); return -1; } *((int64_t *)payload) = (int64_t)result; @@ -676,7 +718,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu } case TSDB_DATA_TYPE_UBIGINT: { - if (convertToInteger(pVariant, &result, type, false, false) < 0) { + if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { + SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo); return -1; } *((uint64_t *)payload) = (uint64_t)result; @@ -696,11 +739,37 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return -1; } + if (converted) { + *converted = true; + } + + if (value > FLT_MAX || value < -FLT_MAX) { + SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo); + return -1; + } SET_FLOAT_VAL(payload, value); } } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { + if (converted) { + *converted = true; + } + + if (pVariant->i64 > FLT_MAX || pVariant->i64 < -FLT_MAX) { + SET_EXT_INFO(converted, pVariant->i64, -FLT_MAX, FLT_MAX, extInfo); + return -1; + } + SET_FLOAT_VAL(payload, pVariant->i64); } else if (IS_FLOAT_TYPE(pVariant->nType)) { + if (converted) { + *converted = true; + } + + if (pVariant->dKey > FLT_MAX || pVariant->dKey < -FLT_MAX) { + SET_EXT_INFO(converted, pVariant->dKey, -FLT_MAX, FLT_MAX, extInfo); + return -1; + } + SET_FLOAT_VAL(payload, pVariant->dKey); } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) { *((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL; @@ -824,6 +893,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return 0; } + /* * In variant, bool/smallint/tinyint/int/bigint share the same attribution of * structure, also ignore the convert the type required @@ -848,7 +918,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: { - convertToInteger(pVariant, &(pVariant->i64), type, true, true); + convertToInteger(pVariant, &(pVariant->i64), type, true, true, NULL); pVariant->nType = TSDB_DATA_TYPE_BIGINT; break; } diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 907562fe26..fbeeeb56d3 100644 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -113,7 +113,6 @@ **/AppMemoryLeakTest.java - **/AuthenticationTest.java **/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java **/DatetimeBefore1970Test.java **/FailOverTest.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index f5f16758c1..521a88b128 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -14,6 +14,8 @@ *****************************************************************************/ package com.taosdata.jdbc; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.*; import java.util.logging.Logger; @@ -127,6 +129,11 @@ public class TSDBDriver extends AbstractDriver { return null; } + if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED); + if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED); + try { TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE)); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java index d626698663..977ae66515 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -33,6 +33,8 @@ public class TSDBError { TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type in tdengine"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PRECISION, "unknown timestamp precision"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, "user is required"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, "password is required"); TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error"); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java index 3c44d69be5..2207db6f93 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -29,6 +29,9 @@ public class TSDBErrorNumbers { public static final int ERROR_UNKNOWN_TIMESTAMP_PRECISION = 0x2316; // unknown timestamp precision public static final int ERROR_RESTFul_Client_Protocol_Exception = 0x2317; public static final int ERROR_RESTFul_Client_IOException = 0x2318; + public static final int ERROR_USER_IS_REQUIRED = 0x2319; // user is required + public static final int ERROR_PASSWORD_IS_REQUIRED = 0x231a; // password is required + public static final int ERROR_UNKNOWN = 0x2350; //unknown error @@ -67,6 +70,8 @@ public class TSDBErrorNumbers { errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE); errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PRECISION); errorNumbers.add(ERROR_RESTFul_Client_IOException); + errorNumbers.add(ERROR_USER_IS_REQUIRED); + errorNumbers.add(ERROR_PASSWORD_IS_REQUIRED); errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 4fdbb308c5..c634fe2e95 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -36,7 +36,6 @@ public class TSDBJNIConnector { static { System.loadLibrary("taos"); - System.out.println("java.library.path:" + System.getProperty("java.library.path")); } public boolean isClosed() { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index 9ab67c5502..0a8809e84f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -7,6 +7,7 @@ import com.taosdata.jdbc.utils.HttpClientPoolUtil; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.Properties; import java.util.logging.Logger; @@ -40,8 +41,13 @@ public class RestfulDriver extends AbstractDriver { String loginUrl = "http://" + host + ":" + port + "/rest/login/" + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + "/" + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD) + ""; try { - String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), "UTF-8"); - String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), "UTF-8"); + if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED); + if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED); + + String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), StandardCharsets.UTF_8.displayName()); + String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), StandardCharsets.UTF_8.displayName()); loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + ""; } catch (UnsupportedEncodingException e) { e.printStackTrace(); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index f8acd8f061..a88dc411f3 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -7,6 +7,7 @@ import com.taosdata.jdbc.AbstractStatement; import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBError; import com.taosdata.jdbc.TSDBErrorNumbers; +import com.taosdata.jdbc.enums.TimestampFormat; import com.taosdata.jdbc.utils.HttpClientPoolUtil; import com.taosdata.jdbc.utils.SqlSyntaxValidator; @@ -45,9 +46,7 @@ public class RestfulStatement extends AbstractStatement { if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); - final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - - return executeOneUpdate(url, sql); + return executeOneUpdate(sql); } @Override @@ -62,34 +61,25 @@ public class RestfulStatement extends AbstractStatement { public boolean execute(String sql) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - if (!SqlSyntaxValidator.isValidForExecute(sql)) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: " + sql); //如果执行了use操作应该将当前Statement的catalog设置为新的database boolean result = true; - String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("TIMESTAMP")) { - url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt"; - } - if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("UTC")) { - url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc"; - } if (SqlSyntaxValidator.isUseSql(sql)) { - HttpClientPoolUtil.execute(url, sql, this.conn.getToken()); + HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); this.database = sql.trim().replace("use", "").trim(); this.conn.setCatalog(this.database); result = false; } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { executeOneQuery(sql); } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { - executeOneUpdate(url, sql); + executeOneUpdate(sql); result = false; } else { if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) { - executeQuery(sql); + executeOneQuery(sql); } else { - executeUpdate(sql); + executeOneUpdate(sql); result = false; } } @@ -97,19 +87,25 @@ public class RestfulStatement extends AbstractStatement { return result; } + private String getUrl() throws SQLException { + TimestampFormat timestampFormat = TimestampFormat.valueOf(conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).trim().toUpperCase()); + String url; + switch (timestampFormat) { + case TIMESTAMP: + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt"; + break; + case UTC: + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc"; + break; + default: + url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + } + return url; + } + private ResultSet executeOneQuery(String sql) throws SQLException { - if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); - // row data - String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - String timestampFormat = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT); - if ("TIMESTAMP".equalsIgnoreCase(timestampFormat)) - url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt"; - if ("UTC".equalsIgnoreCase(timestampFormat)) - url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc"; - - String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken()); + String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); JSONObject resultJson = JSON.parseObject(result); if (resultJson.getString("status").equals("error")) { throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc")); @@ -119,11 +115,8 @@ public class RestfulStatement extends AbstractStatement { return resultSet; } - private int executeOneUpdate(String url, String sql) throws SQLException { - if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); - - String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken()); + private int executeOneUpdate(String sql) throws SQLException { + String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); JSONObject jsonObject = JSON.parseObject(result); if (jsonObject.getString("status").equals("error")) { throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc")); @@ -134,7 +127,7 @@ public class RestfulStatement extends AbstractStatement { } private int getAffectedRows(JSONObject jsonObject) throws SQLException { - // create ... SQLs should return 0 , and Restful result is this: + // create ... SQLs should return 0 , and Restful result like this: // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1} JSONArray head = jsonObject.getJSONArray("head"); if (head.size() != 1 || !"affected_rows".equals(head.getString(0))) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index 0f99ff4f66..cbd806b35a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -16,8 +16,7 @@ package com.taosdata.jdbc.utils; public class SqlSyntaxValidator { - private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe", "reset"}; - private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"}; + private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set", "reset"}; private static final String[] querySQL = {"select", "show", "describe"}; private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"}; @@ -38,14 +37,6 @@ public class SqlSyntaxValidator { return false; } - public static boolean isValidForExecute(String sql) { - for (String prefix : SQL) { - if (sql.trim().toLowerCase().startsWith(prefix)) - return true; - } - return false; - } - public static boolean isDatabaseUnspecifiedQuery(String sql) { for (String databaseObj : databaseUnspecifiedShow) { if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*")) @@ -63,9 +54,5 @@ public class SqlSyntaxValidator { return sql.trim().toLowerCase().startsWith("use"); } - public static boolean isSelectSql(String sql) { - return sql.trim().toLowerCase().startsWith("select"); - } - } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 24c73fdd5c..95307071e1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -69,6 +69,8 @@ public class SubscribeTest { @Before public void createDatabase() throws SQLException { Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java index 6702de9bdb..0ea46dade2 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java @@ -1,6 +1,9 @@ package com.taosdata.jdbc.cases; +import com.taosdata.jdbc.TSDBErrorNumbers; +import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.sql.*; @@ -12,6 +15,47 @@ public class AuthenticationTest { private static final String password = "taos?data"; private Connection conn; + @Test + public void connectWithoutUserByJni() { + try { + DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?"); + } catch (SQLException e) { + Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode()); + Assert.assertEquals("ERROR (2319): user is required", e.getMessage()); + } + } + + @Test + public void connectWithoutUserByRestful() { + try { + DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?"); + } catch (SQLException e) { + Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode()); + Assert.assertEquals("ERROR (2319): user is required", e.getMessage()); + } + } + + @Test + public void connectWithoutPasswordByJni() { + try { + DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root"); + } catch (SQLException e) { + Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode()); + Assert.assertEquals("ERROR (231a): password is required", e.getMessage()); + } + } + + @Test + public void connectWithoutPasswordByRestful() { + try { + DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root"); + } catch (SQLException e) { + Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode()); + Assert.assertEquals("ERROR (231a): password is required", e.getMessage()); + } + } + + @Ignore @Test public void test() { // change password diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index e175d6d114..f2b102cfe7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -29,6 +29,8 @@ public class BatchInsertTest { public void before() { try { Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java index bc11c7f34e..1297a6b4c4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java @@ -21,6 +21,8 @@ public class ImportTest { public static void before() { try { Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java index 9f8243542f..ac254bebf3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java @@ -270,6 +270,41 @@ public class InsertSpecialCharacterJniTest { } } + @Ignore + @Test + public void testSingleQuotaEscape() throws SQLException { + final long now = System.currentTimeMillis(); + final String sql = "insert into t? using ? tags(?) values(?, ?, ?) t? using " + tbname2 + " tags(?) values(?,?,?) "; + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + // t1 + pstmt.setInt(1, 1); + pstmt.setString(2, tbname2); + pstmt.setString(3, special_character_str_5); + pstmt.setTimestamp(4, new Timestamp(now)); + pstmt.setBytes(5, special_character_str_5.getBytes()); + // t2 + pstmt.setInt(7, 2); + pstmt.setString(8, special_character_str_5); + pstmt.setTimestamp(9, new Timestamp(now)); + pstmt.setString(11, special_character_str_5); + + int ret = pstmt.executeUpdate(); + Assert.assertEquals(2, ret); + } + + String query = "select * from ?.t? where ? < ? and ts >= ? and f1 is not null"; + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + pstmt.setString(1, dbName); + pstmt.setInt(2, 1); + pstmt.setString(3, "ts"); + pstmt.setTimestamp(4, new Timestamp(System.currentTimeMillis())); + pstmt.setTimestamp(5, new Timestamp(0)); + + ResultSet rs = pstmt.executeQuery(); + Assert.assertNotNull(rs); + } + } + @Test public void testCase10() throws SQLException { final long now = System.currentTimeMillis(); @@ -293,13 +328,12 @@ public class InsertSpecialCharacterJniTest { Assert.assertEquals(2, ret); } //query t1 - String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null"; + String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null"; try (PreparedStatement pstmt = conn.prepareStatement(query)) { pstmt.setString(1, dbName); pstmt.setInt(2, 1); pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis())); pstmt.setTimestamp(4, new Timestamp(0)); - pstmt.setString(5, "f1"); ResultSet rs = pstmt.executeQuery(); rs.next(); @@ -311,12 +345,11 @@ public class InsertSpecialCharacterJniTest { Assert.assertNull(f2); } // query t2 - query = "select * from t? where ts < ? and ts >= ? and ? is not null"; + query = "select * from t? where ts < ? and ts >= ? and f2 is not null"; try (PreparedStatement pstmt = conn.prepareStatement(query)) { pstmt.setInt(1, 2); pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis())); pstmt.setTimestamp(3, new Timestamp(0)); - pstmt.setString(4, "f2"); ResultSet rs = pstmt.executeQuery(); rs.next(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java index 2e981e7f41..eedccec6f1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java @@ -293,13 +293,12 @@ public class InsertSpecialCharacterRestfulTest { Assert.assertEquals(2, ret); } //query t1 - String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null"; + String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null"; try (PreparedStatement pstmt = conn.prepareStatement(query)) { pstmt.setString(1, dbName); pstmt.setInt(2, 1); pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis())); pstmt.setTimestamp(4, new Timestamp(0)); - pstmt.setString(5, "f1"); ResultSet rs = pstmt.executeQuery(); rs.next(); @@ -311,12 +310,11 @@ public class InsertSpecialCharacterRestfulTest { Assert.assertNull(f2); } // query t2 - query = "select * from t? where ts < ? and ts >= ? and ? is not null"; + query = "select * from t? where ts < ? and ts >= ? and f2 is not null"; try (PreparedStatement pstmt = conn.prepareStatement(query)) { pstmt.setInt(1, 2); pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis())); pstmt.setTimestamp(3, new Timestamp(0)); - pstmt.setString(4, "f2"); ResultSet rs = pstmt.executeQuery(); rs.next(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java index 3fea221446..b4449491a9 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java @@ -22,6 +22,8 @@ public class QueryDataTest { public void createDatabase() { try { Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java index 4eebbd08e8..48753d62f0 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java @@ -1,51 +1,49 @@ package com.taosdata.jdbc.cases; -import com.taosdata.jdbc.TSDBDriver; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import java.sql.*; -import java.util.Properties; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; public class ResetQueryCacheTest { - static Connection connection; - static Statement statement; - static String host = "127.0.0.1"; + @Test + public void jni() throws SQLException { + // given + Connection connection = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:0/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8"); + Statement statement = connection.createStatement(); - @Before - public void init() { - try { - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - statement = connection.createStatement(); - } catch (SQLException e) { - return; - } + // when + boolean execute = statement.execute("reset query cache"); + + // then + assertFalse(execute); + assertEquals(0, statement.getUpdateCount()); + + statement.close(); + connection.close(); } @Test - public void testResetQueryCache() throws SQLException { - String resetSql = "reset query cache"; - statement.execute(resetSql); - } + public void restful() throws SQLException { + // given + Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://127.0.0.1:6041/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8"); + Statement statement = connection.createStatement(); - @After - public void close() { - try { - if (statement != null) - statement.close(); - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } + // when + boolean execute = statement.execute("reset query cache"); + + // then + assertFalse(execute); + assertEquals(0, statement.getUpdateCount()); + + statement.close(); + connection.close(); } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java index 0022ceaf21..b51c0309be 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java @@ -20,6 +20,8 @@ public class SelectTest { public void createDatabaseAndTable() { try { Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java index 1600fec13d..8e10743e5e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java @@ -24,6 +24,8 @@ public class StableTest { public static void createDatabase() { try { Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java deleted file mode 100644 index ccb0941da0..0000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.taosdata.jdbc.utils; - -import org.junit.Assert; -import org.junit.Test; - -public class SqlSyntaxValidatorTest { - - @Test - public void isSelectSQL() { - Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather")); - Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather")); - Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather ")); - Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)")); - } - - @Test - public void isUseSQL() { - Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); - } - -} \ No newline at end of file diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 5b96f0ebd9..fceeaea0ae 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -34,6 +34,7 @@ extern "C" { #define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN}) +#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) #define TSKEY_INITIAL_VAL INT64_MIN diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 2214078f55..368658377c 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -272,7 +272,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query") #define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached") #define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") -#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) //"System error") +#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition") +#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error") // grant diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c2918cfdf7..a74a531361 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -489,7 +489,8 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; SSessionWindow sw; // session window - uint32_t tagCondLen; // tag length in current query + uint16_t tagCondLen; // tag length in current query + uint16_t colCondLen; // column length in current query uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 44e666106a..3f1606e957 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -47,7 +47,8 @@ typedef struct { // this data type is internally used only in 'in' query to hold the values -#define TSDB_DATA_TYPE_ARRAY (1000) +#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) +#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) #define GET_TYPED_DATA(_v, _finalType, _type, _data) \ do { \ @@ -181,6 +182,8 @@ typedef struct tDataTypeDescriptor { int16_t nameLen; int32_t bytes; char * name; + int64_t minValue; + int64_t maxValue; int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize); int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, @@ -200,6 +203,9 @@ const void *getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); +void* getDataMin(int32_t type); +void* getDataMax(int32_t type); int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned); diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index ba88a2b9cd..02f21037b8 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -101,13 +101,17 @@ char *httpGetStatusDesc(int32_t statusCode) { } static void httpCleanupString(HttpString *str) { - free(str->str); - str->str = NULL; - str->pos = 0; - str->size = 0; + if (str->str) { + free(str->str); + str->str = NULL; + str->pos = 0; + str->size = 0; + } } static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { + char *new_str = NULL; + if (str->size == 0) { str->pos = 0; str->size = len + 1; @@ -115,7 +119,16 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { } else if (str->pos + len + 1 >= str->size) { str->size += len; str->size *= 4; - str->str = realloc(str->str, str->size); + + new_str = realloc(str->str, str->size); + if (new_str == NULL && str->str) { + // if str->str was not NULL originally, + // the old allocated memory was left unchanged, + // see man 3 realloc + free(str->str); + } + + str->str = new_str; } else { } @@ -317,7 +330,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { HttpContext *pContext = parser->pContext; - HttpString * buf = &parser->body; + HttpString *buf = &parser->body; if (parser->parseCode != TSDB_CODE_SUCCESS) return -1; if (buf->size <= 0) { @@ -326,6 +339,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { } int32_t newSize = buf->pos + len + 1; + char *newStr = NULL; if (newSize >= buf->size) { if (buf->size >= HTTP_BUFFER_SIZE) { httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size); @@ -336,7 +350,12 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { newSize = MAX(newSize, HTTP_BUFFER_INIT); newSize *= 4; newSize = MIN(newSize, HTTP_BUFFER_SIZE); - buf->str = realloc(buf->str, newSize); + newStr = realloc(buf->str, newSize); + if (newStr == NULL && buf->str) { + free(buf->str); + } + + buf->str = newStr; buf->size = newSize; if (buf->str == NULL) { @@ -374,13 +393,20 @@ static HTTP_PARSER_STATE httpTopStack(HttpParser *parser) { static int32_t httpPushStack(HttpParser *parser, HTTP_PARSER_STATE state) { HttpStack *stack = &parser->stacks; + int8_t *newStacks = NULL; if (stack->size == 0) { stack->pos = 0; stack->size = 32; stack->stacks = malloc(stack->size * sizeof(int8_t)); } else if (stack->pos + 1 > stack->size) { stack->size *= 2; - stack->stacks = realloc(stack->stacks, stack->size * sizeof(int8_t)); + + newStacks = realloc(stack->stacks, stack->size * sizeof(int8_t)); + if (newStacks == NULL && stack->stacks) { + free(stack->stacks); + } + + stack->stacks = newStacks; } else { } diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index a8031d3fd8..ade50bdad6 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -188,13 +188,17 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) { HttpSqlCmds *multiCmds = pContext->multiCmds; - if (cmdSize > HTTP_MAX_CMD_SIZE) { + if (cmdSize <= 0 || cmdSize > HTTP_MAX_CMD_SIZE) { httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user, cmdSize, HTTP_MAX_CMD_SIZE); return false; } - multiCmds->cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd)); + HttpSqlCmd *new_cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd)); + if (new_cmds == NULL && multiCmds->cmds) { + free(multiCmds->cmds); + } + multiCmds->cmds = new_cmds; if (multiCmds->cmds == NULL) { httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; @@ -208,13 +212,17 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) { bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) { HttpSqlCmds *multiCmds = pContext->multiCmds; - if (bufferSize > HTTP_MAX_BUFFER_SIZE) { + if (bufferSize <= 0 || bufferSize > HTTP_MAX_BUFFER_SIZE) { httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd, pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE); return false; } - multiCmds->buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize); + char *new_buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize); + if (new_buffer == NULL && multiCmds->buffer) { + free(multiCmds->buffer); + } + multiCmds->buffer = new_buffer; if (multiCmds->buffer == NULL) { httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize); return false; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ce70a9ba4a..996d925756 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -252,8 +252,10 @@ typedef struct SQueryAttr { int32_t numOfFilterCols; int64_t* fillVal; SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. - SSingleColumnFilterInfo* pFilterInfo; + SSingleColumnFilterInfo* pFilterInfo; + SFilterInfo *pFilters; + void* tsdb; SMemRef memRef; STableGroupInfo tableGroupInfo; // table list SArray @@ -382,6 +384,7 @@ typedef struct SQInfo { typedef struct SQueryParam { char *sql; char *tagCond; + char *colCond; char *tbnameCond; char *prevResult; SArray *pTableIdList; @@ -390,6 +393,8 @@ typedef struct SQueryParam { SExprInfo *pExprs; SExprInfo *pSecExprs; + SFilterInfo *pFilters; + SColIndex *pGroupColIndex; SColumnInfo *pTagColumnInfo; SGroupbyExpr *pGroupbyExpr; @@ -573,6 +578,7 @@ SSDataBlock* doSLimit(void* param, bool* newgroup); int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); +void doSetFilterColInfo(SFilterInfo *pFilters, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); @@ -594,9 +600,11 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo); +int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); + SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo); + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo); int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, int32_t prevResultLen, void* merger); diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h new file mode 100644 index 0000000000..7a7b3157ea --- /dev/null +++ b/src/query/inc/qFilter.h @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_QFILTER_H +#define TDENGINE_QFILTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "texpr.h" +#include "hash.h" +#include "tname.h" + +#define FILTER_DEFAULT_GROUP_SIZE 4 +#define FILTER_DEFAULT_UNIT_SIZE 4 +#define FILTER_DEFAULT_FIELD_SIZE 4 +#define FILTER_DEFAULT_VALUE_SIZE 4 +#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2 + +#define FILTER_DUMMY_EMPTY_OPTR 127 +#define FILTER_DUMMY_RANGE_OPTR 126 + +#define MAX_NUM_STR_SIZE 40 + +enum { + FLD_TYPE_COLUMN = 1, + FLD_TYPE_VALUE = 2, + FLD_TYPE_MAX = 3, + FLD_DESC_NO_FREE = 4, + FLD_DATA_NO_FREE = 8, + FLD_DATA_IS_HASH = 16, +}; + +enum { + MR_ST_START = 1, + MR_ST_FIN = 2, + MR_ST_ALL = 4, + MR_ST_EMPTY = 8, +}; + +enum { + RANGE_FLG_EXCLUDE = 1, + RANGE_FLG_INCLUDE = 2, + RANGE_FLG_NULL = 4, +}; + +enum { + FI_OPTION_NO_REWRITE = 1, + FI_OPTION_TIMESTAMP = 2, + FI_OPTION_NEED_UNIQE = 4, +}; + +enum { + FI_STATUS_ALL = 1, + FI_STATUS_EMPTY = 2, + FI_STATUS_REWRITE = 4, + FI_STATUS_CLONED = 8, +}; + +enum { + RANGE_TYPE_UNIT = 1, + RANGE_TYPE_VAR_HASH = 2, + RANGE_TYPE_MR_CTX = 3, +}; + +typedef struct OptrStr { + uint16_t optr; + char *str; +} OptrStr; + +typedef struct SFilterRange { + int64_t s; + int64_t e; + char sflag; + char eflag; +} SFilterRange; + +typedef struct SFilterColRange { + uint16_t idx; //column field idx + bool isNull; + bool notNull; + bool isRange; + SFilterRange ra; +} SFilterColRange; + +typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); +typedef int32_t(*filter_desc_compare_func)(const void *, const void *); +typedef bool(*filter_exec_func)(void *, int32_t, int8_t*); + +typedef struct SFilterRangeCompare { + int64_t s; + int64_t e; + rangeCompFunc func; +} SFilterRangeCompare; + +typedef struct SFilterRangeNode { + struct SFilterRangeNode* prev; + struct SFilterRangeNode* next; + union { + SFilterRange ra; + SFilterRangeCompare rc; + }; +} SFilterRangeNode; + +typedef struct SFilterRangeCtx { + int32_t type; + int32_t options; + int8_t status; + bool isnull; + bool notnull; + bool isrange; + int16_t colId; + __compar_fn_t pCompareFunc; + SFilterRangeNode *rf; //freed + SFilterRangeNode *rs; +} SFilterRangeCtx ; + +typedef struct SFilterVarCtx { + int32_t type; + int32_t options; + int8_t status; + bool isnull; + bool notnull; + bool isrange; + SHashObj *wild; + SHashObj *value; +} SFilterVarCtx; + +typedef struct SFilterField { + uint16_t flag; + void* desc; + void* data; +} SFilterField; + +typedef struct SFilterFields { + uint16_t size; + uint16_t num; + SFilterField *fields; +} SFilterFields; + +typedef struct SFilterFieldId { + uint16_t type; + uint16_t idx; +} SFilterFieldId; + +typedef struct SFilterGroup { + uint16_t unitSize; + uint16_t unitNum; + uint16_t *unitIdxs; + uint8_t *unitFlags; // !unit result +} SFilterGroup; + +typedef struct SFilterColInfo { + uint8_t type; + int32_t dataType; + void *info; +} SFilterColInfo; + +typedef struct SFilterGroupCtx { + uint16_t colNum; + uint16_t *colIdx; + SFilterColInfo *colInfo; +} SFilterGroupCtx; + +typedef struct SFilterColCtx { + uint16_t colIdx; + void* ctx; +} SFilterColCtx; + +typedef struct SFilterCompare { + uint8_t type; + uint8_t optr; + uint8_t optr2; +} SFilterCompare; + +typedef struct SFilterUnit { + SFilterCompare compare; + SFilterFieldId left; + SFilterFieldId right; + SFilterFieldId right2; +} SFilterUnit; + +typedef struct SFilterComUnit { + void *colData; + void *valData; + void *valData2; + uint16_t dataSize; + uint8_t dataType; + uint8_t optr; + int8_t func; + int8_t rfunc; +} SFilterComUnit; + +typedef struct SFilterPCtx { + SHashObj *valHash; + SHashObj *unitHash; +} SFilterPCtx; + +typedef struct SFilterInfo { + uint32_t options; + uint32_t status; + uint16_t unitSize; + uint16_t unitNum; + uint16_t groupNum; + uint16_t colRangeNum; + SFilterFields fields[FLD_TYPE_MAX]; + SFilterGroup *groups; + uint16_t *cgroups; + SFilterUnit *units; + SFilterComUnit *cunits; + uint8_t *unitRes; // result + uint8_t *unitFlags; // got result + SFilterRangeCtx **colRange; + filter_exec_func func; + + SFilterPCtx pctx; +} SFilterInfo; + +#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) + +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) +#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) + +#define MR_EMPTY_RES(ctx) (ctx->rs == NULL) + +#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) +#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) +#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) +#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) + + +#define FILTER_GET_FLAG(st, f) (st & f) +#define FILTER_SET_FLAG(st, f) st |= (f) +#define FILTER_CLR_FLAG(st, f) st &= (~f) + +#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) +#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) +#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) +#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) + +#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0) +#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0) +#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0) +#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) +#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) + +#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) +#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) +#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) + +#define CHK_RETV(c) do { if (c) { return; } } while (0) +#define CHK_RET(c, r) do { if (c) { return r; } } while (0) +#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) +#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) +#define CHK_LRET(c, r,...) do { if (c) { qError(__VA_ARGS__); return r; } } while (0) + +#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) +#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) +#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) +#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) +#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) + +#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) +#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left) +#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right) +#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type) +#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u)) +#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri) +#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) +#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) +#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) +#define FILTER_UNIT_OPTR(u) ((u)->compare.optr) +#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) + +#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags)) +#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1 +#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx]) +#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx]) +#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v) + +#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0) +#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) +#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) + +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) + +#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) + + +#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) +#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) + + +extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); +extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p); +extern int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data); +extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); +extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); +extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); +extern void filterFreeInfo(SFilterInfo *info); +extern bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_QFILTER_H diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 0dae74ac82..d6b04b0330 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -3,6 +3,7 @@ #include "tsdb.h" //todo tsdb should not be here #include "qSqlparser.h" +#include "qFilter.h" typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result @@ -16,6 +17,14 @@ typedef struct SCond { char * cond; } SCond; +typedef struct STblCond { + uint64_t uid; + int16_t idx; //table index + int32_t len; // length of tag query condition data + char * cond; +} STblCond; + + typedef struct SJoinNode { uint64_t uid; int16_t tagColId; @@ -89,6 +98,11 @@ typedef struct STableMetaInfo { struct SQInfo; // global merge operator struct SQueryAttr; // query object +typedef struct STableFilter { + uint64_t uid; + SFilterInfo info; +} STableFilter; + typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. uint32_t type; // query/insert type @@ -106,8 +120,11 @@ typedef struct SQueryInfo { SLimitVal slimit; STagCond tagCond; + SArray * colCond; + SOrderVal order; int16_t numOfTables; + int16_t curTableIdx; STableMetaInfo **pTableMetaInfo; struct STSBuf *tsBuf; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5323b4306f..22e4f87ef9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2545,77 +2545,14 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) { + if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { return true; } - bool ret = true; - for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) { - SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k]; - int32_t index = -1; - for(int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { - if (pDataStatis[i].colId == pFilterInfo->info.colId) { - index = i; - break; - } - } - - // no statistics data, load the true data block - if (index == -1) { - return true; - } - - // not support pre-filter operation on binary/nchar data type - if (!IS_PREFILTER_TYPE(pFilterInfo->info.type)) { - return true; - } - - // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { - - // if isNULL query exists, load the null data column - for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { - SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; - if (pFilterElem->fp == isNullOperator) { - return true; - } - } - - continue; - } - - SDataStatis* pDataBlockst = &pDataStatis[index]; - - if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) { - float minval = (float)(*(double *)(&pDataBlockst->min)); - float maxval = (float)(*(double *)(&pDataBlockst->max)); - - for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - continue; - } - ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT); - if (ret == false) { - return false; - } - } - } else { - for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) { - continue; - } - ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type); - if (ret == false) { - return false; - } - } - } - } - - return ret; + return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); } static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { @@ -2834,6 +2771,49 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf tfree(p); } +void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = calloc(numOfRows, sizeof(int8_t)); + bool all = true; + + if (pRuntimeEnv->pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; + } + + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } + } + + // save the cursor status + pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + } else { + all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, p); + } + + if (!all) { + doCompactSDataBlock(pBlock, numOfRows, p); + } + + tfree(p); +} + + + static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes); @@ -2875,6 +2855,15 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } + +void doSetFilterColInfo(SFilterInfo * pFilters, SSDataBlock* pBlock) { + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); + + filterSetColFieldData(pFilters, pColInfo->info.colId, pColInfo->pData); + } +} + int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; @@ -2911,7 +2900,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. - if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || + if (pQueryAttr->pFilters || pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { (*status) = BLK_DATA_ALL_NEEDED; } @@ -3013,9 +3002,12 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa return terrno; } - doSetFilterColumnInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols, pBlock); - if (pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) { - filterRowsInDataBlock(pRuntimeEnv, pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols, pBlock, ascQuery); + if (pQueryAttr->pFilters != NULL) { + doSetFilterColInfo(pQueryAttr->pFilters, pBlock); + } + + if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { + filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); } } @@ -6899,7 +6891,8 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - pQueryMsg->tagCondLen = htonl(pQueryMsg->tagCondLen); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); @@ -6930,7 +6923,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pColInfo->colId = htons(pColInfo->colId); pColInfo->type = htons(pColInfo->type); pColInfo->bytes = htons(pColInfo->bytes); - pColInfo->flist.numOfFilters = htons(pColInfo->flist.numOfFilters); + pColInfo->flist.numOfFilters = 0; if (!isValidDataType(pColInfo->type)) { qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); @@ -6938,6 +6931,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { goto _cleanup; } +/* int32_t numOfFilters = pColInfo->flist.numOfFilters; if (numOfFilters > 0) { pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); @@ -6951,8 +6945,21 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { if (code != TSDB_CODE_SUCCESS) { goto _cleanup; } +*/ } + if (pQueryMsg->colCondLen > 0) { + param->colCond = calloc(1, pQueryMsg->colCondLen); + if (param->colCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); + pMsg += pQueryMsg->colCondLen; + } + + param->tableScanOperator = pQueryMsg->tableScanOperator; param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); if (param->pExpr == NULL) { @@ -7526,6 +7533,28 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp return TSDB_CODE_SUCCESS; } +int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { + tExprNode* expr = NULL; + + TRY(TSDB_MAX_TAG_CONDITIONS) { + expr = exprTreeFromBinary(data, len); + } CATCH( code ) { + CLEANUP_EXECUTE(); + return code; + } END_TRY + + if (expr == NULL) { + qError("failed to create expr tree"); + return TSDB_CODE_QRY_APP_ERROR; + } + + int32_t ret = filterInitFromTree(expr, pFilters, 0); + tExprTreeDestroy(expr, NULL); + + return ret; +} + + // todo refactor int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExpr, SExprInfo* prevExpr, SUdfInfo *pUdfInfo) { @@ -7675,7 +7704,7 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { - if (pQueryAttr->tableCols[i].flist.numOfFilters > 0) { + if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { pQueryAttr->numOfFilterCols++; } } @@ -7758,7 +7787,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { } SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, - SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, + SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -7810,7 +7839,8 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; pQueryAttr->stateWindow = pQueryMsg->stateWindow; pQueryAttr->vgId = vgId; - + pQueryAttr->pFilters = pFilters; + pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQueryAttr->tableCols == NULL) { goto _cleanup; @@ -7843,10 +7873,6 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S } doUpdateExprColumnIndex(pQueryAttr); - int32_t ret = createFilterInfo(pQueryAttr, pQInfo->qId); - if (ret != TSDB_CODE_SUCCESS) { - goto _cleanup; - } if (pSecExprs != NULL) { int32_t resultRowSize = 0; @@ -7961,6 +7987,8 @@ _cleanup_qinfo: tfree(pExprs); + filterFreeInfo(pFilters); + _cleanup: freeQInfo(pQInfo); return NULL; @@ -8316,6 +8344,8 @@ void freeQueryAttr(SQueryAttr* pQueryAttr) { taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); tfree(pQueryAttr->pGroupbyExpr); } + + filterFreeInfo(pQueryAttr->pFilters); } } diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c new file mode 100644 index 0000000000..1171bb0896 --- /dev/null +++ b/src/query/src/qFilter.c @@ -0,0 +1,3076 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include "queryLog.h" +#include "qFilter.h" +#include "tcompare.h" +#include "hash.h" +#include "tscUtil.h" + +OptrStr gOptrStr[] = { + {TSDB_RELATION_INVALID, "invalid"}, + {TSDB_RELATION_LESS, "<"}, + {TSDB_RELATION_GREATER, ">"}, + {TSDB_RELATION_EQUAL, "="}, + {TSDB_RELATION_LESS_EQUAL, "<="}, + {TSDB_RELATION_GREATER_EQUAL, ">="}, + {TSDB_RELATION_NOT_EQUAL, "!="}, + {TSDB_RELATION_LIKE, "like"}, + {TSDB_RELATION_ISNULL, "is null"}, + {TSDB_RELATION_NOTNULL, "not null"}, + {TSDB_RELATION_IN, "in"}, + {TSDB_RELATION_AND, "and"}, + {TSDB_RELATION_OR, "or"}, + {TSDB_RELATION_NOT, "not"} +}; + +static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { + const SSchema *sch1 = desc1; + const SSchema *sch2 = desc2; + + return sch1->colId != sch2->colId; +} + +static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { + const tVariant *val1 = desc1; + const tVariant *val2 = desc2; + + return tVariantCompare(val1, val2); +} + + +filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { + NULL, + filterFieldColDescCompare, + filterFieldValDescCompare +}; + +bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0; +} +bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0; +} +bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(minv, maxr) <= 0; +} +bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(minv, maxr) < 0; +} +bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; +} +bool filterRangeCompee (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) < 0; +} +bool filterRangeCompei (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) <= 0; +} +bool filterRangeCompie (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) < 0; +} + +rangeCompFunc filterGetRangeCompFunc(char sflag, char eflag) { + if (FILTER_GET_FLAG(sflag, RANGE_FLG_NULL)) { + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompLe; + } + + return filterRangeCompLi; + } + + if (FILTER_GET_FLAG(eflag, RANGE_FLG_NULL)) { + if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompGe; + } + + return filterRangeCompGi; + } + + if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompee; + } + + return filterRangeCompei; + } + + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompie; + } + + return filterRangeCompii; +} + +rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRangeCompie, filterRangeCompii, filterRangeCompGe, + filterRangeCompGi, filterRangeCompLe, filterRangeCompLi}; + + +int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { + if (optr2) { + assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); + + if (optr == TSDB_RELATION_GREATER) { + if (optr2 == TSDB_RELATION_LESS) { + return 0; + } + + return 1; + } + + if (optr2 == TSDB_RELATION_LESS) { + return 2; + } + + return 3; + } else { + switch (optr) { + case TSDB_RELATION_GREATER: + return 4; + case TSDB_RELATION_GREATER_EQUAL: + return 5; + case TSDB_RELATION_LESS: + return 6; + case TSDB_RELATION_LESS_EQUAL: + return 7; + default: + break; + } + } + + return -1; +} + +__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, + compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, + compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, + setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8 +}; + +int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { + int8_t comparFn = 0; + + if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return 15; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return 16; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return 17; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return 18; + default: + assert(0); + } + } + + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = 2; break; + case TSDB_DATA_TYPE_INT: comparFn = 0; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; + case TSDB_DATA_TYPE_BINARY: { + if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparFn = 7; + } else if (optr == TSDB_RELATION_IN) { + comparFn = 8; + } else { /* normal relational comparFn */ + comparFn = 6; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + if (optr == TSDB_RELATION_LIKE) { + comparFn = 9; + } else if (optr == TSDB_RELATION_IN) { + comparFn = 8; + } else { + comparFn = 10; + } + break; + } + + case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = 12;break; + case TSDB_DATA_TYPE_UINT: comparFn = 13;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = 14;break; + + default: + comparFn = 0; + break; + } + + return comparFn; +} + + +static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { + SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); + if (left->colNum > right->colNum) return 1; + if (left->colNum < right->colNum) return -1; + return 0; +} + +int32_t filterInitUnitsFields(SFilterInfo *info) { + info->unitSize = FILTER_DEFAULT_UNIT_SIZE; + info->units = calloc(info->unitSize, sizeof(SFilterUnit)); + + info->fields[FLD_TYPE_COLUMN].num = 0; + info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; + info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); + info->fields[FLD_TYPE_VALUE].num = 0; + info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; + info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilterRange* ra) { + SFilterRangeNode *r = NULL; + + if (ctx->rf) { + r = ctx->rf; + ctx->rf = ctx->rf->next; + r->prev = NULL; + r->next = NULL; + } else { + r = calloc(1, sizeof(SFilterRangeNode)); + } + + FILTER_COPY_RA(&r->ra, ra); + + return r; +} + +void* filterInitRangeCtx(int32_t type, int32_t options) { + if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + qError("not supported range type:%d", type); + return NULL; + } + + SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); + + ctx->type = type; + ctx->options = options; + ctx->pCompareFunc = getComparFunc(type, 0); + + return ctx; +} + + +int32_t filterResetRangeCtx(SFilterRangeCtx *ctx) { + ctx->status = 0; + + if (ctx->rf == NULL) { + ctx->rf = ctx->rs; + ctx->rs = NULL; + return TSDB_CODE_SUCCESS; + } + + ctx->isnull = false; + ctx->notnull = false; + ctx->isrange = false; + + SFilterRangeNode *r = ctx->rf; + + while (r && r->next) { + r = r->next; + } + + r->next = ctx->rs; + ctx->rs = NULL; + return TSDB_CODE_SUCCESS; +} + +int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) { + filterResetRangeCtx(ctx); + + ctx->type = type; + ctx->options = options; + ctx->pCompareFunc = getComparFunc(type, 0); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type)); + if (sr == 0) { + FILTER_SET_FLAG(ra->sflag, RANGE_FLG_NULL); + } + } + + if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type)); + if (er == 0) { + FILTER_SET_FLAG(ra->eflag, RANGE_FLG_NULL); + } + } + + + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL) && FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + *notNull = true; + } else { + *notNull = false; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (optr == TSDB_RELATION_AND) { + SET_AND_OPTR(ctx, raOptr); + if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { + FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); + *empty = true; + } + } else { + SET_OR_OPTR(ctx, raOptr); + if (CHK_OR_OPTR(ctx)) { + FILTER_SET_FLAG(ctx->status, MR_ST_ALL); + *all = true; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (ctx->rs == NULL) { + if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) + || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) + || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { + APPEND_RANGE(ctx, ctx->rs, ra); + FILTER_SET_FLAG(ctx->status, MR_ST_START); + } + + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + int32_t cr = 0; + + if (optr == TSDB_RELATION_AND) { + while (r != NULL) { + cr = ctx->pCompareFunc(&r->ra.s, &ra->e); + if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { + FREE_FROM_RANGE(ctx, r); + break; + } + + cr = ctx->pCompareFunc(&ra->s, &r->ra.e); + if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { + rn = r->next; + FREE_RANGE(ctx, r); + r = rn; + continue; + } + + cr = ctx->pCompareFunc(&ra->s, &r->ra.s); + if (FILTER_GREATER(cr, ra->sflag, r->ra.sflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); + cr == 0 ? (r->ra.sflag |= ra->sflag) : (r->ra.sflag = ra->sflag); + } + + cr = ctx->pCompareFunc(&r->ra.e, &ra->e); + if (FILTER_GREATER(cr, r->ra.eflag, ra->eflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); + cr == 0 ? (r->ra.eflag |= ra->eflag) : (r->ra.eflag = ra->eflag); + break; + } + + r = r->next; + } + + return TSDB_CODE_SUCCESS; + } + + + //TSDB_RELATION_OR + + bool smerged = false; + bool emerged = false; + + while (r != NULL) { + cr = ctx->pCompareFunc(&r->ra.s, &ra->e); + if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { + if (emerged == false) { + INSERT_RANGE(ctx, r, ra); + } + + break; + } + + if (smerged == false) { + cr = ctx->pCompareFunc(&ra->s, &r->ra.e); + if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { + if (r->next) { + r= r->next; + continue; + } + + APPEND_RANGE(ctx, r, ra); + break; + } + + cr = ctx->pCompareFunc(&r->ra.s, &ra->s); + if (FILTER_GREATER(cr, r->ra.sflag, ra->sflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); + cr == 0 ? (r->ra.sflag &= ra->sflag) : (r->ra.sflag = ra->sflag); + } + + smerged = true; + } + + if (emerged == false) { + cr = ctx->pCompareFunc(&ra->e, &r->ra.e); + if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); + if (cr == 0) { + r->ra.eflag &= ra->eflag; + break; + } + + r->ra.eflag = ra->eflag; + emerged = true; + r = r->next; + continue; + } + + break; + } + + cr = ctx->pCompareFunc(&ra->e, &r->ra.e); + if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { + rn = r->next; + FREE_RANGE(ctx, r); + r = rn; + + continue; + } else { + SIMPLE_COPY_VALUES(&r->prev->ra.e, (char *)&r->ra.e); + cr == 0 ? (r->prev->ra.eflag &= r->ra.eflag) : (r->prev->ra.eflag = r->ra.eflag); + FREE_RANGE(ctx, r); + + break; + } + } + + if (ctx->rs && ctx->rs->next == NULL) { + bool notnull; + filterConvertRange(ctx, &ctx->rs->ra, ¬null); + if (notnull) { + bool all = false; + FREE_FROM_RANGE(ctx, ctx->rs); + filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); + if (all) { + FILTER_SET_FLAG(ctx->status, MR_ST_ALL); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type)); + //FILTER_CLR_FLAG(ra->sflag, RA_NULL); + } + + if (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type)); + //FILTER_CLR_FLAG(ra->eflag, RA_NULL); + } + + return filterAddRangeImpl(h, ra, optr); +} + + +int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { + SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; + SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; + + assert(optr == TSDB_RELATION_OR); + + if (sctx->rs == NULL) { + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = sctx->rs; + + while (r) { + filterAddRange(dctx, &r->ra, optr); + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterCopyRangeCtx(void *dst, void *src) { + SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; + SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; + + dctx->status = sctx->status; + + dctx->isnull = sctx->isnull; + dctx->notnull = sctx->notnull; + dctx->isrange = sctx->isrange; + + SFilterRangeNode *r = sctx->rs; + SFilterRangeNode *dr = dctx->rs; + + while (r) { + APPEND_RANGE(dctx, dr, &r->ra); + if (dr == NULL) { + dr = dctx->rs; + } else { + dr = dr->next; + } + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterFinishRange(void* h) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (FILTER_GET_FLAG(ctx->status, MR_ST_FIN)) { + return TSDB_CODE_SUCCESS; + } + + if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + + while (r && r->next) { + int64_t tmp = 1; + operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); + if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { + rn = r->next; + SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); + FREE_RANGE(ctx, r); + r = rn; + + continue; + } + + r = r->next; + } + } + + FILTER_SET_FLAG(ctx->status, MR_ST_FIN); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGetRangeNum(void* h, int32_t* num) { + filterFinishRange(h); + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + *num = 0; + + SFilterRangeNode *r = ctx->rs; + + while (r) { + ++(*num); + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterGetRangeRes(void* h, SFilterRange *ra) { + filterFinishRange(h); + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + uint32_t num = 0; + SFilterRangeNode* r = ctx->rs; + + while (r) { + FILTER_COPY_RA(ra, &r->ra); + + ++num; + r = r->next; + ++ra; + } + + if (num == 0) { + qError("no range result"); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, bool *empty, bool *all) { + SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; + + if (src->isnull){ + filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + if (src->notnull) { + filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + if (src->isrange) { + filterAddRangeOptr(ctx, 0, optr, empty, all); + + if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { + filterAddRangeCtx(ctx, src, optr); + } + + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterFreeRangeCtx(void* h) { + if (h == NULL) { + return TSDB_CODE_SUCCESS; + } + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + + while (r) { + rn = r->next; + free(r); + r = rn; + } + + r = ctx->rf; + while (r) { + rn = r->next; + free(r); + r = rn; + } + + free(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group) { + SFilterGroup gp = {0}; + + gp.unitNum = gp1->unitNum + gp2->unitNum; + gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); + memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); + memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); + + gp.unitFlags = NULL; + + taosArrayPush(group, &gp); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { + int32_t leftSize = (int32_t)taosArrayGetSize(left); + int32_t rightSize = (int32_t)taosArrayGetSize(right); + + CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + + for (int32_t l = 0; l < leftSize; ++l) { + SFilterGroup *gp1 = taosArrayGet(left, l); + + for (int32_t r = 0; r < rightSize; ++r) { + SFilterGroup *gp2 = taosArrayGet(right, r); + + filterDetachCnfGroup(gp1, gp2, group); + } + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { + for (uint16_t i = 0; i < fields->num; ++i) { + if (0 == gDescCompare[type](fields->fields[i].desc, v)) { + return i; + } + } + + return -1; +} + + +int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { + if (type == FLD_TYPE_VALUE) { + if (info->pctx.valHash == false) { + qError("value hash is empty"); + return -1; + } + + void *hv = taosHashGet(info->pctx.valHash, v, dataLen); + if (hv) { + return *(int32_t *)hv; + } + } + + return -1; +} + + +int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { + int32_t idx = -1; + uint16_t *num; + + num = &info->fields[type].num; + + if (*num > 0) { + if (type == FLD_TYPE_COLUMN) { + idx = filterGetFiledByDesc(&info->fields[type], type, desc); + } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + idx = filterGetFiledByData(info, type, *data, dataLen); + } + } + + if (idx < 0) { + idx = *num; + if (idx >= info->fields[type].size) { + info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; + info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); + } + + info->fields[type].fields[idx].flag = type; + info->fields[type].fields[idx].desc = desc; + info->fields[type].fields[idx].data = data ? *data : NULL; + + if (type == FLD_TYPE_COLUMN) { + FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE); + } + + ++(*num); + + if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (info->pctx.valHash == NULL) { + info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + } + + taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); + } + } else { + if (freeIfExists) { + tfree(desc); + } + + if (data && freeIfExists) { + tfree(*data); + } + } + + fid->type = type; + fid->idx = idx; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { + filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); + + FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); + FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { + CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); + CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR); + + int32_t type; + void *v; + + if (node->nodeType == TSQL_NODE_COL) { + type = FLD_TYPE_COLUMN; + v = node->pSchema; + node->pSchema = NULL; + } else { + type = FLD_TYPE_VALUE; + v = node->pVal; + node->pVal = NULL; + } + + filterAddField(info, v, NULL, type, fid, 0, true); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { + if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (info->pctx.unitHash == NULL) { + info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); + } else { + int64_t v = 0; + FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); + void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); + if (hu) { + *uidx = *(uint16_t *)hu; + return TSDB_CODE_SUCCESS; + } + } + } + + if (info->unitNum >= info->unitSize) { + uint16_t psize = info->unitSize; + info->unitSize += FILTER_DEFAULT_UNIT_SIZE; + info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); + memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); + } + + SFilterUnit *u = &info->units[info->unitNum]; + + u->compare.optr = optr; + u->left = *left; + if (right) { + u->right = *right; + } + + if (u->right.type == FLD_TYPE_VALUE) { + SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); + assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); + } else { + assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); + } + + SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); + assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); + + info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); + + *uidx = info->unitNum; + + if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + int64_t v = 0; + FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); + taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); + } + + ++info->unitNum; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { + if (group->unitNum >= group->unitSize) { + group->unitSize += FILTER_DEFAULT_UNIT_SIZE; + group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); + } + + group->unitIdxs[group->unitNum++] = unitIdx; + + return TSDB_CODE_SUCCESS; +} + +int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { + SBufferReader br = tbufInitReader(buf, len, false); + uint32_t sType = tbufReadUint32(&br); + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); + int32_t code = 0; + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); + + int dummy = -1; + tVariant tmpVar = {0}; + size_t t = 0; + int32_t sz = tbufReadInt32(&br); + void *pvar = NULL; + int64_t val = 0; + int32_t bufLen = 0; + if (IS_NUMERIC_TYPE(sType)) { + bufLen = 60; // The maximum length of string that a number is converted to. + } else { + bufLen = 128; + } + + char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + + for (int32_t i = 0; i < sz; i++) { + switch (sType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + *(double *)&val = tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(float *)&val = (float)tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_BINARY: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + default: + taosHashCleanup(pObj); + *q = NULL; + assert(0); + } + + tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); + + if (bufLen < t) { + tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + bufLen = (int32_t)t; + } + + bool converted = false; + char extInfo = 0; + + switch (tType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (tVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_BINARY: { + if (tVariantDump(&tmpVar, tmp, tType, true)) { + goto _return; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + if (tVariantDump(&tmpVar, tmp, tType, true)) { + goto _return; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + default: + goto _return; + } + + taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + } + + *q = (void *)pObj; + pObj = NULL; + +_return: + tVariantDestroy(&tmpVar); + taosHashCleanup(pObj); + tfree(tmp); + + return code; +} + + + +int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { + SFilterFieldId left = {0}, right = {0}; + + filterAddFieldFromNode(info, tree->_node.pLeft, &left); + + tVariant* var = tree->_node.pRight->pVal; + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + int32_t len = 0; + uint16_t uidx = 0; + + if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { + void *data = NULL; + filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); + CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + if (taosHashGetSize((SHashObj *)data) <= 0) { + filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + taosHashCleanup(data); + + return TSDB_CODE_SUCCESS; + } + + void *p = taosHashIterate((SHashObj *)data, NULL); + while(p) { + void *key = taosHashGetDataKey((SHashObj *)data, p); + void *fdata = NULL; + + if (IS_VAR_DATA_TYPE(type)) { + len = (int32_t)taosHashGetDataKeyLen((SHashObj *)data, p); + fdata = malloc(len + VARSTR_HEADER_SIZE); + varDataLen(fdata) = len; + memcpy(varDataVal(fdata), key, len); + len += VARSTR_HEADER_SIZE; + } else { + fdata = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(fdata, key); + len = tDataTypes[type].bytes; + } + + filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); + + filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + + p = taosHashIterate((SHashObj *)data, p); + } + + taosHashCleanup(data); + } else { + filterAddFieldFromNode(info, tree->_node.pRight, &right); + + filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { + SFilterFieldId left, right, *pright = &right; + int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint16_t flag = FLD_DESC_NO_FREE; + + filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); + SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); + FILTER_SET_FLAG(t->flag, flag); + + if (u->right.type == FLD_TYPE_VALUE) { + void *data = FILTER_UNIT_VAL_DATA(src, u); + if (IS_VAR_DATA_TYPE(type)) { + if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); + + t = FILTER_GET_FIELD(dst, right); + + FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); + } else { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); + } + } else { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); + } + + flag = FLD_DATA_NO_FREE; + t = FILTER_UNIT_RIGHT_FIELD(src, u); + FILTER_SET_FLAG(t->flag, flag); + } else { + pright = NULL; + } + + return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); +} + +int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { + SFilterUnit *u = &info->units[uidx]; + + u->compare.optr2 = optr; + u->right2 = *right; + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { + SFilterFieldId left, right, right2; + uint16_t uidx = 0; + + SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); + + filterAddColFieldFromField(dst, col, &left); + + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); + + if (optr == TSDB_RELATION_AND) { + if (ctx->isnull) { + assert(ctx->notnull == false && ctx->isrange == false); + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + + if (ctx->notnull) { + assert(ctx->isnull == false && ctx->isrange == false); + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + + if (!ctx->isrange) { + assert(ctx->isnull || ctx->notnull); + return TSDB_CODE_SUCCESS; + } + + assert(ctx->rs && ctx->rs->next == NULL); + + SFilterRange *ra = &ctx->rs->ra; + + assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); + + if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { + __compar_fn_t func = getComparFunc(type, 0); + if (func(&ra->s, &ra->e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } else { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + void *data2 = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data2, &ra->e); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + } + + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->e); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + return TSDB_CODE_SUCCESS; + } + + // OR PROCESS + + SFilterGroup ng = {0}; + g = &ng; + + assert(ctx->isnull || ctx->notnull || ctx->isrange); + + if (ctx->isnull) { + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + taosArrayPush(res, g); + } + + if (ctx->notnull) { + assert(!ctx->isrange); + memset(g, 0, sizeof(*g)); + + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + taosArrayPush(res, g); + } + + if (!ctx->isrange) { + assert(ctx->isnull || ctx->notnull); + g->unitNum = 0; + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = ctx->rs; + + while (r) { + memset(g, 0, sizeof(*g)); + + if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { + __compar_fn_t func = getComparFunc(type, 0); + if (func(&r->ra.s, &r->ra.e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } else { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + void *data2 = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data2, &r->ra.e); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnitToGroup(g, uidx); + } + + taosArrayPush(res, g); + + r = r->next; + + continue; + } + + if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.e); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + assert (g->unitNum > 0); + + taosArrayPush(res, g); + + r = r->next; + } + + g->unitNum = 0; + + return TSDB_CODE_SUCCESS; +} + + +static void filterFreeGroup(void *pItem) { + if (pItem == NULL) { + return; + } + + SFilterGroup* p = (SFilterGroup*) pItem; + tfree(p->unitIdxs); + tfree(p->unitFlags); +} + + +int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* leftGroup = NULL; + SArray* rightGroup = NULL; + + if (tree->nodeType != TSQL_NODE_EXPR) { + qError("invalid nodeType:%d", tree->nodeType); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (tree->_node.optr == TSDB_RELATION_AND) { + leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); + rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); + ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); + ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); + + ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); + + taosArrayDestroyEx(leftGroup, filterFreeGroup); + taosArrayDestroyEx(rightGroup, filterFreeGroup); + + return TSDB_CODE_SUCCESS; + } + + if (tree->_node.optr == TSDB_RELATION_OR) { + ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); + ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); + + return TSDB_CODE_SUCCESS; + } + + code = filterAddGroupUnitFromNode(info, tree, group); + + +_return: + + taosArrayDestroyEx(leftGroup, filterFreeGroup); + taosArrayDestroyEx(rightGroup, filterFreeGroup); + + return code; +} + +#if 0 +int32_t filterInitUnitFunc(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit* unit = &info->units[i]; + + info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + } + + return TSDB_CODE_SUCCESS; +} +#endif + + +void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { + if (qDebugFlag & DEBUG_DEBUG) { + CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); + + if (options == 0) { + qDebug("%s - FilterInfo:", msg); + qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema *sch = field->desc; + qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + } + + qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); + for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { + SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; + if (field->desc) { + tVariant *var = field->desc; + if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { + qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); + } else { + qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + } + } else if (field->data) { + qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO + } + } + + qDebug("UNIT Num:%u", info->unitNum); + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + int32_t type = FILTER_UNIT_DATA_TYPE(unit); + int32_t len = 0; + int32_t tlen = 0; + char str[256] = {0}; + + SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); + SSchema *sch = left->desc; + len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); + } else { + strcat(str, "NULL"); + } + strcat(str, "]"); + + qDebug("%s", str); //TODO + } + + qDebug("GROUP Num:%u", info->groupNum); + for (uint16_t i = 0; i < info->groupNum; ++i) { + SFilterGroup *group = &info->groups[i]; + qDebug("Group%d : unit num[%u]", i, group->unitNum); + + for (uint16_t u = 0; u < group->unitNum; ++u) { + qDebug("unit id:%u", group->unitIdxs[u]); + } + } + + return; + } + + qDebug("%s - RANGE info:", msg); + + qDebug("RANGE Num:%u", info->colRangeNum); + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx *ctx = info->colRange[i]; + qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + if (ctx->isrange) { + SFilterRangeNode *r = ctx->rs; + while (r) { + char str[256] = {0}; + int32_t tlen = 0; + if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + strcat(str,"(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + strcat(str, " - "); + if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + strcat(str, "(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + qDebug("range: %s", str); + + r = r->next; + } + } + } + } +} + +void filterFreeColInfo(void *data) { + SFilterColInfo* info = (SFilterColInfo *)data; + + if (info->info == NULL) { + return; + } + + if (info->type == RANGE_TYPE_VAR_HASH) { + //TODO + } else if (info->type == RANGE_TYPE_MR_CTX) { + filterFreeRangeCtx(info->info); + } else if (info->type == RANGE_TYPE_UNIT) { + taosArrayDestroy((SArray *)info->info); + } + + //NO NEED TO FREE UNIT + + info->type = 0; + info->info = NULL; +} + +void filterFreeColCtx(void *data) { + SFilterColCtx* ctx = (SFilterColCtx *)data; + + if (ctx->ctx) { + filterFreeRangeCtx(ctx->ctx); + } +} + + +void filterFreeGroupCtx(SFilterGroupCtx* gRes) { + if (gRes == NULL) { + return; + } + + tfree(gRes->colIdx); + + int16_t i = 0, j = 0; + + while (i < gRes->colNum) { + if (gRes->colInfo[j].info) { + filterFreeColInfo(&gRes->colInfo[j]); + ++i; + } + + ++j; + } + + tfree(gRes->colInfo); + tfree(gRes); +} + +void filterFreeField(SFilterField* field, int32_t type) { + if (field == NULL) { + return; + } + + if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { + if (type == FLD_TYPE_VALUE) { + tVariantDestroy(field->desc); + } + + tfree(field->desc); + } + + if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { + if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { + taosHashCleanup(field->data); + } else { + tfree(field->data); + } + } +} + +void filterFreePCtx(SFilterPCtx *pctx) { + taosHashCleanup(pctx->valHash); + taosHashCleanup(pctx->unitHash); +} + +void filterFreeInfo(SFilterInfo *info) { + CHK_RETV(info == NULL); + + tfree(info->cunits); + + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { + for (uint16_t f = 0; f < info->fields[i].num; ++f) { + filterFreeField(&info->fields[i].fields[f], i); + } + + tfree(info->fields[i].fields); + } + + for (int32_t i = 0; i < info->groupNum; ++i) { + filterFreeGroup(&info->groups[i]); + } + + tfree(info->groups); + + tfree(info->units); + + tfree(info->unitRes); + + tfree(info->unitFlags); + + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + filterFreeRangeCtx(info->colRange[i]); + } + + tfree(info->colRange); + + filterFreePCtx(&info->pctx); + + if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { + tfree(info); + } +} + +int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { + assert(extInfo > 0 || extInfo < 0); + + uint8_t optr = FILTER_UNIT_OPTR(unit); + switch (optr) { + case TSDB_RELATION_GREATER: + case TSDB_RELATION_GREATER_EQUAL: + unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; + break; + case TSDB_RELATION_LESS: + case TSDB_RELATION_LESS_EQUAL: + unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; + break; + case TSDB_RELATION_EQUAL: + unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; + break; + default: + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterInitValFieldData(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit* unit = &info->units[i]; + if (unit->right.type != FLD_TYPE_VALUE) { + assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + continue; + } + + SFilterField* right = FILTER_UNIT_RIGHT_FIELD(info, unit); + + assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); + + uint32_t type = FILTER_UNIT_DATA_TYPE(unit); + SFilterField* fi = right; + + tVariant* var = fi->desc; + + if (var == NULL) { + assert(fi->data != NULL); + continue; + } + + if (unit->compare.optr == TSDB_RELATION_IN) { + filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); + CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); + + continue; + } + + if (type == TSDB_DATA_TYPE_BINARY) { + size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + } else { + if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE + fi->data = calloc(var->nLen, tDataTypes[type].bytes); + for (int32_t a = 0; a < var->nLen; ++a) { + int64_t *v = taosArrayGet(var->arr, a); + assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); + } + + continue; + } else { + fi->data = calloc(1, sizeof(int64_t)); + } + } + + bool converted = false; + char extInfo = 0; + if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { + if (converted) { + filterHandleValueExtInfo(unit, extInfo); + + continue; + } + qError("dump value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + return TSDB_CODE_SUCCESS; +} + + +bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { + int32_t ret = func(left, right); + + switch (optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_GREATER_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_GREATER: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + case TSDB_RELATION_IN: { + return ret == 1; + } + + default: + assert(false); + } + + return true; +} + + +int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *ctx, int32_t optr) { + int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint8_t uoptr = FILTER_UNIT_OPTR(u); + void *val = FILTER_UNIT_VAL_DATA(info, u); + SFilterRange ra = {0}; + int64_t tmp = 0; + + switch (uoptr) { + case TSDB_RELATION_GREATER: + SIMPLE_COPY_VALUES(&ra.s, val); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_GREATER_EQUAL: + SIMPLE_COPY_VALUES(&ra.s, val); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_LESS: + SIMPLE_COPY_VALUES(&ra.e, val); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_LESS_EQUAL: + SIMPLE_COPY_VALUES(&ra.e, val); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_NOT_EQUAL: + assert(type == TSDB_DATA_TYPE_BOOL); + if (GET_INT8_VAL(val)) { + SIMPLE_COPY_VALUES(&ra.s, &tmp); + SIMPLE_COPY_VALUES(&ra.e, &tmp); + } else { + *(bool *)&tmp = true; + SIMPLE_COPY_VALUES(&ra.s, &tmp); + SIMPLE_COPY_VALUES(&ra.e, &tmp); + } + break; + case TSDB_RELATION_EQUAL: + SIMPLE_COPY_VALUES(&ra.s, val); + SIMPLE_COPY_VALUES(&ra.e, val); + break; + default: + assert(0); + } + + filterAddRange(ctx, &ra, optr); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { + CHK_JMP(ctx1->status != ctx2->status); + CHK_JMP(ctx1->isnull != ctx2->isnull); + CHK_JMP(ctx1->notnull != ctx2->notnull); + CHK_JMP(ctx1->isrange != ctx2->isrange); + + SFilterRangeNode *r1 = ctx1->rs; + SFilterRangeNode *r2 = ctx2->rs; + + while (r1 && r2) { + CHK_JMP(r1->ra.sflag != r2->ra.sflag); + CHK_JMP(r1->ra.eflag != r2->ra.eflag); + CHK_JMP(r1->ra.s != r2->ra.s); + CHK_JMP(r1->ra.e != r2->ra.e); + + r1 = r1->next; + r2 = r2->next; + } + + CHK_JMP(r1 != r2); + + *equal = true; + + return TSDB_CODE_SUCCESS; + +_return: + *equal = false; + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { + SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; + int32_t size = (int32_t)taosArrayGetSize(colArray); + int32_t type = gRes->colInfo[colIdx].dataType; + SFilterRangeCtx* ctx = filterInitRangeCtx(type, 0); + + for (uint32_t i = 0; i < size; ++i) { + SFilterUnit* u = taosArrayGetP(colArray, i); + uint8_t optr = FILTER_UNIT_OPTR(u); + + filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); + CHK_JMP(*empty); + + if (!FILTER_NO_MERGE_OPTR(optr)) { + filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); + CHK_JMP(MR_EMPTY_RES(ctx)); + } + } + + taosArrayDestroy(colArray); + + FILTER_PUSH_CTX(gRes->colInfo[colIdx], ctx); + + return TSDB_CODE_SUCCESS; + +_return: + + *empty = true; + + filterFreeRangeCtx(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { + bool empty = false; + uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); + uint16_t colIdxi = 0; + uint16_t gResIdx = 0; + + for (uint16_t i = 0; i < info->groupNum; ++i) { + SFilterGroup* g = info->groups + i; + + gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); + gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); + colIdxi = 0; + empty = false; + + for (uint16_t j = 0; j < g->unitNum; ++j) { + SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); + uint16_t cidx = FILTER_UNIT_COL_IDX(u); + + if (gRes[gResIdx]->colInfo[cidx].info == NULL) { + gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); + colIdx[colIdxi++] = cidx; + ++gRes[gResIdx]->colNum; + } else { + if (!FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))) { + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + } + } + + FILTER_PUSH_UNIT(gRes[gResIdx]->colInfo[cidx], u); + } + + if (colIdxi > 1) { + qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + } + + for (uint16_t l = 0; l < colIdxi; ++l) { + int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; + + if (FILTER_NO_MERGE_DATA_TYPE(type)) { + continue; + } + + filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty); + + if (empty) { + break; + } + } + + if (empty) { + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + filterFreeGroupCtx(gRes[gResIdx]); + gRes[gResIdx] = NULL; + + continue; + } + + gRes[gResIdx]->colNum = colIdxi; + FILTER_COPY_IDX(&gRes[gResIdx]->colIdx, colIdx, colIdxi); + ++gResIdx; + } + + tfree(colIdx); + + *gResNum = gResIdx; + + if (gResIdx == 0) { + FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); + } + + return TSDB_CODE_SUCCESS; +} + +void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { + uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + bool equal = false; + + for (; m < gRes1->colNum; ++m) { + idx1 = gRes1->colIdx[m]; + + equal = false; + + for (; n < gRes2->colNum; ++n) { + idx2 = gRes2->colIdx[n]; + if (idx1 < idx2) { + *conflict = true; + return; + } + + if (idx1 > idx2) { + continue; + } + + if (FILTER_NO_MERGE_DATA_TYPE(gRes1->colInfo[idx1].dataType)) { + *conflict = true; + return; + } + + ++n; + equal = true; + break; + } + + if (!equal) { + *conflict = true; + return; + } + } + + *conflict = false; + return; +} + + +int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { + SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + + if ((*ctx) == NULL) { + *ctx = filterInitRangeCtx(type, 0); + } else { + filterReuseRangeCtx(*ctx, type, 0); + } + + assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + + filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); + filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilterGroupCtx** gRes2, bool *all) { + bool conflict = false; + + filterCheckColConflict(*gRes1, *gRes2, &conflict); + if (conflict) { + return TSDB_CODE_SUCCESS; + } + + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + + uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; + bool equal = false; + uint16_t equal1 = 0, equal2 = 0, merNum = 0; + SFilterRangeCtx *ctx = NULL; + SFilterColCtx colCtx = {0}; + SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); + + for (; m < (*gRes1)->colNum; ++m) { + idx1 = (*gRes1)->colIdx[m]; + + for (; n < (*gRes2)->colNum; ++n) { + idx2 = (*gRes2)->colIdx[n]; + + if (idx1 > idx2) { + continue; + } + + assert(idx1 == idx2); + + ++merNum; + + filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); + + CHK_JMP(*all); + + if (numEqual) { + if ((*gRes1)->colNum == 1) { + ++equal1; + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + taosArrayPush(colCtxs, &colCtx); + break; + } else { + filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); + if (equal) { + ++equal1; + } + + filterCompareRangeCtx(ctx, (*gRes2)->colInfo[idx2].info, &equal); + if (equal) { + ++equal2; + } + + CHK_JMP(equal1 != merNum && equal2 != merNum); + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + ctx = NULL; + taosArrayPush(colCtxs, &colCtx); + } + } else { + filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); + if (equal) { + ++equal1; + } + + CHK_JMP(equal1 != merNum); + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + ctx = NULL; + taosArrayPush(colCtxs, &colCtx); + } + + ++n; + break; + } + } + + assert(merNum > 0); + + SFilterColInfo *colInfo = NULL; + assert (merNum == equal1 || merNum == equal2); + + filterFreeGroupCtx(*gRes2); + *gRes2 = NULL; + + assert(colCtxs && taosArrayGetSize(colCtxs) > 0); + + int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); + SFilterColCtx *pctx = NULL; + + for (int32_t i = 0; i < ctxSize; ++i) { + pctx = taosArrayGet(colCtxs, i); + colInfo = &(*gRes1)->colInfo[pctx->colIdx]; + + filterFreeColInfo(colInfo); + FILTER_PUSH_CTX((*gRes1)->colInfo[pctx->colIdx], pctx->ctx); + } + + taosArrayDestroy(colCtxs); + + return TSDB_CODE_SUCCESS; + +_return: + + if (colCtxs) { + if (taosArrayGetSize(colCtxs) > 0) { + taosArrayDestroyEx(colCtxs, filterFreeColCtx); + } else { + taosArrayDestroy(colCtxs); + } + } + + filterFreeRangeCtx(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gResNum) { + if (*gResNum <= 1) { + return TSDB_CODE_SUCCESS; + } + + qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); + + int32_t pEnd = 0, cStart = 0, cEnd = 0; + uint16_t pColNum = 0, cColNum = 0; + int32_t movedNum = 0; + bool all = false; + + cColNum = gRes[0]->colNum; + + for (int32_t i = 1; i <= *gResNum; ++i) { + if (i < (*gResNum) && gRes[i]->colNum == cColNum) { + continue; + } + + cEnd = i - 1; + + movedNum = 0; + if (pColNum > 0) { + for (int32_t m = 0; m <= pEnd; ++m) { + for (int32_t n = cStart; n <= cEnd; ++n) { + assert(m < n); + filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); + + CHK_JMP(all); + + if (gRes[n] == NULL) { + if (n < ((*gResNum) - 1)) { + memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); + } + + --cEnd; + --(*gResNum); + ++movedNum; + --n; + } + } + } + } + + for (int32_t m = cStart; m < cEnd; ++m) { + for (int32_t n = m + 1; n <= cEnd; ++n) { + assert(m < n); + filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); + + CHK_JMP(all); + + if (gRes[n] == NULL) { + if (n < ((*gResNum) - 1)) { + memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); + } + + --cEnd; + --(*gResNum); + ++movedNum; + --n; + } + } + } + + pColNum = cColNum; + pEnd = cEnd; + + i -= movedNum; + + if (i >= (*gResNum)) { + break; + } + + cStart = i; + cEnd = i; + cColNum = gRes[i]->colNum; + } + + return TSDB_CODE_SUCCESS; + +_return: + + FILTER_SET_FLAG(info->status, FI_STATUS_ALL); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { + size_t groupSize = taosArrayGetSize(group); + + info->groupNum = (uint16_t)groupSize; + + if (info->groupNum > 0) { + info->groups = calloc(info->groupNum, sizeof(*info->groups)); + } + + for (size_t i = 0; i < groupSize; ++i) { + SFilterGroup *pg = taosArrayGet(group, i); + pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); + info->groups[i] = *pg; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { + if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { + qDebug("no need rewrite"); + return TSDB_CODE_SUCCESS; + } + + SFilterInfo oinfo = *info; + + FILTER_SET_FLAG(oinfo.status, FI_STATUS_CLONED); + + SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + SFilterGroupCtx *res = NULL; + SFilterColInfo *colInfo = NULL; + int32_t optr = 0; + uint16_t uidx = 0; + + memset(info, 0, sizeof(*info)); + + info->colRangeNum = oinfo.colRangeNum; + info->colRange = oinfo.colRange; + oinfo.colRangeNum = 0; + oinfo.colRange = NULL; + + FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); + + filterInitUnitsFields(info); + + for (int32_t i = 0; i < gResNum; ++i) { + res = gRes[i]; + + optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + + SFilterGroup ng = {0}; + + for (uint16_t m = 0; m < res->colNum; ++m) { + colInfo = &res->colInfo[res->colIdx[m]]; + if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { + assert(colInfo->type == RANGE_TYPE_UNIT); + int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); + + for (int32_t n = 0; n < usize; ++n) { + SFilterUnit* u = taosArrayGetP((SArray *)colInfo->info, n); + + filterAddUnitFromUnit(info, &oinfo, u, &uidx); + filterAddUnitToGroup(&ng, uidx); + } + + continue; + } + + assert(colInfo->type == RANGE_TYPE_MR_CTX); + + filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); + } + + if (ng.unitNum > 0) { + taosArrayPush(group, &ng); + } + } + + filterConvertGroupFromArray(info, group); + + taosArrayDestroy(group); + + filterFreeInfo(&oinfo); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { + uint16_t *idxs = NULL; + uint16_t colNum = 0; + SFilterGroupCtx *res = NULL; + uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + + for (int32_t i = 0; i < gResNum; ++i) { + for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { + SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; + if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { + continue; + } + + ++idxNum[gRes[i]->colIdx[m]]; + } + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + if (idxNum[i] < gResNum) { + continue; + } + + assert(idxNum[i] == gResNum); + + if (idxs == NULL) { + idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); + } + + idxs[colNum++] = i; + } + + CHK_JMP(colNum <= 0); + + info->colRangeNum = colNum; + info->colRange = calloc(colNum, POINTER_BYTES); + + for (int32_t i = 0; i < gResNum; ++i) { + res = gRes[i]; + uint16_t n = 0; + + for (uint16_t m = 0; m < info->colRangeNum; ++m) { + for (; n < res->colNum; ++n) { + if (res->colIdx[n] < idxs[m]) { + continue; + } + + assert(res->colIdx[n] == idxs[m]); + + SFilterColInfo * colInfo = &res->colInfo[res->colIdx[n]]; + if (info->colRange[m] == NULL) { + info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); + SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); + info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; + } + + assert(colInfo->type == RANGE_TYPE_MR_CTX); + + bool all = false; + filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); + if (all) { + filterFreeRangeCtx(info->colRange[m]); + info->colRange[m] = NULL; + + if (m < (info->colRangeNum - 1)) { + memmove(&info->colRange[m], &info->colRange[m + 1], (info->colRangeNum - m - 1) * POINTER_BYTES); + memmove(&idxs[m], &idxs[m + 1], (info->colRangeNum - m - 1) * sizeof(*idxs)); + } + + --info->colRangeNum; + --m; + + CHK_JMP(info->colRangeNum <= 0); + } + + ++n; + break; + } + } + } + +_return: + tfree(idxNum); + tfree(idxs); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterPostProcessRange(SFilterInfo *info) { + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx* ctx = info->colRange[i]; + SFilterRangeNode *r = ctx->rs; + while (r) { + r->rc.func = filterGetRangeCompFunc(r->ra.sflag, r->ra.eflag); + r = r->next; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterGenerateComInfo(SFilterInfo *info) { + uint16_t n = 0; + + info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); + + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + + info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); + info->cunits[i].optr = FILTER_UNIT_OPTR(unit); + info->cunits[i].colData = NULL; + + if (unit->right.type == FLD_TYPE_VALUE) { + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } else { + info->cunits[i].valData = NULL; + } + if (unit->right2.type == FLD_TYPE_VALUE) { + info->cunits[i].valData2 = FILTER_GET_VAL_FIELD_DATA(FILTER_GET_FIELD(info, unit->right2)); + } else { + info->cunits[i].valData2 = info->cunits[i].valData; + } + + info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); + info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); + } + + uint16_t cgroupNum = info->groupNum + 1; + + for (uint16_t i = 0; i < info->groupNum; ++i) { + cgroupNum += info->groups[i].unitNum; + } + + info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); + + for (uint16_t i = 0; i < info->groupNum; ++i) { + info->cgroups[n++] = info->groups[i].unitNum; + + for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { + info->cgroups[n++] = info->groups[i].unitIdxs[m]; + } + } + + info->cgroups[n] = 0; + + return TSDB_CODE_SUCCESS; +} + +int32_t filterUpdateComUnits(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + + info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); + } + + return TSDB_CODE_SUCCESS; +} + + +static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t* p) { + return true; +} +static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t* p) { + return false; +} +static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t* p) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + p[i] = isNull(colData, info->cunits[uidx].dataType); + if (p[i] == 0) { + all = false; + } + } + + return all; +} +static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t* p) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + p[i] = !isNull(colData, info->cunits[uidx].dataType); + if (p[i] == 0) { + all = false; + } + } + + return all; +} + +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t* p) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + uint16_t dataSize = info->cunits[0].dataSize; + char *colData = (char *)info->cunits[0].colData; + rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; + void *valData = info->cunits[0].valData; + void *valData2 = info->cunits[0].valData2; + __compar_fn_t func = gDataCompare[info->cunits[0].func]; + + for (int32_t i = 0; i < numOfRows; ++i) { + if (isNull(colData, info->cunits[0].dataType)) { + all = false; + colData += dataSize; + continue; + } + + p[i] = (*rfunc)(colData, colData, valData, valData2, func); + + if (p[i] == 0) { + all = false; + } + + colData += dataSize; + } + + return all; +} + +bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t* p) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + if (isNull(colData, info->cunits[uidx].dataType)) { + all = false; + continue; + } + + p[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + + if (p[i] == 0) { + all = false; + } + } + + return all; +} + + +bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t* p) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + //FILTER_UNIT_CLR_F(info); + + for (uint32_t g = 0; g < info->groupNum; ++g) { + SFilterGroup *group = &info->groups[g]; + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + SFilterComUnit *cunit = &info->cunits[uidx]; + void *colData = (char *)cunit->colData + cunit->dataSize * i; + + //if (FILTER_UNIT_GET_F(info, uidx)) { + // p[i] = FILTER_UNIT_GET_R(info, uidx); + //} else { + uint8_t optr = cunit->optr; + + if (isNull(colData, cunit->dataType)) { + p[i] = optr == TSDB_RELATION_ISNULL ? true : false; + } else { + if (optr == TSDB_RELATION_NOTNULL) { + p[i] = 1; + } else if (optr == TSDB_RELATION_ISNULL) { + p[i] = 0; + } else if (cunit->rfunc >= 0) { + p[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + p[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } + + //FILTER_UNIT_SET_R(info, uidx, p[i]); + //FILTER_UNIT_SET_F(info, uidx); + } + + if (p[i] == 0) { + break; + } + } + + if (p[i]) { + break; + } + } + + if (p[i] == 0) { + all = false; + } + } + + return all; +} + +FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p) { + return (*info->func)(info, numOfRows, p); +} + +int32_t filterSetExecFunc(SFilterInfo *info) { + if (FILTER_ALL_RES(info)) { + info->func = filterExecuteImplAll; + return TSDB_CODE_SUCCESS; + } + + if (FILTER_EMPTY_RES(info)) { + info->func = filterExecuteImplEmpty; + return TSDB_CODE_SUCCESS; + } + + if (info->unitNum > 1) { + info->func = filterExecuteImpl; + return TSDB_CODE_SUCCESS; + } + + if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { + info->func = filterExecuteImplIsNull; + return TSDB_CODE_SUCCESS; + } + + if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { + info->func = filterExecuteImplNotNull; + return TSDB_CODE_SUCCESS; + } + + if (info->cunits[0].rfunc >= 0) { + info->func = filterExecuteImplRange; + return TSDB_CODE_SUCCESS; + } + + info->func = filterExecuteImplMisc; + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterPreprocess(SFilterInfo *info) { + SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); + int32_t gResNum = 0; + + filterMergeGroupUnits(info, gRes, &gResNum); + + filterMergeGroups(info, gRes, &gResNum); + + if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { + qInfo("Final - FilterInfo: [ALL]"); + goto _return; + } + + + if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { + qInfo("Final - FilterInfo: [EMPTY]"); + goto _return; + } + + filterGenerateColRange(info, gRes, gResNum); + + filterDumpInfoToString(info, "Final", 1); + + filterPostProcessRange(info); + + filterRewrite(info, gRes, gResNum); + + filterGenerateComInfo(info); + +_return: + + filterSetExecFunc(info); + + for (int32_t i = 0; i < gResNum; ++i) { + filterFreeGroupCtx(gRes[i]); + } + + tfree(gRes); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data) { + CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); + CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); + + if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema* sch = fi->desc; + if (sch->colId == colId) { + fi->data = data; + + break; + } + } + + filterUpdateComUnits(info); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { + int32_t code = TSDB_CODE_SUCCESS; + SFilterInfo *info = NULL; + + CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); + + if (*pinfo == NULL) { + *pinfo = calloc(1, sizeof(SFilterInfo)); + } + + info = *pinfo; + + info->options = options; + + SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + + filterInitUnitsFields(info); + + code = filterTreeToGroup(tree, info, group); + + ERR_JRET(code); + + filterConvertGroupFromArray(info, group); + + ERR_JRET(filterInitValFieldData(info)); + + if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { + filterDumpInfoToString(info, "Before preprocess", 0); + + ERR_JRET(filterPreprocess(info)); + + CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); + + if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { + taosArrayDestroy(group); + return code; + } + + //ERR_JRET(filterInitUnitFunc(info)); + } + + info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); + info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); + + filterDumpInfoToString(info, "Final", 0); + + taosArrayDestroy(group); + + return code; + +_return: + qInfo("No filter, code:%d", code); + + taosArrayDestroy(group); + + filterFreeInfo(*pinfo); + + *pinfo = NULL; + + return code; +} + + + + +bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows) { + if (FILTER_EMPTY_RES(info)) { + return false; + } + + if (FILTER_ALL_RES(info)) { + return true; + } + + bool ret = true; + void *minVal, *maxVal; + + for (int32_t k = 0; k < info->colRangeNum; ++k) { + int32_t index = -1; + SFilterRangeCtx *ctx = info->colRange[k]; + for(int32_t i = 0; i < numOfCols; ++i) { + if (pDataStatis[i].colId == ctx->colId) { + index = i; + break; + } + } + + // no statistics data, load the true data block + if (index == -1) { + return true; + } + + // not support pre-filter operation on binary/nchar data type + if (FILTER_NO_MERGE_DATA_TYPE(ctx->type)) { + return true; + } + + if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { + return false; + } + + // all data in current column are NULL, no need to check its boundary value + if (pDataStatis[index].numOfNull == numOfRows) { + + // if isNULL query exists, load the null data column + if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + return false; + } + + continue; + } + + SDataStatis* pDataBlockst = &pDataStatis[index]; + + SFilterRangeNode *r = ctx->rs; + + if (ctx->type == TSDB_DATA_TYPE_FLOAT) { + float minv = (float)(*(double *)(&pDataBlockst->min)); + float maxv = (float)(*(double *)(&pDataBlockst->max)); + + minVal = &minv; + maxVal = &maxv; + } else { + minVal = &pDataBlockst->min; + maxVal = &pDataBlockst->max; + } + + while (r) { + ret = r->rc.func(minVal, maxVal, &r->rc.s, &r->rc.e, ctx->pCompareFunc); + if (ret) { + break; + } + r = r->next; + } + + CHK_RET(!ret, ret); + } + + return ret; +} + + + +int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { + SFilterRange ra = {0}; + SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *cur = NULL; + int32_t num = 0; + int32_t optr = 0; + int32_t code = 0; + bool empty = false, all = false; + + for (int32_t i = 0; i < info->groupNum; ++i) { + SFilterGroup *group = &info->groups[i]; + if (group->unitNum > 1) { + cur = tmpc; + optr = TSDB_RELATION_AND; + } else { + cur = prev; + optr = TSDB_RELATION_OR; + } + + for (int32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + SFilterUnit *unit = &info->units[uidx]; + + uint8_t raOptr = FILTER_UNIT_OPTR(unit); + + filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); + CHK_JMP(empty); + + if (FILTER_NO_MERGE_OPTR(raOptr)) { + continue; + } + + SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); + void *s = FILTER_GET_VAL_FIELD_DATA(right); + void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + + SIMPLE_COPY_VALUES(&ra.s, s); + SIMPLE_COPY_VALUES(&ra.e, e); + + filterAddRange(cur, &ra, optr); + } + + if (cur->notnull) { + prev->notnull = true; + break; + } + + if (group->unitNum > 1) { + filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); + filterResetRangeCtx(cur); + if (all) { + break; + } + } + } + + if (prev->notnull) { + *win = TSWINDOW_INITIALIZER; + } else { + filterGetRangeNum(prev, &num); + if (num > 1) { + qError("only one time range accepted, num:%d", num); + ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); + } + + CHK_JMP(num < 1); + + SFilterRange tra; + filterGetRangeRes(prev, &tra); + win->skey = tra.s; + win->ekey = tra.e; + } + + filterFreeRangeCtx(prev); + filterFreeRangeCtx(tmpc); + + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + return TSDB_CODE_SUCCESS; + +_return: + + *win = TSWINDOW_DESC_INITIALIZER; + + filterFreeRangeCtx(prev); + filterFreeRangeCtx(tmpc); + + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + + return code; +} + + +int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + if (type == TSDB_DATA_TYPE_NCHAR) { + SFilterField nfi = {0}; + nfi.desc = fi->desc; + int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); + nfi.data = malloc(rows * bytes); + int32_t bufSize = bytes - VARSTR_HEADER_SIZE; + for (int32_t j = 0; j < rows; ++j) { + char *src = FILTER_GET_COL_FIELD_DATA(fi, j); + char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); + int32_t len = 0; + taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + varDataLen(dst) = len; + } + + fi->data = nfi.data; + + *gotNchar = true; + } + } + + if (*gotNchar) { + filterUpdateComUnits(info); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterFreeNcharColumns(SFilterInfo* info) { + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + if (type == TSDB_DATA_TYPE_NCHAR) { + tfree(fi->data); + } + } + + return TSDB_CODE_SUCCESS; +} + + + + + diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index e93553c4df..011892fe93 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -423,6 +423,9 @@ tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) { pExpr->pRight = tSqlExprClone(pSrc->pRight); } + memset(&pExpr->value, 0, sizeof(pExpr->value)); + tVariantAssign(&pExpr->value, &pSrc->value); + //we don't clone paramList now because clone is only used for between/and assert(pSrc->Expr.paramList == NULL); return pExpr; @@ -478,9 +481,7 @@ static void doDestroySqlExprNode(tSqlExpr *pExpr) { return; } - if (pExpr->tokenId == TK_STRING) { - tVariantDestroy(&pExpr->value); - } + tVariantDestroy(&pExpr->value); tSqlExprListDestroy(pExpr->Expr.paramList); free(pExpr); diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 7d30f7c668..403b51426f 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -103,6 +103,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi } } + if (param.colCond != NULL) { + if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _over; @@ -162,13 +168,19 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi assert(pQueryMsg->stableQuery == isSTableQuery); (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, vgId, param.sql, qId, param.pUdfInfo); + param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); param.sql = NULL; param.pExprs = NULL; param.pSecExprs = NULL; param.pGroupbyExpr = NULL; param.pTagColumnInfo = NULL; + param.pFilters = NULL; + + if ((*pQInfo) == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _over; + } param.pUdfInfo = NULL; code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); @@ -178,6 +190,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi taosArrayDestroy(param.pGroupbyExpr->columnInfo); } + tfree(param.colCond); + destroyUdfInfo(param.pUdfInfo); taosArrayDestroy(param.pTableIdList); @@ -190,6 +204,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); } + filterFreeInfo(param.pFilters); + //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; if (code != TSDB_CODE_SUCCESS) { *pQInfo = NULL; diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index cc4b607bb4..349d511f15 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -27,3 +27,4 @@ SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w) diff --git a/src/query/tests/rangeMergeTest.cpp b/src/query/tests/rangeMergeTest.cpp new file mode 100644 index 0000000000..e65508a300 --- /dev/null +++ b/src/query/tests/rangeMergeTest.cpp @@ -0,0 +1,367 @@ +#include +#include + +#include "qResultbuf.h" +#include "taos.h" +#include "taosdef.h" + +#include "qFilter.h" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" + +extern "C" { + extern void* filterInitRangeCtx(int32_t type, int32_t options); + extern int32_t filterGetRangeNum(void* h, int32_t* num); + extern int32_t filterGetRangeRes(void* h, SFilterRange *ra); + extern int32_t filterFreeRangeCtx(void* h); + extern int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr); +} + +namespace { + + +void intDataTest() { + printf("running %s\n", __FUNCTION__); + int32_t asize = 0; + SFilterRange ra[10] = {0}; + int64_t *s =NULL; + int64_t *e =NULL; + int64_t s0[3] = {-100, 1, 3}; + int64_t e0[3] = {0 , 2, 4}; + int64_t s1[3] = {INT64_MIN, 0 , 3}; + int64_t e1[3] = {100 , 50, 4}; + int64_t s2[5] = {1 , 3 , 10,30,70}; + int64_t e2[5] = {10, 100, 20,50,120}; + int64_t s3[3] = {1 , 20 , 5}; + int64_t e3[3] = {10, 100, 25}; + int64_t s4[2] = {10, 0}; + int64_t e4[2] = {20, 5}; + int64_t s5[3] = {0, 6 ,7}; + int64_t e5[3] = {4, 10,20}; + + int64_t rs[10]; + int64_t re[10]; + + int32_t num = 0; + void *h = NULL; + + s = s0; + e = e0; + asize = sizeof(s0)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + filterAddRange(h, ra, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 3); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, -100); + ASSERT_EQ(ra[0].e, 0); + ASSERT_EQ(ra[1].s, 1); + ASSERT_EQ(ra[1].e, 2); + ASSERT_EQ(ra[2].s, 3); + ASSERT_EQ(ra[2].e, 4); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, FI_OPTION_TIMESTAMP); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, -100); + ASSERT_EQ(ra[0].e, 4); + filterFreeRangeCtx(h); + + + s = s1; + e = e1; + asize = sizeof(s1)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 3); + ASSERT_EQ(ra[0].e, 4); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, INT64_MIN); + ASSERT_EQ(ra[0].e, 100); + filterFreeRangeCtx(h); + + + + s = s2; + e = e2; + asize = sizeof(s2)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 1); + ASSERT_EQ(ra[0].e, 120); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, i % 2 ? TSDB_RELATION_OR : TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, i % 2 ? TSDB_RELATION_AND : TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 70); + ASSERT_EQ(ra[0].e, 120); + filterFreeRangeCtx(h); + + + s = s3; + e = e3; + asize = sizeof(s3)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 1); + ASSERT_EQ(ra[0].e, 100); + filterFreeRangeCtx(h); + + + + + s = s4; + e = e4; + asize = sizeof(s4)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 2); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 0); + ASSERT_EQ(ra[0].e, 5); + ASSERT_EQ(ra[1].s, 10); + ASSERT_EQ(ra[1].e, 20); + filterFreeRangeCtx(h); + + + s = s5; + e = e5; + asize = sizeof(s5)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 2); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 0); + ASSERT_EQ(ra[0].e, 4); + ASSERT_EQ(ra[1].s, 6); + ASSERT_EQ(ra[1].e, 20); + filterFreeRangeCtx(h); + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].s = s[i]; + ra[0].e = e[i]; + + filterAddRange(h, ra, (i == (asize -1)) ? TSDB_RELATION_AND : TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 1); + filterGetRangeRes(h, ra); + ASSERT_EQ(ra[0].s, 7); + ASSERT_EQ(ra[0].e, 10); + filterFreeRangeCtx(h); + + + + int64_t s6[2] = {0, 4}; + int64_t e6[2] = {4, 6}; + s = s6; + e = e6; + asize = sizeof(s6)/sizeof(s[0]); + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].eflag = 1; + ra[1].sflag = 4; + + ra[i].s = s[i]; + ra[i].e = e[i]; + + filterAddRange(h, ra + i, TSDB_RELATION_AND); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 0); + filterFreeRangeCtx(h); + + + + memset(ra, 0, sizeof(ra)); + h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0); + for (int32_t i = 0; i < asize; ++i) { + ra[0].eflag = 1; + ra[1].sflag = 1; + + ra[i].s = s[i]; + ra[i].e = e[i]; + + filterAddRange(h, ra + i, TSDB_RELATION_OR); + } + filterGetRangeNum(h, &num); + ASSERT_EQ(num, 2); + ASSERT_EQ(ra[0].s, 0); + ASSERT_EQ(ra[0].e, 4); + ASSERT_EQ(ra[0].eflag, 1); + ASSERT_EQ(ra[1].s, 4); + ASSERT_EQ(ra[1].e, 6); + ASSERT_EQ(ra[1].sflag, 1); + filterFreeRangeCtx(h); + +} + + +} // namespace + +TEST(testCase, rangeMergeTest) { + intDataTest(); + +} diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 6c98283189..8f5f885d69 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -722,7 +722,7 @@ static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { return -1; } - pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pCommith->pDataCols == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyCommitH(pCommith); @@ -920,7 +920,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - // if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } @@ -1277,6 +1276,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt if (key1 < key2) { for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } @@ -1308,6 +1308,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(!isRowDel); for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 62f9e41119..5ccb9e90f2 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -296,7 +296,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { return -1; } - pComph->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pComph->pDataCols == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyCompactH(pComph); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 8bb2d1c44e..e766d97a97 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -702,11 +702,12 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { } //row1 has higher priority -static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, STSchema **ppSchema1, STSchema **ppSchema2, STable* pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) { +static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, + STSchema **ppSchema1, STSchema **ppSchema2, + STable* pTable, int32_t* pPoints, SMemRow* pLastRow) { //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) { - (*pAffectedRows)++; (*pPoints)++; return NULL; } @@ -715,7 +716,6 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1)); if(pMem == NULL) return NULL; memRowCpy(pMem, row1); - (*pAffectedRows)++; (*pPoints)++; *pLastRow = pMem; return pMem; @@ -750,18 +750,16 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep if(pMem == NULL) return NULL; memRowCpy(pMem, tmp); - (*pAffectedRows)++; (*pPoints)++; - *pLastRow = pMem; return pMem; } static void* tsdbInsertDupKeyMergePacked(void** args) { - return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7], args[8]); + return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]); } -static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) { +static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) { if(pSkipList->insertHandleFn == NULL) { tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9); @@ -769,17 +767,16 @@ static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STa dupHandleSavedFunc->args[3] = NULL; dupHandleSavedFunc->args[4] = NULL; dupHandleSavedFunc->args[5] = pTable; - dupHandleSavedFunc->args[6] = pAffectedRows; - dupHandleSavedFunc->args[7] = pPoints; - dupHandleSavedFunc->args[8] = pLastRow; pSkipList->insertHandleFn = dupHandleSavedFunc; } + pSkipList->insertHandleFn->args[6] = pPoints; + pSkipList->insertHandleFn->args[7] = pLastRow; } static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) { STsdbMeta *pMeta = pRepo->tsdbMeta; - int64_t points = 0; + int32_t points = 0; STable *pTable = NULL; SSubmitBlkIter blkIter = {0}; SMemTable *pMemTable = NULL; @@ -830,9 +827,10 @@ static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t * SMemRow lastRow = NULL; int64_t osize = SL_SIZE(pTableData->pData); - tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, pAffectedRows, &points, &lastRow); + tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow); tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext); int64_t dsize = SL_SIZE(pTableData->pData) - osize; + (*pAffectedRows) += points; if(lastRow != NULL) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 619b32b3d9..21150c66e2 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -17,7 +17,6 @@ #define TSDB_SUPER_TABLE_SL_LEVEL 5 #define DEFAULT_TAG_INDEX_COLUMN 0 -static int tsdbCompareSchemaVersion(const void *key1, const void *key2); static char * getTagIndexKey(const void *pData); static STable *tsdbNewTable(); static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c578555df2..716f82d154 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -466,7 +466,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); - pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); + pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); if (pQueryHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1446,7 +1446,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { +static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; @@ -1481,7 +1481,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; } - if (pColInfo->info.colId == src->colId) { + if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) { if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { memmove(pData, (char*)src->pData + bytes * start, bytes * num); } else { // handle the var-string diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index 711c32535b..74d41cce19 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -42,14 +42,14 @@ int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) { return -1; } - pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pReadh->pDCols[0] == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyReadH(pReadh); return -1; } - pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pReadh->pDCols[1] == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyReadH(pReadh); @@ -463,7 +463,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat SDataCol *pDataCol = &(pDataCols->cols[dcol]); if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dataColReset(pDataCol); dcol++; continue; } @@ -503,7 +503,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat ccol++; } else { // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dataColReset(pDataCol); dcol++; } } @@ -608,7 +608,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * } if (pBlockCol == NULL) { - dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dataColReset(pDataCol); continue; } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 6c4145810b..7a93745dc8 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -42,7 +42,7 @@ typedef struct SHashNode { #define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) #define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); +#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, @@ -170,6 +170,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p); void taosHashCancelIterate(SHashObj *pHashObj, void *p); +void *taosHashGetDataKey(SHashObj *pHashObj, void *data); + +uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data); + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index fe46f00086..cf61b7165a 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -53,6 +53,38 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr); int32_t taosArrayCompareString(const void* a, const void* b); +int32_t setCompareBytes1(const void *pLeft, const void *pRight); + +int32_t setCompareBytes2(const void *pLeft, const void *pRight); + +int32_t setCompareBytes4(const void *pLeft, const void *pRight); +int32_t setCompareBytes8(const void *pLeft, const void *pRight); + +int32_t compareInt32Val(const void *pLeft, const void *pRight); +int32_t compareInt64Val(const void *pLeft, const void *pRight); + +int32_t compareInt16Val(const void *pLeft, const void *pRight); + +int32_t compareInt8Val(const void *pLeft, const void *pRight); + +int32_t compareUint32Val(const void *pLeft, const void *pRight); +int32_t compareUint64Val(const void *pLeft, const void *pRight); + +int32_t compareUint16Val(const void *pLeft, const void *pRight); + +int32_t compareUint8Val(const void* pLeft, const void* pRight); + +int32_t compareFloatVal(const void *pLeft, const void *pRight); + +int32_t compareDoubleVal(const void *pLeft, const void *pRight); + +int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); + +int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); +int32_t compareStrPatternComp(const void* pLeft, const void* pRight); +int32_t compareFindItemInSet(const void *pLeft, const void* pRight); +int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 6118aa7bef..a22ce34a0e 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -776,6 +776,17 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } +FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { + SHashNode * node = GET_HASH_PNODE(data); + return GET_HASH_NODE_KEY(node); +} + +FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { + SHashNode * node = GET_HASH_PNODE(data); + return node->keyLen; +} + + // release the pNode, return next pNode, and lock the current entry static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index e953f4c464..7577451f88 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -19,6 +19,22 @@ #include "tarray.h" #include "hash.h" +int32_t setCompareBytes1(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; +} + +int32_t setCompareBytes2(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; +} + +int32_t setCompareBytes4(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; +} + +int32_t setCompareBytes8(const void *pLeft, const void *pRight) { + return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; +} + int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); if (left > right) return 1; @@ -48,21 +64,21 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) { } int32_t compareUint32Val(const void *pLeft, const void *pRight) { - int32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); + uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; } int32_t compareUint64Val(const void *pLeft, const void *pRight) { - int64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); + uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; } int32_t compareUint16Val(const void *pLeft, const void *pRight) { - int16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); + uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; @@ -262,7 +278,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { +int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; char pattern[128] = {0}; @@ -290,11 +306,11 @@ int32_t taosArrayCompareString(const void* a, const void* b) { // const SArray* arr = (const SArray*) pRight; // return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1; //} -static int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { +int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } -static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { +int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; wchar_t pattern[128] = {0}; @@ -309,6 +325,29 @@ static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; + + if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setCompareBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setCompareBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setCompareBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setCompareBytes8; + default: + assert(0); + } + } switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -334,6 +373,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_NCHAR: { if (optr == TSDB_RELATION_LIKE) { comparFn = compareWStrPatternComp; + } else if (optr == TSDB_RELATION_IN) { + comparFn = compareFindItemInSet; } else { comparFn = compareLenPrefixedWStr; } diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 42fc76e6c9..49e46cdde8 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -280,6 +280,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION, "One valid time range condition expected") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") diff --git a/tests/pytest/query/queryBetweenAnd.py b/tests/pytest/query/queryBetweenAnd.py index cd4320f523..3d038b3d2f 100644 --- a/tests/pytest/query/queryBetweenAnd.py +++ b/tests/pytest/query/queryBetweenAnd.py @@ -101,7 +101,7 @@ class TDTestCase: # tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}") # tdSql.checkRows(1) tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}") - tdSql.checkRows(0) + tdSql.checkRows(2) tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}") # tdSql.checkRows(3) @@ -203,4 +203,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryWildcardLength.py b/tests/pytest/query/queryWildcardLength.py new file mode 100644 index 0000000000..d15085f751 --- /dev/null +++ b/tests/pytest/query/queryWildcardLength.py @@ -0,0 +1,217 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from copy import deepcopy +import string +import random +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def cleanTb(self): + query_sql = "show stables" + res_row_list = tdSql.query(query_sql, True) + stb_list = map(lambda x: x[0], res_row_list) + for stb in stb_list: + tdSql.execute(f'drop table if exists {stb}') + + query_sql = "show tables" + res_row_list = tdSql.query(query_sql, True) + tb_list = map(lambda x: x[0], res_row_list) + for tb in tb_list: + tdSql.execute(f'drop table if exists {tb}') + + def getLongWildcardStr(self, len=None): + """ + generate long wildcard str + """ + maxWildCardsLength = int(tdSql.getVariable('maxWildCardsLength')[0]) + if len: + chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len)) + else: + chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(maxWildCardsLength+1)) + return chars + + def genTableName(self): + ''' + generate table name + hp_name--->'%str' + lp_name--->'str%' + ul_name--->'st_r' + ''' + table_name = self.getLongWildcardStr() + table_name_list = list(table_name) + table_name_list.pop(-1) + + if len(table_name_list) > 1: + lp_name = deepcopy(table_name_list) + lp_name[-1] = '%' + lp_name = ''.join(lp_name) + + ul_name = list(lp_name) + ul_name[int(len(ul_name)/2)] = '_' + ul_name = ''.join(ul_name) + + table_name_list = list(table_name) + hp_name = deepcopy(table_name_list) + hp_name.pop(1) + hp_name[0] = '%' + hp_name = ''.join(hp_name) + else: + hp_name = '%' + lp_name = '%' + ul_name = '_' + return table_name, hp_name, lp_name, ul_name + + def checkRegularTableWildcardLength(self): + ''' + check regular table wildcard length with % and _ + ''' + self.cleanTb() + table_name, hp_name, lp_name, ul_name = self.genTableName() + tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, a1 int)") + sql_list = [f'show tables like "{hp_name}"', f'show tables like "{lp_name}"', f'show tables like "{ul_name}"'] + for sql in sql_list: + tdSql.query(sql) + if len(table_name) >= 1: + tdSql.checkRows(1) + else: + tdSql.error(sql) + + exceed_sql_list = [f'show tables like "%{hp_name}"', f'show tables like "{lp_name}%"', f'show tables like "{ul_name}%"'] + for sql in exceed_sql_list: + tdSql.error(sql) + + def checkSuperTableWildcardLength(self): + ''' + check super table wildcard length with % and _ + ''' + self.cleanTb() + table_name, hp_name, lp_name, ul_name = self.genTableName() + tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, c1 int) tags (t1 int)") + sql_list = [f'show stables like "{hp_name}"', f'show stables like "{lp_name}"', f'show stables like "{ul_name}"'] + for sql in sql_list: + tdSql.query(sql) + if len(table_name) >= 1: + tdSql.checkRows(1) + else: + tdSql.error(sql) + + exceed_sql_list = [f'show stables like "%{hp_name}"', f'show stables like "{lp_name}%"', f'show stables like "{ul_name}%"'] + for sql in exceed_sql_list: + tdSql.error(sql) + + def checkRegularWildcardSelectLength(self): + ''' + check regular table wildcard select length with % and _ + ''' + self.cleanTb() + table_name, hp_name, lp_name, ul_name = self.genTableName() + tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, bi1 binary(200), nc1 nchar(200))") + tdSql.execute(f'insert into {table_name} values (now, "{table_name}", "{table_name}")') + sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"', + f'select * from {table_name} where bi1 like "{lp_name}"', + f'select * from {table_name} where bi1 like "{ul_name}"', + f'select * from {table_name} where nc1 like "{hp_name}"', + f'select * from {table_name} where nc1 like "{lp_name}"', + f'select * from {table_name} where nc1 like "{ul_name}"'] + for sql in sql_list: + tdSql.query(sql) + if len(table_name) >= 1: + tdSql.checkRows(1) + else: + tdSql.error(sql) + + exceed_sql_list = [f'select * from {table_name} where bi1 like "%{hp_name}"', + f'select * from {table_name} where bi1 like "{lp_name}%"', + f'select * from {table_name} where bi1 like "{ul_name}%"', + f'select * from {table_name} where nc1 like "%{hp_name}"', + f'select * from {table_name} where nc1 like "{lp_name}%"', + f'select * from {table_name} where nc1 like "{ul_name}%"'] + for sql in exceed_sql_list: + tdSql.error(sql) + + def checkStbWildcardSelectLength(self): + ''' + check stb wildcard select length with % and _ + ''' + self.cleanTb() + table_name, hp_name, lp_name, ul_name = self.genTableName() + + tdSql.execute(f'CREATE TABLE {table_name} (ts timestamp, bi1 binary(200), nc1 nchar(200)) tags (si1 binary(200), sc1 nchar(200))') + tdSql.execute(f'create table {table_name}_sub1 using {table_name} tags ("{table_name}", "{table_name}")') + tdSql.execute(f'insert into {table_name}_sub1 values (now, "{table_name}", "{table_name}");') + + # TODO sc1 leave a bug ---> TD-5918 + # sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"', + # f'select * from {table_name} where bi1 like "{lp_name}"', + # f'select * from {table_name} where bi1 like "{ul_name}"', + # f'select * from {table_name} where nc1 like "{hp_name}"', + # f'select * from {table_name} where nc1 like "{lp_name}"', + # f'select * from {table_name} where nc1 like "{ul_name}"', + # f'select * from {table_name} where si1 like "{hp_name}"', + # f'select * from {table_name} where si1 like "{lp_name}"', + # f'select * from {table_name} where si1 like "{ul_name}"', + # f'select * from {table_name} where sc1 like "{hp_name}"', + # f'select * from {table_name} where sc1 like "{lp_name}"', + # f'select * from {table_name} where sc1 like "{ul_name}"'] + sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"', + f'select * from {table_name} where bi1 like "{lp_name}"', + f'select * from {table_name} where bi1 like "{ul_name}"', + f'select * from {table_name} where nc1 like "{hp_name}"', + f'select * from {table_name} where nc1 like "{lp_name}"', + f'select * from {table_name} where nc1 like "{ul_name}"', + f'select * from {table_name} where si1 like "{hp_name}"', + f'select * from {table_name} where si1 like "{lp_name}"', + f'select * from {table_name} where si1 like "{ul_name}"'] + for sql in sql_list: + tdSql.query(sql) + if len(table_name) >= 1: + tdSql.checkRows(1) + else: + tdSql.error(sql) + exceed_sql_list = [f'select * from {table_name} where bi1 like "%{hp_name}"', + f'select * from {table_name} where bi1 like "{lp_name}%"', + f'select * from {table_name} where bi1 like "{ul_name}%"', + f'select * from {table_name} where nc1 like "%{hp_name}"', + f'select * from {table_name} where nc1 like "{lp_name}%"', + f'select * from {table_name} where nc1 like "{ul_name}%"', + f'select * from {table_name} where si1 like "%{hp_name}"', + f'select * from {table_name} where si1 like "{lp_name}%"', + f'select * from {table_name} where si1 like "{ul_name}%"', + f'select * from {table_name} where sc1 like "%{hp_name}"', + f'select * from {table_name} where sc1 like "{lp_name}%"', + f'select * from {table_name} where sc1 like "{ul_name}%"'] + for sql in exceed_sql_list: + tdSql.error(sql) + + def run(self): + tdSql.prepare() + self.checkRegularTableWildcardLength() + self.checkSuperTableWildcardLength() + self.checkRegularWildcardSelectLength() + self.checkStbWildcardSelectLength() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b42af27d06..dfe1e4a582 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -81,6 +81,22 @@ class TDSql: return self.queryResult return self.queryRows + def getVariable(self, search_attr): + ''' + get variable of search_attr access "show variables" + ''' + try: + sql = 'show variables' + param_list = self.query(sql, row_tag=True) + for param in param_list: + if param[0] == search_attr: + return param[1], param_list + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + def getColNameList(self, sql, col_tag=None): self.sql = sql try: diff --git a/tests/script/general/parser/condition.sim b/tests/script/general/parser/condition.sim new file mode 100644 index 0000000000..56706467f1 --- /dev/null +++ b/tests/script/general/parser/condition.sim @@ -0,0 +1,111 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) + +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) + +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) + +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sleep 100 + +sql connect + +run general/parser/condition_query.sim + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 100 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 100 + +run general/parser/condition_query.sim + diff --git a/tests/script/general/parser/condition_query.sim b/tests/script/general/parser/condition_query.sim new file mode 100644 index 0000000000..7600e510d3 --- /dev/null +++ b/tests/script/general/parser/condition_query.sim @@ -0,0 +1,2628 @@ + +sql use cdb; + +print "column test" +sql select * from stb1 +if $rows != 29 then + return -1 +endi +sql select * from stb1 where c1 > 0 +if $rows != 28 then + return -1 +endi + +sql_error select * from stb1 where c8 > 0 +sql_error select * from stb1 where c7 in (0,2,3,1); +sql_error select * from stb1 where c8 in (true); +sql_error select * from stb1 where c8 in (1,2); +sql_error select * from stb1 where t2 in (3.0); +sql_error select ts,c1,c7 from stb1 where c7 > false +sql_error select * from stb1 where c1 > NULL; +sql_error select * from stb1 where c1 = NULL; +sql_error select * from stb1 where c1 LIKE '%1'; +sql_error select * from stb1 where c2 LIKE '%1'; +sql_error select * from stb1 where c3 LIKE '%1'; +sql_error select * from stb1 where c4 LIKE '%1'; +sql_error select * from stb1 where c5 LIKE '%1'; +sql_error select * from stb1 where c6 LIKE '%1'; +sql_error select * from stb1 where c7 LIKE '%1'; +sql_error select * from stb1 where c1 = 'NULL'; +sql_error select * from stb1 where c2 > 'NULL'; +sql_error select * from stb1 where c3 <> 'NULL'; +sql_error select * from stb1 where c4 != 'null'; +sql_error select * from stb1 where c5 >= 'null'; +sql_error select * from stb1 where c6 <= 'null'; +sql_error select * from stb1 where c7 < 'nuLl'; +sql_error select * from stb1 where c8 < 'nuLl'; +sql_error select * from stb1 where c9 > 'nuLl'; +sql_error select * from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b; +sql_error select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50 or b.c1 < 60; +sql_error select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and ((a.c1 > 50 and a.c1 < 60) or (b.c2 > 60)); +sql_error select * from stb1 where 'c2' is null; +sql_error select * from stb1 where 'c2' is not null; + +sql select * from stb1 where c2 > 3.0 or c2 < 60; +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c2 > 3.0 or c2 < 60 and c2 > 50; +if $rows != 25 then + return -1 +endi +sql select * from stb1 where (c2 > 3.0 or c2 < 60) and c2 > 50; +if $rows != 8 then + return -1 +endi + +sql select * from stb1 where (c2 > 3.0 or c2 < 60) and c2 > 50 and (c2 != 53 and c2 != 63); +if $rows != 6 then + return -1 +endi + +sql select * from stb1 where (c2 > 3.0 or c2 < 60) and c2 > 50 and (c2 != 53 or c2 != 63); +if $rows != 8 then + return -1 +endi + +sql select * from stb1 where (c3 > 3.0 or c3 < 60) and c3 > 50 and (c3 != 53 or c3 != 63); +if $rows != 8 then + return -1 +endi + +sql select * from stb1 where (c4 > 3.0 or c4 < 60) and c4 > 50 and (c4 != 53 or c4 != 63); +if $rows != 8 then + return -1 +endi + +sql select * from stb1 where (c5 > 3.0 or c5 < 60) and c5 > 50 and (c5 != 53 or c5 != 63); +if $rows != 8 then + return -1 +endi + +sql select * from stb1 where (c6 > 3.0 or c6 < 60) and c6 > 50 and (c6 != 53 or c6 != 63); +if $rows != 8 then + return -1 +endi + +sql select * from stb1 where c8 = '51'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi + +sql select * from stb1 where c8 != '51'; +if $rows != 27 then + return -1 +endi + +#xxx +sql select * from stb1 where c8 = '51' and c8 != '51'; +if $rows != 0 then + return -1 +endi + +#xxx +sql select * from stb1 where c8 = '51' or c8 != '51'; +if $rows != 28 then + return -1 +endi + +sql select * from stb1 where c9 = '51'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi + +sql select * from stb1 where c9 != '51'; +if $rows != 27 then + return -1 +endi + +sql select * from stb1 where c9 = '51' and c9 != '51'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c9 = '51' or c9 != '51'; +if $rows != 28 then + return -1 +endi + +sql select ts,c1,c7 from stb1 where c7 = false +if $rows != 14 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data01 != 3 then + return -1 +endi +if $data02 != 0 then + return -1 +endi +if $data10 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data11 != 4 then + return -1 +endi +if $data12 != 0 then + return -1 +endi +if $data20 != @21-05-05 18:19:06.000@ then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data22 != 0 then + return -1 +endi +if $data30 != @21-05-05 18:19:07.000@ then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data32 != 0 then + return -1 +endi + + +sql select ts,c1,c7 from stb1 where c7 = true +if $rows != 14 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data11 != 2 then + return -1 +endi +if $data12 != 1 then + return -1 +endi +if $data20 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data21 != 11 then + return -1 +endi +if $data22 != 1 then + return -1 +endi +if $data30 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data31 != 12 then + return -1 +endi +if $data32 != 1 then + return -1 +endi + + +sql select * from stb1 where c8 = '51' or c8 = '4' +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data01 != 4 then + return -1 +endi +if $data10 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data11 != 51 then + return -1 +endi + + +sql select * from stb1 where c1 > 50 and c1 > 53 +if $rows != 5 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 +if $rows != 8 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 and c1 < 52 +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 or c1 < 51 +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 or c1 < 51 +if $rows != 25 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 and c1 < 51 +if $rows != 8 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 and c1 > 51 and c1 > 54 +if $rows != 4 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 and c1 > 51 or c1 > 54 +if $rows != 5 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 and c1 < 51 or c1 > 54 +if $rows != 4 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 or c1 < 51 and c1 > 54 +if $rows != 5 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 or c1 > 51 and c1 < 54 +if $rows != 7 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 and c1 < 51 and c1 > 54 +if $rows != 8 then + return -1 +endi +sql select * from stb1 where c1 > 50 and c1 > 53 or c1 < 51 or c1 > 54 +if $rows != 25 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 and c1 < 51 or c1 > 54 +if $rows != 8 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 or c1 < 51 and c1 > 54 +if $rows != 8 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 or c1 > 51 and c1 > 54 +if $rows != 8 then + return -1 +endi +sql select * from stb1 where c1 > 50 or c1 > 53 or c1 < 51 or c1 > 54 +if $rows != 28 then + return -1 +endi + +sql select * from stb1 where (c1 > 50 and c1 > 53) and c1 < 52 +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 and c1 < 52) +if $rows != 0 then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53) or c1 < 51 +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 or c1 < 51) +if $rows != 28 then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53) or c1 < 51 +if $rows != 25 then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 or c1 < 51) +if $rows != 5 then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53) and c1 < 51 +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 and c1 < 51) +if $rows != 8 then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53) and (c1 < 51 and c1 > 54) +if $rows != 0 then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53 and c1 < 51) and c1 > 54 +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 and c1 < 51) and c1 > 54 +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 and c1 < 51 or c1 > 54) +if $rows != 4 then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53) or (c1 < 51 and c1 > 54) +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 or c1 < 51) and c1 > 54 +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53 or c1 < 51) and c1 > 54 +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 or c1 < 51 and c1 > 54) +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53) and (c1 < 51 and c1 > 54) +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 and c1 < 51 and c1 > 54) +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53 and c1 < 51) and c1 > 54 +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 and c1 < 51) and c1 > 54 +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53) or (c1 < 51 or c1 > 54) +if $rows != 25 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 or c1 < 51 or c1 > 54) +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 and c1 > 53 or c1 < 51) or c1 > 54 +if $rows != 25 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 and (c1 > 53 or c1 < 51) or c1 > 54 +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53) and (c1 < 51 or c1 > 54) +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 and c1 < 51 or c1 > 54) +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53 and c1 < 51) or c1 > 54 +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 and c1 < 51) or c1 > 54 +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53) or (c1 < 51 and c1 > 54) +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53 or c1 < 51) and c1 > 54 +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 or c1 < 51 and c1 > 54) +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 or c1 < 51) and c1 > 54 +if $rows != 8 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where c1 > 62 or (c1 > 53 or c1 < 51) and c1 > 54 +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53) or (c1 < 51 or c1 > 54) +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 or c1 < 51 or c1 > 54) +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 50 or c1 > 53 or c1 < 51) or c1 > 54 +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +sql select * from stb1 where c1 > 50 or (c1 > 53 or c1 < 51) or c1 > 54 +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +sql select ts,c1 from stb1 where (c1 > 60 or c1 < 10 or (c1 > 20 and c1 < 30)) and ts > '2021-05-05 18:19:00.000' and ts < '2021-05-05 18:19:25.000' and c1 != 21 and c1 != 22 +if $rows != 6 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data20 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data21 != 4 then + return -1 +endi +if $data30 != @21-05-05 18:19:10.000@ then + return -1 +endi +if $data31 != 23 then + return -1 +endi +if $data40 != @21-05-05 18:19:11.000@ then + return -1 +endi +if $data41 != 24 then + return -1 +endi +if $data50 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data51 != 61 then + return -1 +endi + + +sql select * from stb1 where (c1 > 40 or c1 < 20) and (c2 < 53 or c2 >= 63) and c3 > 1 and c3 < 5 +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data20 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data21 != 4 then + return -1 +endi + +sql select * from stb1 where (c1 > 52 or c1 < 10) and (c2 > 1 and c2 < 61) +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data20 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data21 != 4 then + return -1 +endi +if $data30 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data31 != 53 then + return -1 +endi +if $data40 != @21-05-05 18:19:23.000@ then + return -1 +endi +if $data41 != 54 then + return -1 +endi + +sql select * from stb1 where (c3 > 52 or c3 < 10) and (c4 > 1 and c4 < 61) and (c5 = 2 or c6 = 3.0 or c6 = 4.0 or c6 = 53); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data20 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data21 != 4 then + return -1 +endi +if $data30 != @21-05-05 18:19:22.000@ then + return -1 +endi +if $data31 != 53 then + return -1 +endi + +sql select * from stb1 where c1 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c2 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c3 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c4 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c5 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c6 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c7 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c8 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi + +#xxx +sql select * from stb1 where c8 like '1'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi + +#xxx +sql select * from stb1 where c8 like '1%' and c8 like '%1'; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:04.000@ then + return -1 +endi + +#xxx +sql select * from stb1 where c8 like '1' and c8 like '2'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c9 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi + +sql select * from stb1 where c1 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c2 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c3 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c4 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c5 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c6 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c7 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c8 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c9 is not null; +if $rows != 28 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from stb1 where c1 > 63 or c1 is null; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:27.000@ then + return -1 +endi +if $data01 != 64 then + return -1 +endi +if $data10 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data11 != NULL then + return -1 +endi +sql select * from stb1 where c1 is null and c2 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +sql select * from stb1 where c1 is null and c2 is null and c3 is not null; +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 is null and c2 is null and ts > '2021-05-05 18:19:00.000' and ts < '2021-05-05 18:19:28.000'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c1 is null and c1 > 0; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c1 is null or c1 is not null or c1 > 1; +if $rows != 29 then + return -1 +endi + +sql select * from stb1 where (c1 is null or c1 > 40) and c1 < 44; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:16.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:17.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:18.000@ then + return -1 +endi + +sql select * from stb1 where c1 in (11,21,31,41) and c1 in (11,42); +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi + +sql select * from stb1 where c8 in ('11','21','31','41') and c8 in ('11','42'); +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi + +sql select * from stb1 where (c1 > 60 and c2 > 40) or (c1 > 62 and c2 > 50); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where c1 = 3 or c1 = 5 or c1 >= 44 and c1 <= 52; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:19.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:21.000@ then + return -1 +endi + +sql select * from stb1 where c8 LIKE '%1'; +if $rows != 7 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:08.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:12.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:16.000@ then + return -1 +endi +if $data50 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data60 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where c9 LIKE '%1'; +if $rows != 7 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:08.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:12.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:16.000@ then + return -1 +endi +if $data50 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data60 != @21-05-05 18:19:24.000@ then + return -1 +endi +sql select * from stb1 where (c8 LIKE '%1' or c9 like '_2') and (c5 > 50 or c6 > 30) and ( c8 like '3_' or c9 like '4_') and (c4 <= 31 or c4 >= 42); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:12.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:17.000@ then + return -1 +endi + +sql select * from stb1 where c1 in (1,3); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi + +sql select * from stb1 where c3 in (11,22); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:09.000@ then + return -1 +endi + +sql select * from stb1 where c4 in (3,33); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:14.000@ then + return -1 +endi + +sql select * from stb1 where c5 in (3,33) and c8 in ('22','55'); +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c5 in (3,33) and c8 in ('33','54'); +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:14.000@ then + return -1 +endi + +sql select * from stb1 where c5 in (3,33) or c8 in ('22','54'); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:09.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:14.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:23.000@ then + return -1 +endi + +sql select * from stb1 where (c9 in ('3','1','2','4','5') or c9 in ('33','11','22','44','55')) and c9 in ('1','3','11','13'); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:04.000@ then + return -1 +endi + +sql select * from stb2 where (u1 in (1) or u2 in (5,6)) and (u3 in (3,6) or u4 in (7,8)) and ts2 in ('2021-05-05 18:28:02.000','2021-05-05 18:28:15.000','2021-05-05 18:28:01.000'); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi + +sql select * from stb2 where u2 in (2) and u3 in (1,2,3) and u4 in (1,2,4,5) and u1 > 3 and u1 < 6 and u1 != 4; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:08.000@ then + return -1 +endi + +sql select avg(c1) from tb1 where (c1 > 12 or c2 > 10) and (c3 < 12 or c3 > 13); +if $rows != 1 then + return -1 +endi +if $data00 != 12.500000000 then + return -1 +endi + +sql select count(c1),sum(c3) from tb1 where ((c7 = true and c6 > 2) or (c1 > 10 or c3 < 3)) and ((c8 like '1%') or (c9 like '%2' or c9 like '%3')) interval(5s); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data01 != 3 then + return -1 +endi +if $data02 != 14 then + return -1 +endi +if $data10 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data12 != 39 then + return -1 +endi + +sql select * from stb1 where c8 = 'null'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c8 = 'NULL'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c9 = 'null'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c9 = 'NULL'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c2 in (0,1); +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +sql select * from stb1 where c6 in (0,2,3,1); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +sql select ts,c1 from (select * from stb1 where (c1 > 60 or c1 < 10) and (c7 = true or c5 > 2 and c5 < 63)) where (c3 > 61 or c3 < 3); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:25.000@ then + return -1 +endi + +#sql select a.* from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50; +sql select a.ts from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:25.000@ then + return -1 +endi + +#sql select a.ts,a.c1,a.c8,a.c9 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50 and b.c1 < 60; +sql select a.ts,a.c1,a.c8 from (select * from stb1 where c7=true) a, (select * from stb1 where c1 > 30) b where a.ts=b.ts and a.c1 > 50 and b.c1 < 60; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:20.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:21.000@ then + return -1 +endi + +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and (a.c1 < 10 or a.c1 > 30) and (b.u1 < 5 or b.u1 > 5); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:12.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:14.000@ then + return -1 +endi + +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and a.c1 < 30 and b.u1 > 1 and a.c1 > 10 and b.u1 < 8 and b.u1<>5; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:06.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:10.000@ then + return -1 +endi + +sql select * from stb1 where c1 is null and c1 is not null; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c1 is null or c1 is not null; +if $rows != 29 then + return -1 +endi +sql select * from stb1 where c1 is null or c1 > 20 or c1 < 25; +if $rows != 29 then + return -1 +endi +sql select * from stb1 where (c1 > 20 or c1 < 25) and c1 is null; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where (c1 > 20 or c1 < 25) and (c1 > 62 or c1 < 3); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 11 and c1 != 11 and c1 != 14 and c1 < 14; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:06.000@ then + return -1 +endi +sql select * from stb1 where (c1 > 60 or c1 < 4 or c1 > 10 and c1 < 20 and c1 != 13 or c1 < 2 or c1 > 50) +if $rows != 14 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:04.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 62 or c1 >= 62; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 62 and c1 >= 62; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where c1 >= 62 and c1 != 62; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where c1 >= 62 or c1 != 62; +if $rows != 28 then + return -1 +endi + +sql select * from stb1 where c1 >= 62 and c1 = 62; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:25.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 62 and c1 != 62; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 62 and c1 = 62; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c1 is not null and c1 is not null; +if $rows != 28 then + return -1 +endi + +sql select * from stb1 where c1 is not null or c1 is not null; +if $rows != 28 then + return -1 +endi + +sql select * from stb1 where c1 is null and c1 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi + +sql select * from stb1 where c1 is null or c1 is null; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:28.000@ then + return -1 +endi + +sql select * from stb1 where c2 > 3 and c2 < 3; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c2 = 3; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi + +sql select * from stb1 where c2 > 3 and c2 <= 3; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where c2 >= 3 and c2 <= 3; +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi + +sql select * from stb1 where (c2 in (1,2,3,4) or c2 in (11,12,13,14)) and c2 != 11 and c2 >2 and c2 != 14; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:06.000@ then + return -1 +endi + +sql select * from stb1 where (c1 > 60 or c1 < 4 or c1 > 10 and c1 < 20 and c1 != 13 or c1 < 2 or c1 > 50) and (c1 != 51 and c1 <= 54 and c1 != 54 and c1 >=1 and c1 != 1) and (c1 >= 11 and c1 <=52 and c1 != 52 and c1 != 11); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:07.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 1 and c1 is not null and c1 < 5; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:03.000@ then + return -1 +endi + +sql select * from (select * from stb1 where c2 > 10 and c6 < 40) where c9 in ('11','21','31'); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:08.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:12.000@ then + return -1 +endi + +sql select * from stb1 where c1 > 40 and c2 > 50 and c3 > 62 or c1 < 2 and c2 < 3; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:27.000@ then + return -1 +endi + +sql select * from stb1 where (c1 > 3 and c2 > 4) or (c1 < 60 and c2 < 30); +if $rows != 28 then + return -1 +endi + +sql select * from stb1 where (c1 > 3 and c2 > 4) or (c1 < 60 and c2 < 30) or (c1 is null and c2 is null); +if $rows != 29 then + return -1 +endi + +sql select * from stb1 where (c1 > 3 and c2 > 4) or (c1 < 60 and c3 < 30) or (c1 is null and c2 is null); +if $rows != 29 then + return -1 +endi + +sql select * from stb1 where (c1 > 60 and c2 < 63) or (c1 >62 and c3 < 30) or (c1 is null and c2 is null); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:28.000@ then + return -1 +endi + +sql select * from stb1 where c1 between 60 and 9999999999; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +sql select * from stb1 where c1 > 9999999999; +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 < 9999999999; +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c1 = 9999999999; +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c1 <> 9999999999; +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c4 < -9999999999; +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c4 > -9999999999; +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c4 = -9999999999; +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c4 <> -9999999999; +if $rows != 28 then + return -1 +endi +sql select * from stb1 where c5 in (-9999999999); +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c5 in (9999999999); +if $rows != 0 then + return -1 +endi +sql select * from stb1 where c5 in (-9999999999,3,4,9999999999); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:03.000@ then + return -1 +endi + +sql select * from stb3 where c1 > 3 and c1 < 2; +if $rows != 0 then + return -1 +endi + +sql select * from stb3 where c1 is null order by ts; +if $rows != 4 then + return -1 +endi +if $data00 != @21-06-05 18:19:28.000@ then + return -1 +endi +if $data10 != @21-06-06 18:19:28.000@ then + return -1 +endi +if $data20 != @21-07-05 18:19:28.000@ then + return -1 +endi +if $data30 != @21-07-06 18:19:28.000@ then + return -1 +endi + +sql select * from stb3 where c1 is not null order by ts; +if $rows != 10 then + return -1 +endi +if $data00 != @21-01-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-01-06 18:19:00.000@ then + return -1 +endi +if $data20 != @21-02-05 18:19:01.000@ then + return -1 +endi +if $data30 != @21-02-06 18:19:01.000@ then + return -1 +endi +if $data40 != @21-03-05 18:19:02.000@ then + return -1 +endi +if $data50 != @21-03-06 18:19:02.000@ then + return -1 +endi +if $data60 != @21-04-05 18:19:03.000@ then + return -1 +endi +if $data70 != @21-04-06 18:19:03.000@ then + return -1 +endi + + +sql select * from stb3 where c1 > 11; +if $rows != 4 then + return -1 +endi +if $data00 != @21-02-06 18:19:01.000@ then + return -1 +endi +if $data10 != @21-03-06 18:19:02.000@ then + return -1 +endi +if $data20 != @21-04-06 18:19:03.000@ then + return -1 +endi +if $data30 != @21-05-06 18:19:28.000@ then + return -1 +endi + +sql select * from stb3 where c1 is not null or c1 is null; +if $rows != 14 then + return -1 +endi + + + +print "ts test" +sql_error select ts,c1,c7 from stb1 where ts != '2021-05-05 18:19:27' +sql_error select ts,c1,c7 from stb1 where ts > '2021-05-05 18:19:03.000' or ts < '2021-05-05 18:19:02.000'; +sql_error select ts,c1,c7 from stb1 where ts > '2021-05-05 18:19:03.000' and ts > '2021-05-05 18:19:20.000' and ts != '2021-05-05 18:19:22.000'; +sql_error select * from stb1 where ts2 like '2021-05-05%'; +sql_error select ts,c1,c2 from stb1 where (ts > '2021-05-05 18:19:25.000' or ts < '2021-05-05 18:19:05.000') and ts > '2021-05-05 18:19:01.000' and ts < '2021-05-05 18:19:27.000'; +sql_error select ts,c1,c2 from stb1 where (ts > '2021-05-05 18:19:20.000' or ts < '2021-05-05 18:19:05.000') and ts != '2021-05-05 18:19:25.000'; +sql_error select ts,c1,c2 from stb1 where ((ts >= '2021-05-05 18:19:05.000' and ts <= '2021-05-05 18:19:10.000') or (ts >= '2021-05-05 18:19:15.000' and ts <= '2021-05-05 18:19:20.000') or (ts >= '2021-05-05 18:19:11.000' and ts <= '2021-05-05 18:19:14.000')); +sql_error select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' or ts < '2021-05-05 18:19:24.000'; +sql select * from stb1 where ts is null; +if $rows != 0 then + return -1 +endi +sql select * from stb1 where ts is not null and ts is null; +if $rows != 0 then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' and ts < '2021-05-05 18:19:10.000'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where ts > '2021-05-05 18:19:03.000' and ts < '2021-05-05 18:19:02'; +if $rows != 0 then + return -1 +endi + +sql select * from stb1 where ts is not null; +if $rows != 29 then + return -1 +endi + +sql select * from stb1 where ts is not null or ts is null; +if $rows != 29 then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' or ts < '2021-05-05 18:19:25.000'; +if $rows != 29 then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' and ts < '2021-05-05 18:19:26.000'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:25.000@ then + return -1 +endi +sql select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' or ts < '2021-05-05 18:19:28.000'; +if $rows != 29 then + return -1 +endi +sql select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' or ts > '2021-05-05 18:19:27.000'; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:27.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:28.000@ then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ts > '2021-05-05 18:19:20.000' or ts < '2021-05-05 18:19:05.000' or ts != '2021-05-05 18:19:25.000'; +if $rows != 29 then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ts >= '2021-05-05 18:19:25.000' or ts <> '2021-05-05 18:19:25.000'; +if $rows != 29 then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ((ts >= '2021-05-05 18:19:05.000' and ts <= '2021-05-05 18:19:10.999') or (ts >= '2021-05-05 18:19:15.000' and ts <= '2021-05-05 18:19:20.000') or (ts >= '2021-05-05 18:19:11.000' and ts <= '2021-05-05 18:19:14.999')); +if $rows != 16 then + return -1 +endi +if $data00 != @21-05-05 18:19:05.000@ then + return -1 +endi + +sql select ts,c1,c2 from stb1 where (ts >= '2021-05-05 18:19:05.000' and ts <= '2021-05-05 18:19:10.000') or (ts >= '2021-05-05 18:19:12.000' and ts <= '2021-05-05 18:19:14.000') or (ts >= '2021-05-05 18:19:08.000' and ts <= '2021-05-05 18:19:17.000'); +if $rows != 13 then + return -1 +endi +if $data00 != @21-05-05 18:19:05.000@ then + return -1 +endi + +sql select ts,c1,c2 from stb1 where (ts >= '2021-05-05 18:19:05.000' and ts <= '2021-05-05 18:19:10.000') or (ts >= '2021-05-05 18:19:02.000' and ts <= '2021-05-05 18:19:03.000') or (ts >= '2021-05-05 18:19:01.000' and ts <= '2021-05-05 18:19:08.000'); +if $rows != 10 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi + +sql select ts,c1,c2 from stb1 where ((ts >= '2021-05-05 18:19:08.000' and ts <= '2021-05-05 18:19:10.000') or (ts >= '2021-05-05 18:19:02.000' and ts <= '2021-05-05 18:19:03.000') or (ts >= '2021-05-05 18:19:05.000' and ts <= '2021-05-05 18:19:06.000') or (ts >= '2021-05-05 18:19:03.000' and ts <= '2021-05-05 18:19:12.000')) and (ts >= '2021-05-05 18:19:10.000'); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:10.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:11.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:12.000@ then + return -1 +endi + +sql select ts,c1,c7 from stb1 where ts > '2021-05-05 18:19:25.000' and ts != '2021-05-05 18:19:18'; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:27.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:28.000@ then + return -1 +endi + + +sql select * from stb1 where ts > '2021-05-05 18:19:03.000' and ts > '2021-05-05 18:19:25'; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:27.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:28.000@ then + return -1 +endi + +sql select * from stb1 where ts < '2021-05-05 18:19:03.000' and ts < '2021-05-05 18:19:25'; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi + +sql select * from stb1 where ts > '2021-05-05 18:19:23.000' and ts < '2021-05-05 18:19:25'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi + +sql select * from stb1 where ts > '2021-05-05 18:19:03.000' or ts > '2021-05-05 18:19:25'; +if $rows != 25 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:06.000@ then + return -1 +endi + +sql select * from stb1 where ts < '2021-05-05 18:19:03.000' or ts < '2021-05-05 18:19:25'; +if $rows != 25 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi + +sql select * from stb1 where ts > '2021-05-05 18:19:23.000' or ts < '2021-05-05 18:19:25'; +if $rows != 29 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi + +sql select * from stb1 where (ts > '2021-05-05 18:19:23.000' or ts < '2021-05-05 18:19:25') and (ts > '2021-05-05 18:19:23.000' and ts < '2021-05-05 18:19:26'); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi + +sql select * from stb1 where (ts > '2021-05-05 18:19:23.000' or ts < '2021-05-05 18:19:25') and (ts > '2021-05-05 18:19:23.000' or ts > '2021-05-05 18:19:26'); +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:26.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:28.000@ then + return -1 +endi + + +sql select * from stb2 where ts2 in ('2021-05-05 18:28:03','2021-05-05 18:28:05','2021-05-05 18:28:08'); +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:07.000@ then + return -1 +endi + +sql select * from stb2 where t3 in ('2021-05-05 18:38:38','2021-05-05 18:38:28','2021-05-05 18:38:08') and ts2 in ('2021-05-05 18:28:04','2021-05-05 18:28:04','2021-05-05 18:28:03'); +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:03.000@ then + return -1 +endi + +sql select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and (a.ts < '2021-05-05 18:19:03.000' or a.ts >= '2021-05-05 18:19:13.000') and (b.ts >= '2021-05-05 18:19:01.000' and b.ts <= '2021-05-05 18:19:14.000'); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:13.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:14.000@ then + return -1 +endi + +sql select a.ts,c.ts,b.c1,c.u1,c.u2 from (select * from stb1) a, (select * from stb1) b, (select * from stb2) c where a.ts=b.ts and b.ts=c.ts and a.ts <= '2021-05-05 18:19:12.000' and b.ts >= '2021-05-05 18:19:06.000' and c.ts >= '2021-05-05 18:19:08.000' and c.ts <= '2021-05-05 18:19:11.000' and a.ts != '2021-05-05 18:19:10.000'; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:08.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:09.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:11.000@ then + return -1 +endi + +sql select ts,c1,c2,c8 from (select * from stb1) where (ts <= '2021-05-05 18:19:06.000' or ts >= '2021-05-05 18:19:13.000') and (ts >= '2021-05-05 18:19:02.000' and ts <= '2021-05-05 18:19:14.000') and ts != '2021-05-05 18:19:04.000'; +if $rows != 6 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:06.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:13.000@ then + return -1 +endi +if $data50 != @21-05-05 18:19:14.000@ then + return -1 +endi + +sql select ts,c1,c2,c8 from (select * from stb1) where (ts <= '2021-05-05 18:19:03.000' or ts > '2021-05-05 18:19:26.000' or ts = '2021-05-05 18:19:26.000') and ts != '2021-05-05 18:19:03.000' and ts != '2021-05-05 18:19:26.000'; +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:27.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:28.000@ then + return -1 +endi + +print "tbname test" +sql_error select * from stb1 where tbname like '%3' and tbname like '%4'; + +sql select * from stb1 where tbname like 'tb%'; +if $rows != 29 then + return -1 +endi + +sql select * from stb1 where tbname like '%2'; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:08.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:09.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:10.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:11.000@ then + return -1 +endi + +print "tag test" +sql select * from stb1 where t1 in (1,2) and t1 in (2,3); +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:08.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:09.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:10.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:11.000@ then + return -1 +endi + +sql select * from stb2 where t1 in (1,2) and t2 in (2) and t3 in ('2021-05-05 18:58:57.000'); +if $rows != 0 then + return -1 +endi + +print "join test" +sql_error select * from tb1, tb2_1 where tb1.ts=tb2_1.ts or tb1.ts =tb2_1.ts; +sql select tb1.ts from tb1, tb2_1 where tb1.ts=tb2_1.ts and tb1.ts > '2021-05-05 18:19:03.000' and tb2_1.ts < '2021-05-05 18:19:06.000'; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:05.000@ then + return -1 +endi +sql select tb1.ts,tb1.*,tb2_1.* from tb1, tb2_1 where tb1.ts=tb2_1.ts and tb1.ts > '2021-05-05 18:19:03.000' and tb2_1.u1 < 5; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:06.000@ then + return -1 +endi + +sql select tb1.ts,tb1.*,tb2_1.* from tb1, tb2_1 where tb1.ts=tb2_1.ts and tb1.ts >= '2021-05-05 18:19:03.000' and tb1.c7=false and tb2_1.u3>4; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:07.000@ then + return -1 +endi + +sql select stb1.ts,stb1.c1,stb1.t1,stb2.ts,stb2.u1,stb2.t4 from stb1, stb2 where stb1.ts=stb2.ts and stb1.t1 = stb2.t4; +if $rows != 9 then + return -1 +endi +if $data00 != @21-05-05 18:19:00.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:01.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data40 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data50 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data60 != @21-05-05 18:19:06.000@ then + return -1 +endi +if $data70 != @21-05-05 18:19:07.000@ then + return -1 +endi +if $data80 != @21-05-05 18:19:11.000@ then + return -1 +endi + +sql select stb1.ts,stb1.c1,stb1.t1,stb2.ts,stb2.u1,stb2.t4 from stb1, stb2 where stb1.ts=stb2.ts and stb1.t1 = stb2.t4 and stb1.c1 > 2 and stb2.u1 <=4; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:04.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:06.000@ then + return -1 +endi + +print "column&ts test" +sql_error select count(*) from stb1 where ts > 0 or c1 > 0; +sql select * from stb1 where ts > '2021-05-05 18:19:03.000' and ts < '2021-05-05 18:19:20.000' and (c1 > 23 or c1 < 14) and c7 in (true) and c8 like '%2'; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:13.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:17.000@ then + return -1 +endi + +print "column&tbname test" +sql_error select count(*) from stb1 where tbname like 'tb%' or c1 > 0; +sql select * from stb1 where tbname like '%3' and c6 < 34 and c5 != 33 and c4 > 31; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:13.000@ then + return -1 +endi + +print "column&tag test" +sql_error select * from stb1 where t1 > 0 or c1 > 0 +sql_error select * from stb1 where c1 > 0 or t1 > 0 +sql_error select * from stb1 where t1 > 0 or c1 > 0 or t1 > 1 +sql_error select * from stb1 where c1 > 0 or t1 > 0 or c1 > 1 +sql_error select * from stb1 where t1 > 0 and c1 > 0 or t1 > 1 +sql_error select * from stb1 where c1 > 0 or t1 > 0 and c1 > 1 +sql_error select * from stb1 where c1 > 0 or t1 > 0 and c1 > 1 +sql_error select * from stb1 where t1 > 0 or t1 > 0 and c1 > 1 +sql_error select * from stb1 where (c1 > 0 and t1 > 0 ) or (t1 > 1 and c1 > 3) +sql_error select * from stb1 where (c1 > 0 and t1 > 0 ) or t1 > 1 +sql_error select a.ts,b.ts,a.c1,b.u1,b.u2 from (select * from stb1) a, (select * from stb2) b where a.ts=b.ts and a.t1=b.t1; + +sql select * from stb1 where c1 < 63 and t1 > 5 +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:24.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:25.000@ then + return -1 +endi +sql select * from stb1 where t1 > 3 and t1 < 5 and c1 != 42 and c1 != 44; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:16.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:18.000@ then + return -1 +endi +sql select * from stb1 where t1 > 1 and c1 > 21 and t1 < 3 and c1 < 24 and t1 != 3 and c1 != 23; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:09.000@ then + return -1 +endi +sql select * from stb1 where c1 > 1 and (t1 > 3 or t1 < 2) and (c2 > 2 and c2 < 62 and t1 != 4) and (t1 > 2 and t1 < 6) and c7 = true and c8 like '%2'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:21.000@ then + return -1 +endi + +sql select * from stb1 where c1!=31 and c1 !=32 and c1 <> 63 and c1 <>1 and c1 <> 21 and c1 <> 2 and c7 <> true and c8 <> '3' and c9 <> '4' and c2<>13 and c3 <> 23 and c4 <> 33 and c5 <> 34 and c6 <> 43 and c2 <> 53 and t1 <> 5 and t2 <>4; +if $rows != 3 then + return -1 +endi +if $data00 != @21-05-05 18:19:07.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:11.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:27.000@ then + return -1 +endi + + +print "column&join test" +sql_error select tb1.ts,tb1.c1,tb2_1.u1 from tb1, tb2_1 where tb1.ts=tb2_1.ts or tb1.c1 > 0; + + +print "ts&tbname test" +sql_error select count(*) from stb1 where ts > 0 or tbname like 'tb%'; + +print "ts&tag test" +sql_error select count(*) from stb1 where ts > 0 or t1 > 0; + +sql select * from stb2 where t1!=1 and t2=2 and t3 in ('2021-05-05 18:58:58.000') and ts < '2021-05-05 18:19:13.000'; +if $rows != 2 then + return -1 +endi +if $data00 != @21-05-05 18:19:11.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:12.000@ then + return -1 +endi + +print "ts&join test" +sql_error select tb1.ts,tb1.c1,tb2_1.u1 from tb1, tb2_1 where tb1.ts=tb2_1.ts or tb1.ts > 0; +sql select tb1.ts,tb1.c1,tb2_1.u1 from tb1, tb2_1 where tb1.ts=tb2_1.ts and (tb1.ts > '2021-05-05 18:19:05.000' or tb1.ts < '2021-05-05 18:19:03.000' or tb1.ts > 0); + + +print "tbname&tag test" +sql select * from stb1 where tbname like 'tb%' and (t1=1 or t2=2 or t3=3) and t1 > 2; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:12.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:13.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:14.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:15.000@ then + return -1 +endi + +print "tbname&join test" + +print "tag&join test" + + + + + +print "column&ts&tbname test" +sql_error select count(*) from stb1 where tbname like 'tb%' or c1 > 0 or ts > 0; + +print "column&ts&tag test" +sql_error select count(*) from stb1 where t1 > 0 or c1 > 0 or ts > 0; +sql_error select count(*) from stb1 where c1 > 0 or t1 > 0 or ts > 0; + +sql select * from stb1 where (t1 > 0 or t1 > 2 ) and ts > '2021-05-05 18:19:10.000' and (c1 > 1 or c1 > 3) and (c6 > 40 or c6 < 30) and (c8 like '%3' or c8 like '_4') and (c9 like '1%' or c9 like '6%' or (c9 like '%3' and c9 != '23')) and ts > '2021-05-05 18:19:22.000' and ts <= '2021-05-05 18:19:26.000'; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:26.000@ then + return -1 +endi +sql select * from stb1 where ts > '2021-05-05 18:19:00.000' and c1 > 2 and t1 != 1 and c2 >= 23 and t2 >= 3 and c3 < 63 and c7 = false and t3 > 3 and t3 < 6 and c8 like '4%' and ts < '2021-05-05 18:19:19.000' and c2 > 40 and c3 != 42; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-05 18:19:18.000@ then + return -1 +endi +print "column&ts&join test" + +print "column&tbname&tag test" +sql_error select count(*) from stb1 where c1 > 0 or tbname in ('tb1') or t1 > 0; + +print "column&tbname&join test" +print "column&tag&join test" +print "ts&tbname&tag test" +sql_error select count(*) from stb1 where ts > 0 or tbname in ('tb1') or t1 > 0; + +print "ts&tbname&join test" +print "ts&tag&join test" +print "tbname&tag&join test" + + + + +print "column&ts&tbname&tag test" +sql_error select * from stb1 where (tbname like 'tb%' or ts > '2021-05-05 18:19:01.000') and (t1 > 5 or t1 < 4) and c1 > 0; +sql_error select * from stb1 where (ts > '2021-05-05 18:19:01.000') and (ts > '2021-05-05 18:19:02.000' or t1 > 3) and (t1 > 5 or t1 < 4) and c1 > 0; +sql_error select ts,c1,c7 from stb1 where ts > '2021-05-05 18:19:03.000' or ts > '2021-05-05 18:19:20.000' and col > 0 and t1 > 0; + + +print "column&ts&tbname&join test" +print "column&ts&tag&join test" +print "column&tbname&tag&join test" +print "ts&tbname&tag&join test" + + +print "column&ts&tbname&tag&join test" + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/set_tag_vals.sim b/tests/script/general/parser/set_tag_vals.sim index 74184f94d4..4a63f9c6f1 100644 --- a/tests/script/general/parser/set_tag_vals.sim +++ b/tests/script/general/parser/set_tag_vals.sim @@ -83,10 +83,7 @@ while $i < $tbNum endw print ================== all tags have been changed! -sql select tbname from $stb where t3 = 'NULL' -if $rows != 0 then - return -1 -endi +sql_error select tbname from $stb where t3 = 'NULL' print ================== set tag to NULL sql create table stb1_tg (ts timestamp, c1 int) tags(t1 int,t2 bigint,t3 double,t4 float,t5 smallint,t6 tinyint) @@ -227,4 +224,4 @@ if $data04 != NULL then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index 781b2725b5..6b789de490 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -65,7 +65,14 @@ $tb = $tbPrefix . $i sql_error select * from $tb where c7 # TBASE-654 : invalid filter expression cause server crashed -sql_error select count(*) from $tb where c1<10 and c1<>2 +sql select count(*) from $tb where c1<10 and c1<>2 +if $rows != 1 then + return -1 +endi +if $data00 != 900 then + return -1 +endi + sql select * from $tb where c7 = false $val = $rowNum / 100 @@ -253,30 +260,11 @@ sql insert into tb_where_NULL values ('2019-01-01 09:00:02.000', 2, 'val2') sql_error select * from tb_where_NULL where c1 = NULL sql_error select * from tb_where_NULL where c1 <> NULL sql_error select * from tb_where_NULL where c1 < NULL -sql select * from tb_where_NULL where c1 = "NULL" -if $rows != 0 then - return -1 -endi - -sql select * from tb_where_NULL where c1 <> "NULL" -if $rows != 2 then - return -1 -endi -sql select * from tb_where_NULL where c1 <> "nulL" -if $rows != 2 then - return -1 -endi - -sql select * from tb_where_NULL where c1 > "NULL" -if $rows != 0 then - return -1 -endi - -sql select * from tb_where_NULL where c1 >= "NULL" -if $rows != 0 then - return -1 -endi - +sql_error select * from tb_where_NULL where c1 = "NULL" +sql_error select * from tb_where_NULL where c1 <> "NULL" +sql_error select * from tb_where_NULL where c1 <> "nulL" +sql_error select * from tb_where_NULL where c1 > "NULL" +sql_error select * from tb_where_NULL where c1 >= "NULL" sql select * from tb_where_NULL where c2 = "NULL" if $rows != 0 then return -1 diff --git a/tests/script/test.sh b/tests/script/test.sh index 29a15db4dc..f2dc578987 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -121,6 +121,7 @@ echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 143" >> $TAOS_CFG echo "udebugFlag 143" >> $TAOS_CFG +echo "debugFlag 143" >> $TAOS_CFG echo "wal 0" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG