xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into test/skci_dev

This commit is contained in:
Yiqing Liu 2021-08-10 18:08:38 +08:00 committed by GitHub
parent 78844c5ea8 db2b89741b
commit 9dd7b0e65f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
69 changed files with 9038 additions and 1183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Jenkinsfile vendored
View File

@ -4,7 +4,9 @@ properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine.git'
}
def skipbuild=0
def abortPreviousBuilds() {
def currentJobName = env.JOB_NAME
def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@ -146,6 +148,7 @@ pipeline {
skipbuild='2'
skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true)
println skipbuild
}
sh'''
rm -rf ${WORKSPACE}.tes
@ -160,6 +163,7 @@ pipeline {
changeRequest()
expression{
return skipbuild.trim() == '2'
}
}
}

7
src/client/inc/tscUtil.h
View File

@ -61,6 +61,7 @@ typedef struct SJoinSupporter {
uint64_t uid; // query table uid
SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution
SArray* exprList;
SArray* colCond;
SFieldInfo fieldsInfo;
STagCond tagCond;
SGroupbyExpr groupInfo; // group by info
@ -244,8 +245,9 @@ SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid);
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw);
int32_t tscTagCondCopy(STagCond* dest, const STagCond* src);
int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx);
void tscTagCondRelease(STagCond* pCond);
void tscColCondRelease(SArray** pCond);
void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
bool tscShouldBeFreed(SSqlObj* pSql);
@ -340,7 +342,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
uint32_t tscGetTableMetaMaxSize();
int32_t tscCreateTableMetaFromSTableMeta(STableMeta** pChild, const char* name, size_t *tableMetaCapacity);
int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable);
STableMeta* tscTableMetaDup(STableMeta* pTableMeta);
SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo);
@ -355,6 +357,7 @@ char* strdup_throw(const char* str);
bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx);
void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id);

505
src/client/inc/tsclient.h
View File

@ -84,9 +84,14 @@ typedef struct SParamInfo {
} SParamInfo;
typedef struct SBoundColumn {
bool hasVal; // denote if current column has bound or not
int32_t offset; // all column offset value
int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future
uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val)
} SBoundColumn;
typedef enum {
VAL_STAT_HAS = 0x0, // 0 means has val
VAL_STAT_NONE = 0x01, // 1 means no val
} EValStat;
typedef struct {
uint16_t schemaColIdx;
@ -99,32 +104,106 @@ typedef enum _COL_ORDER_STATUS {
ORDER_STATUS_ORDERED = 1,
ORDER_STATUS_DISORDERED = 2,
} EOrderStatus;
typedef struct SParsedDataColInfo {
int16_t numOfCols;
int16_t numOfBound;
int32_t * boundedColumns; // bounded column idx according to schema
uint16_t flen; // TODO: get from STSchema
uint16_t allNullLen; // TODO: get from STSchema
uint16_t extendedVarLen;
int32_t * boundedColumns; // bound column idx according to schema
SBoundColumn * cols;
SBoundIdxInfo *colIdxInfo;
int8_t orderStatus; // bounded columns:
int8_t orderStatus; // bound columns
} SParsedDataColInfo;
#define IS_DATA_COL_ORDERED(s) ((s) == (int8_t)ORDER_STATUS_ORDERED)
#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
typedef struct {
SSchema * pSchema;
int16_t sversion;
int32_t flen;
uint16_t nCols;
void * buf;
void * pDataBlock;
SSubmitBlk *pSubmitBlk;
int32_t dataLen; // len of SDataRow
int32_t kvLen; // len of SKVRow
} SMemRowInfo;
typedef struct {
uint8_t memRowType;
uint8_t compareStat; // 0 unknown, 1 need compare, 2 no need
TDRowTLenT dataRowInitLen;
TDRowTLenT kvRowInitLen;
SMemRowInfo *rowInfo;
} SMemRowBuilder;
typedef struct {
TDRowLenT allNullLen;
} SMemRowHelper;
typedef enum {
ROW_COMPARE_UNKNOWN = 0,
ROW_COMPARE_NEED = 1,
ROW_COMPARE_NO_NEED = 2,
} ERowCompareStat;
int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec);
int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols,
int32_t allNullLen);
void destroyMemRowBuilder(SMemRowBuilder *pBuilder);
/**
* @brief
*
* @param memRowType
* @param spd
* @param idx the absolute bound index of columns
* @return FORCE_INLINE
*/
static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd,
int32_t idx, int32_t *toffset, int16_t *colId) {
int32_t schemaIdx = 0;
if (IS_DATA_COL_ORDERED(spd)) {
schemaIdx = spd->boundedColumns[idx];
if (isDataRowT(memRowType)) {
*toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart
} else {
*toffset = idx * sizeof(SColIdx); // the offset of SColIdx
}
} else {
ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
if (isDataRowT(memRowType)) {
*toffset = (spd->cols + schemaIdx)->toffset;
} else {
*toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx);
}
}
*colId = pSchema[schemaIdx].colId;
}
/**
* @brief Applicable to consume by multi-columns
*
* @param row
* @param value
* @param isCopyVarData In some scenario, the varVal is copied to row directly before calling tdAppend***ColVal()
* @param colId
* @param colType
* @param idx index in SSchema
* @param pBuilder
* @param spd
* @return FORCE_INLINE
*/
static FORCE_INLINE void tscAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
int8_t colType, int32_t toffset, SMemRowBuilder *pBuilder,
int32_t rowNum) {
tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
if (pBuilder->compareStat == ROW_COMPARE_NEED) {
SMemRowInfo *pRowInfo = pBuilder->rowInfo + rowNum;
tdGetColAppendDeltaLen(value, colType, &pRowInfo->dataLen, &pRowInfo->kvLen);
}
}
// Applicable to consume by one row
static FORCE_INLINE void tscAppendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen,
uint8_t compareStat) {
tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
if (compareStat == ROW_COMPARE_NEED) {
tdGetColAppendDeltaLen(value, colType, dataLen, kvLen);
}
}
typedef struct STableDataBlocks {
SName tableName;
int8_t tsSource; // where does the UNIX timestamp come from, server or client
@ -146,7 +225,7 @@ typedef struct STableDataBlocks {
uint32_t numOfAllocedParams;
uint32_t numOfParams;
SParamInfo * params;
SMemRowHelper rowHelper;
SMemRowBuilder rowBuilder;
} STableDataBlocks;
typedef struct {
@ -435,8 +514,398 @@ int16_t getNewResColId(SSqlCmd* pCmd);
int32_t schemaIdxCompar(const void *lhs, const void *rhs);
int32_t boundIdxCompar(const void *lhs, const void *rhs);
int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen);
int32_t getExtendedRowSize(STableComInfo *tinfo);
static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize);
return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
}
static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) {
if (isDataRow(row)) {
if (kvLen < (dataLen * KVRatioConvert)) {
memRowSetConvert(row);
}
} else if (kvLen > dataLen) {
memRowSetConvert(row);
}
}
static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
memRowSetType(row, memRowType);
if (isDataRowT(memRowType)) {
dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion);
dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen));
} else {
ASSERT(nBoundCols > 0);
memRowSetKvVersion(row, pBlock->pTableMeta->sversion);
kvRowSetNCols(memRowKvBody(row), nBoundCols);
kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
}
}
/**
* TODO: Move to tdataformat.h and refactor when STSchema available.
* - fetch flen and toffset from STSChema and remove param spd
*/
static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols,
SParsedDataColInfo *spd) {
ASSERT(isKvRow(src));
SKVRow kvRow = memRowKvBody(src);
SDataRow dataRow = memRowDataBody(dest);
memRowSetType(dest, SMEM_ROW_DATA);
dataRowSetVersion(dataRow, memRowKvVersion(src));
dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen));
int32_t kvIdx = 0;
for (int i = 0; i < nCols; ++i) {
SSchema *schema = pSchema + i;
void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx);
tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type,
(spd->cols + i)->toffset);
}
}
// TODO: Move to tdataformat.h and refactor when STSchema available.
static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols,
SParsedDataColInfo *spd) {
ASSERT(isDataRow(src));
SDataRow dataRow = memRowDataBody(src);
SKVRow kvRow = memRowKvBody(dest);
memRowSetType(dest, SMEM_ROW_KV);
memRowSetKvVersion(kvRow, dataRowVersion(dataRow));
kvRowSetNCols(kvRow, nBoundCols);
kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
int32_t toffset = 0, kvOffset = 0;
for (int i = 0; i < nCols; ++i) {
if ((spd->cols + i)->valStat == VAL_STAT_HAS) {
SSchema *schema = pSchema + i;
toffset = (spd->cols + i)->toffset;
void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE);
tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset);
kvOffset += sizeof(SColIdx);
}
}
}
// TODO: Move to tdataformat.h and refactor when STSchema available.
static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) {
STableMeta * pTableMeta = pBlock->pTableMeta;
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
SSchema * pSchema = tscGetTableSchema(pTableMeta);
SParsedDataColInfo *spd = &pBlock->boundColumnInfo;
ASSERT(dest != src);
if (isDataRow(src)) {
// TODO: Can we use pBlock -> numOfParam directly?
ASSERT(spd->numOfBound > 0);
convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd);
} else {
convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd);
}
}
static bool isNullStr(SStrToken *pToken) {
return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
static FORCE_INLINE int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) {
errno = 0;
*value = strtold(pToken->z, endPtr);
// not a valid integer number, return error
if ((*endPtr - pToken->z) != pToken->n) {
return TK_ILLEGAL;
}
return pToken->type;
}
static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str,
bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId,
int32_t *dataLen, int32_t *kvLen, uint8_t compareStat) {
int64_t iv;
int32_t ret;
char * endptr = NULL;
if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) {
return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z);
}
switch (pSchema->type) {
case TSDB_DATA_TYPE_BOOL: { // bool
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) {
tscAppendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
} else if (strncmp(pToken->z, "false", pToken->n) == 0) {
tscAppendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z);
}
} else if (pToken->type == TK_INTEGER) {
iv = strtoll(pToken->z, NULL, 10);
tscAppendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset,
dataLen, kvLen, compareStat);
} else if (pToken->type == TK_FLOAT) {
double dv = strtod(pToken->z, NULL);
tscAppendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset,
dataLen, kvLen, compareStat);
} else {
return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z);
}
}
break;
}
case TSDB_DATA_TYPE_TINYINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z);
} else if (!IS_VALID_TINYINT(iv)) {
return tscInvalidOperationMsg(msg, "data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_UTINYINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z);
} else if (!IS_VALID_UTINYINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_SMALLINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z);
} else if (!IS_VALID_SMALLINT(iv)) {
return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z);
}
int16_t tmpVal = (int16_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_USMALLINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z);
} else if (!IS_VALID_USMALLINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z);
}
uint16_t tmpVal = (uint16_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_INT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid int data", pToken->z);
} else if (!IS_VALID_INT(iv)) {
return tscInvalidOperationMsg(msg, "int data overflow", pToken->z);
}
int32_t tmpVal = (int32_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_UINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z);
} else if (!IS_VALID_UINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z);
}
uint32_t tmpVal = (uint32_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_BIGINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z);
} else if (!IS_VALID_BIGINT(iv)) {
return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z);
}
tscAppendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_UBIGINT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z);
} else if (!IS_VALID_UBIGINT((uint64_t)iv)) {
return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z);
}
uint64_t tmpVal = (uint64_t)iv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_FLOAT:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
double dv;
if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
return tscInvalidOperationMsg(msg, "illegal float data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
isnan(dv)) {
return tscInvalidOperationMsg(msg, "illegal float data", pToken->z);
}
float tmpVal = (float)dv;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_DOUBLE:
if (isNullStr(pToken)) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
double dv;
if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
}
tscAppendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_BINARY:
// binary data cannot be null-terminated char string, otherwise the last char of the string is lost
if (pToken->type == TK_NULL) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else { // too long values will return invalid sql, not be truncated automatically
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor
return tscInvalidOperationMsg(msg, "string data overflow", pToken->z);
}
// STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
char *rowEnd = memRowEnd(row);
STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n);
tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_NCHAR:
if (pToken->type == TK_NULL) {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
} else {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
char * rowEnd = memRowEnd(row);
if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE,
&output)) {
char buf[512] = {0};
snprintf(buf, tListLen(buf), "%s", strerror(errno));
return tscInvalidOperationMsg(msg, buf, pToken->z);
}
varDataSetLen(rowEnd, output);
tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
case TSDB_DATA_TYPE_TIMESTAMP: {
if (pToken->type == TK_NULL) {
if (primaryKey) {
// When building SKVRow primaryKey, we should not skip even with NULL value.
int64_t tmpVal = 0;
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
} else {
tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
compareStat);
}
} else {
int64_t tmpVal;
if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z);
}
tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
}
break;
}
}
return TSDB_CODE_SUCCESS;
}
#ifdef __cplusplus
}

574
src/client/src/tscParseInsert.c
View File

@ -38,43 +38,60 @@ enum {
TSDB_USE_CLI_TS = 1,
};
static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows);
static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema,
char *str, char **end);
int32_t getExtendedRowSize(STableComInfo *tinfo) {
return tinfo->rowSize + PAYLOAD_HEADER_LEN + PAYLOAD_COL_HEAD_LEN * tinfo->numOfColumns;
int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols,
int32_t allNullLen) {
ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols));
if (nRows > 0) {
// already init(bind multiple rows by single column)
if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) {
return TSDB_CODE_SUCCESS;
}
int initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen) {
pHelper->allNullLen = allNullColsLen; // TODO: get allNullColsLen when creating or altering table meta
if (pHelper->allNullLen == 0) {
for (uint16_t i = 0; i < nCols; ++i) {
uint8_t type = pSSchema[i].type;
int32_t typeLen = TYPE_BYTES[type];
pHelper->allNullLen += typeLen;
if (TSDB_DATA_TYPE_BINARY == type) {
pHelper->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
} else if (TSDB_DATA_TYPE_NCHAR == type) {
int len = VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE;
pHelper->allNullLen += len;
}
}
}
return 0;
}
static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) {
errno = 0;
*value = strtold(pToken->z, endPtr);
// not a valid integer number, return error
if ((*endPtr - pToken->z) != pToken->n) {
return TK_ILLEGAL;
}
return pToken->type;
if (nBoundCols == 0) { // file input
pBuilder->memRowType = SMEM_ROW_DATA;
pBuilder->compareStat = ROW_COMPARE_NO_NEED;
return TSDB_CODE_SUCCESS;
} else {
float boundRatio = ((float)nBoundCols / (float)nCols);
if (boundRatio < KVRatioKV) {
pBuilder->memRowType = SMEM_ROW_KV;
pBuilder->compareStat = ROW_COMPARE_NO_NEED;
return TSDB_CODE_SUCCESS;
} else if (boundRatio > KVRatioData) {
pBuilder->memRowType = SMEM_ROW_DATA;
pBuilder->compareStat = ROW_COMPARE_NO_NEED;
return TSDB_CODE_SUCCESS;
}
pBuilder->compareStat = ROW_COMPARE_NEED;
if (boundRatio < KVRatioPredict) {
pBuilder->memRowType = SMEM_ROW_KV;
} else {
pBuilder->memRowType = SMEM_ROW_DATA;
}
}
pBuilder->dataRowInitLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
if (nRows > 0) {
pBuilder->rowInfo = tcalloc(nRows, sizeof(SMemRowInfo));
if (pBuilder->rowInfo == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
for (int i = 0; i < nRows; ++i) {
(pBuilder->rowInfo + i)->dataLen = pBuilder->dataRowInitLen;
(pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
}
}
return TSDB_CODE_SUCCESS;
}
int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@ -146,10 +163,6 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1
return TSDB_CODE_SUCCESS;
}
static bool isNullStr(SStrToken* pToken) {
return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey,
int16_t timePrec) {
int64_t iv;
@ -400,342 +413,6 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE TDRowLenT tsSetPayloadColValue(char *payloadStart, char *payload, int16_t columnId,
uint8_t columnType, const void *value, uint16_t valueLen, TDRowTLenT tOffset) {
payloadColSetId(payload, columnId);
payloadColSetType(payload, columnType);
memcpy(POINTER_SHIFT(payloadStart,tOffset), value, valueLen);
return valueLen;
}
static int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, char *payloadStart, char *primaryKeyStart,
char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec,
TDRowTLenT tOffset, TDRowLenT *sizeAppend, TDRowLenT *dataRowColDeltaLen,
TDRowLenT *kvRowColLen) {
int64_t iv;
int32_t ret;
char * endptr = NULL;
if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) {
return tscInvalidOperationMsg(msg, "invalid numeric data", pToken->z);
}
switch (pSchema->type) {
case TSDB_DATA_TYPE_BOOL: { // bool
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_BOOL), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset);
} else {
if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
if (strncmp(pToken->z, "true", pToken->n) == 0) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &TRUE_VALUE,
TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]);
} else if (strncmp(pToken->z, "false", pToken->n) == 0) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &FALSE_VALUE,
TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]);
} else {
return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z);
}
} else if (pToken->type == TK_INTEGER) {
iv = strtoll(pToken->z, NULL, 10);
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]);
} else if (pToken->type == TK_FLOAT) {
double dv = strtod(pToken->z, NULL);
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), TYPE_BYTES[TSDB_DATA_TYPE_BOOL], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BOOL]);
} else {
return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z);
}
}
break;
}
case TSDB_DATA_TYPE_TINYINT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_TINYINT), TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid tinyint data", pToken->z);
} else if (!IS_VALID_TINYINT(iv)) {
return tscInvalidOperationMsg(msg, "data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_TINYINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TINYINT]);
}
break;
case TSDB_DATA_TYPE_UTINYINT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_UTINYINT), TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned tinyint data", pToken->z);
} else if (!IS_VALID_UTINYINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned tinyint data overflow", pToken->z);
}
uint8_t tmpVal = (uint8_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UTINYINT]);
}
break;
case TSDB_DATA_TYPE_SMALLINT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_SMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid smallint data", pToken->z);
} else if (!IS_VALID_SMALLINT(iv)) {
return tscInvalidOperationMsg(msg, "smallint data overflow", pToken->z);
}
int16_t tmpVal = (int16_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_SMALLINT]);
}
break;
case TSDB_DATA_TYPE_USMALLINT:
if (isNullStr(pToken)) {
*sizeAppend =
tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_USMALLINT), TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned smallint data", pToken->z);
} else if (!IS_VALID_USMALLINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned smallint data overflow", pToken->z);
}
uint16_t tmpVal = (uint16_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_USMALLINT]);
}
break;
case TSDB_DATA_TYPE_INT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_INT), TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid int data", pToken->z);
} else if (!IS_VALID_INT(iv)) {
return tscInvalidOperationMsg(msg, "int data overflow", pToken->z);
}
int32_t tmpVal = (int32_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_INT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_INT]);
}
break;
case TSDB_DATA_TYPE_UINT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_UINT), TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned int data", pToken->z);
} else if (!IS_VALID_UINT(iv)) {
return tscInvalidOperationMsg(msg, "unsigned int data overflow", pToken->z);
}
uint32_t tmpVal = (uint32_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_UINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UINT]);
}
break;
case TSDB_DATA_TYPE_BIGINT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_BIGINT), TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid bigint data", pToken->z);
} else if (!IS_VALID_BIGINT(iv)) {
return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z);
}
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &iv,
TYPE_BYTES[TSDB_DATA_TYPE_BIGINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_BIGINT]);
}
break;
case TSDB_DATA_TYPE_UBIGINT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_UBIGINT), TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset);
} else {
ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
if (ret != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid unsigned bigint data", pToken->z);
} else if (!IS_VALID_UBIGINT((uint64_t)iv)) {
return tscInvalidOperationMsg(msg, "unsigned bigint data overflow", pToken->z);
}
uint64_t tmpVal = (uint64_t)iv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_UBIGINT]);
}
break;
case TSDB_DATA_TYPE_FLOAT:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_FLOAT), TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset);
} else {
double dv;
if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
return tscInvalidOperationMsg(msg, "illegal float data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
isnan(dv)) {
return tscInvalidOperationMsg(msg, "illegal float data", pToken->z);
}
float tmpVal = (float)dv;
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_FLOAT], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_FLOAT]);
}
break;
case TSDB_DATA_TYPE_DOUBLE:
if (isNullStr(pToken)) {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_DOUBLE), TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset);
} else {
double dv;
if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
}
if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
}
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type, &dv,
TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_DOUBLE]);
}
break;
case TSDB_DATA_TYPE_BINARY:
// binary data cannot be null-terminated char string, otherwise the last char of the string is lost
if (pToken->type == TK_NULL) {
payloadColSetId(payload, pSchema->colId);
payloadColSetType(payload, pSchema->type);
memcpy(POINTER_SHIFT(payloadStart, tOffset), getNullValue(TSDB_DATA_TYPE_BINARY), VARSTR_HEADER_SIZE + CHAR_BYTES);
*sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + CHAR_BYTES);
} else { // too long values will return invalid sql, not be truncated automatically
if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor
return tscInvalidOperationMsg(msg, "string data overflow", pToken->z);
}
// STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
payloadColSetId(payload, pSchema->colId);
payloadColSetType(payload, pSchema->type);
varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), pToken->n);
memcpy(varDataVal(POINTER_SHIFT(payloadStart,tOffset)), pToken->z, pToken->n);
*sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + pToken->n);
*dataRowColDeltaLen += (TDRowLenT)(pToken->n - CHAR_BYTES);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + pToken->n);
}
break;
case TSDB_DATA_TYPE_NCHAR:
if (pToken->type == TK_NULL) {
payloadColSetId(payload, pSchema->colId);
payloadColSetType(payload, pSchema->type);
memcpy(POINTER_SHIFT(payloadStart,tOffset), getNullValue(TSDB_DATA_TYPE_NCHAR), VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
*sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
} else {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t output = 0;
payloadColSetId(payload, pSchema->colId);
payloadColSetType(payload, pSchema->type);
if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(POINTER_SHIFT(payloadStart,tOffset)),
pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
char buf[512] = {0};
snprintf(buf, tListLen(buf), "%s", strerror(errno));
return tscInvalidOperationMsg(msg, buf, pToken->z);
}
varDataSetLen(POINTER_SHIFT(payloadStart,tOffset), output);
*sizeAppend = (TDRowLenT)(VARSTR_HEADER_SIZE + output);
*dataRowColDeltaLen += (TDRowLenT)(output - sizeof(uint32_t));
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + VARSTR_HEADER_SIZE + output);
}
break;
case TSDB_DATA_TYPE_TIMESTAMP: {
if (pToken->type == TK_NULL) {
if (primaryKey) {
// When building SKVRow primaryKey, we should not skip even with NULL value.
int64_t tmpVal = 0;
*sizeAppend = tsSetPayloadColValue(payloadStart, primaryKeyStart, pSchema->colId, pSchema->type, &tmpVal,
TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]);
} else {
*sizeAppend = tsSetPayloadColValue(payloadStart, payload, pSchema->colId, pSchema->type,
getNullValue(TSDB_DATA_TYPE_TIMESTAMP),
TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset);
}
} else {
int64_t tmpVal;
if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) {
return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z);
}
*sizeAppend = tsSetPayloadColValue(payloadStart, primaryKey ? primaryKeyStart : payload, pSchema->colId,
pSchema->type, &tmpVal, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], tOffset);
*kvRowColLen += (TDRowLenT)(sizeof(SColIdx) + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP]);
}
break;
}
}
return TSDB_CODE_SUCCESS;
}
/*
* The server time/client time should not be mixed up in one sql string
* Do not employ sort operation is not involved if server time is used.
@ -777,31 +454,24 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
int32_t index = 0;
SStrToken sToken = {0};
SMemRowHelper *pHelper = &pDataBlocks->rowHelper;
char * payload = pDataBlocks->pData + pDataBlocks->size;
char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header
SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo;
SSchema * schema = tscGetTableSchema(pDataBlocks->pTableMeta);
STableMeta * pTableMeta = pDataBlocks->pTableMeta;
SSchema * schema = tscGetTableSchema(pTableMeta);
SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder;
int32_t dataLen = pBuilder->dataRowInitLen;
int32_t kvLen = pBuilder->kvRowInitLen;
bool isParseBindParam = false;
TDRowTLenT dataRowLen = pHelper->allNullLen;
TDRowTLenT kvRowLen = TD_MEM_ROW_KV_VER_SIZE;
TDRowTLenT payloadValOffset = 0;
TDRowLenT colValOffset = 0;
ASSERT(dataRowLen > 0);
payloadSetNCols(payload, spd->numOfBound);
payloadValOffset = payloadValuesOffset(payload); // rely on payloadNCols
// payloadSetTLen(payload, payloadValOffset);
char *kvPrimaryKeyStart = payload + PAYLOAD_HEADER_LEN; // primaryKey in 1st column tuple
char *kvStart = kvPrimaryKeyStart + PAYLOAD_COL_HEAD_LEN; // the column tuple behind the primaryKey
initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound);
// 1. set the parsed value from sql string
for (int i = 0; i < spd->numOfBound; ++i) {
// the start position in data block buffer of current value in sql
int32_t colIndex = spd->boundedColumns[i];
char *start = payload + spd->cols[colIndex].offset;
char *start = row + spd->cols[colIndex].offset;
SSchema *pSchema = &schema[colIndex]; // get colId here
@ -810,6 +480,9 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
*str += index;
if (sToken.type == TK_QUESTION) {
if (!isParseBindParam) {
isParseBindParam = true;
}
if (pInsertParam->insertType != TSDB_QUERY_TYPE_STMT_INSERT) {
return tscSQLSyntaxErrMsg(pInsertParam->msg, "? only allowed in binding insertion", *str);
}
@ -861,52 +534,43 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
}
bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX);
TDRowLenT dataRowDeltaColLen = 0; // When combine the data as SDataRow, the delta len between all NULL columns.
TDRowLenT kvRowColLen = 0;
TDRowLenT colValAppended = 0;
int32_t toffset = -1;
int16_t colId = -1;
tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId);
if (!IS_DATA_COL_ORDERED(spd->orderStatus)) {
ASSERT(spd->colIdxInfo != NULL);
if(!isPrimaryKey) {
kvStart = POINTER_SHIFT(kvPrimaryKeyStart, spd->colIdxInfo[i].finalIdx * PAYLOAD_COL_HEAD_LEN);
} else {
ASSERT(spd->colIdxInfo[i].finalIdx == 0);
}
}
// the primary key locates in 1st column
int32_t ret = tsParseOneColumnKV(pSchema, &sToken, payload, kvPrimaryKeyStart, kvStart, pInsertParam->msg, str,
isPrimaryKey, timePrec, payloadValOffset + colValOffset, &colValAppended,
&dataRowDeltaColLen, &kvRowColLen);
int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset,
colId, &dataLen, &kvLen, pBuilder->compareStat);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
if (isPrimaryKey) {
if (tsCheckTimestamp(pDataBlocks, payloadValues(payload)) != TSDB_CODE_SUCCESS) {
TSKEY tsKey = memRowKey(row);
if (tsCheckTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) {
tscInvalidOperationMsg(pInsertParam->msg, "client time/server time can not be mixed up", sToken.z);
return TSDB_CODE_TSC_INVALID_TIME_STAMP;
}
payloadColSetOffset(kvPrimaryKeyStart, colValOffset);
} else {
payloadColSetOffset(kvStart, colValOffset);
if (IS_DATA_COL_ORDERED(spd->orderStatus)) {
kvStart += PAYLOAD_COL_HEAD_LEN; // move to next column
}
}
colValOffset += colValAppended;
kvRowLen += kvRowColLen;
dataRowLen += dataRowDeltaColLen;
if (!isParseBindParam) {
// 2. check and set convert flag
if (pBuilder->compareStat == ROW_COMPARE_NEED) {
checkAndConvertMemRow(row, dataLen, kvLen);
}
if (kvRowLen < dataRowLen) {
payloadSetType(payload, SMEM_ROW_KV);
} else {
payloadSetType(payload, SMEM_ROW_DATA);
// 3. set the null value for the columns that do not assign values
if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) {
SDataRow dataRow = memRowDataBody(row);
for (int32_t i = 0; i < spd->numOfCols; ++i) {
if (spd->cols[i].valStat == VAL_STAT_NONE) {
tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset);
}
}
}
}
*len = (int32_t)(payloadValOffset + colValOffset);
payloadSetTLen(payload, *len);
*len = getExtendedRowSize(pDataBlocks);
return TSDB_CODE_SUCCESS;
}
@ -957,11 +621,13 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn
int32_t precision = tinfo.precision;
int32_t extendedRowSize = getExtendedRowSize(&tinfo);
initSMemRowHelper(&pDataBlock->rowHelper, tscGetTableSchema(pDataBlock->pTableMeta),
tscGetNumOfColumns(pDataBlock->pTableMeta), 0);
int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
if (TSDB_CODE_SUCCESS !=
(code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound,
pDataBlock->boundColumnInfo.allNullLen))) {
return code;
}
while (1) {
index = 0;
sToken = tStrGetToken(*str, &index, false);
@ -1012,19 +678,37 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn
void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) {
pColInfo->numOfCols = numOfCols;
pColInfo->numOfBound = numOfCols;
pColInfo->orderStatus = ORDER_STATUS_ORDERED;
pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode
pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t));
pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn));
pColInfo->colIdxInfo = NULL;
pColInfo->flen = 0;
pColInfo->allNullLen = 0;
int32_t nVar = 0;
for (int32_t i = 0; i < pColInfo->numOfCols; ++i) {
uint8_t type = pSchema[i].type;
if (i > 0) {
pColInfo->cols[i].offset = pSchema[i - 1].bytes + pColInfo->cols[i - 1].offset;
pColInfo->cols[i].toffset = pColInfo->flen;
}
pColInfo->flen += TYPE_BYTES[type];
switch (type) {
case TSDB_DATA_TYPE_BINARY:
pColInfo->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
++nVar;
break;
case TSDB_DATA_TYPE_NCHAR:
pColInfo->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
++nVar;
break;
default:
break;
}
pColInfo->cols[i].hasVal = true;
pColInfo->boundedColumns[i] = i;
}
pColInfo->allNullLen += pColInfo->flen;
pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT));
}
int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) {
@ -1137,22 +821,16 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk
}
memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc);
int32_t extendedRowSize = getExtendedRowSize(dataBuf);
SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
char * pBlockData = pBlocks->data;
TDRowTLenT totolPayloadTLen = 0;
TDRowTLenT payloadTLen = 0;
int n = 0;
while (n < nRows) {
pBlkKeyTuple->skey = payloadTSKey(pBlockData);
pBlkKeyTuple->skey = memRowKey(pBlockData);
pBlkKeyTuple->payloadAddr = pBlockData;
payloadTLen = payloadTLen(pBlockData);
#if 0
ASSERT(payloadNCols(pBlockData) <= 4096);
ASSERT(payloadTLen(pBlockData) < 65536);
#endif
totolPayloadTLen += payloadTLen;
// next loop
pBlockData += payloadTLen;
pBlockData += extendedRowSize;
++pBlkKeyTuple;
++n;
}
@ -1169,7 +847,6 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk
TSKEY tj = (pBlkKeyTuple + j)->skey;
if (ti == tj) {
totolPayloadTLen -= payloadTLen(pBlkKeyTuple + j);
++j;
continue;
}
@ -1185,17 +862,15 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk
pBlocks->numOfRows = i + 1;
}
dataBuf->size = sizeof(SSubmitBlk) + totolPayloadTLen;
dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize;
dataBuf->prevTS = INT64_MIN;
return 0;
}
static int32_t doParseInsertStatement(SInsertStatementParam *pInsertParam, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) {
STableComInfo tinfo = tscGetTableInfo(dataBuf->pTableMeta);
int32_t maxNumOfRows;
int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(&tinfo), &maxNumOfRows);
int32_t code = tscAllocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows);
if (TSDB_CODE_SUCCESS != code) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@ -1533,7 +1208,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
pColInfo->numOfBound = 0;
memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols);
for (int32_t i = 0; i < nCols; ++i) {
pColInfo->cols[i].hasVal = false;
pColInfo->cols[i].valStat = VAL_STAT_NONE;
}
int32_t code = TSDB_CODE_SUCCESS;
@ -1572,12 +1247,12 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
int32_t nScanned = 0, t = lastColIdx + 1;
while (t < nCols) {
if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) {
if (pColInfo->cols[t].hasVal == true) {
if (pColInfo->cols[t].valStat == VAL_STAT_HAS) {
code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z);
goto _clean;
}
pColInfo->cols[t].hasVal = true;
pColInfo->cols[t].valStat = VAL_STAT_HAS;
pColInfo->boundedColumns[pColInfo->numOfBound] = t;
++pColInfo->numOfBound;
findColumnIndex = true;
@ -1595,12 +1270,12 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
int32_t nRemain = nCols - nScanned;
while (t < nRemain) {
if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) {
if (pColInfo->cols[t].hasVal == true) {
if (pColInfo->cols[t].valStat == VAL_STAT_HAS) {
code = tscInvalidOperationMsg(pInsertParam->msg, "duplicated column name", sToken.z);
goto _clean;
}
pColInfo->cols[t].hasVal = true;
pColInfo->cols[t].valStat = VAL_STAT_HAS;
pColInfo->boundedColumns[pColInfo->numOfBound] = t;
++pColInfo->numOfBound;
findColumnIndex = true;
@ -1835,7 +1510,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
goto _clean;
}
if (dataBuf->boundColumnInfo.cols[0].hasVal == false) {
if (dataBuf->boundColumnInfo.cols[0].valStat == VAL_STAT_NONE) {
code = tscInvalidOperationMsg(pInsertParam->msg, "primary timestamp column can not be null", NULL);
goto _clean;
}
@ -2046,15 +1721,18 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
goto _error;
}
tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(&tinfo), &maxRows);
tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(pTableDataBlock), &maxRows);
tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW);
if (tokenBuf == NULL) {
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
initSMemRowHelper(&pTableDataBlock->rowHelper, tscGetTableSchema(pTableDataBlock->pTableMeta),
tscGetNumOfColumns(pTableDataBlock->pTableMeta), 0);
if (TSDB_CODE_SUCCESS !=
(ret = initMemRowBuilder(&pTableDataBlock->rowBuilder, 0, tinfo.numOfColumns, pTableDataBlock->numOfParams,
pTableDataBlock->boundColumnInfo.allNullLen))) {
goto _error;
}
while ((readLen = tgetline(&line, &n, fp)) != -1) {
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {

2
src/client/src/tscPrepare.c
View File

@ -299,7 +299,7 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) {
SSchema *schema = (SSchema*)pBlock->pTableMeta->schema;
for (int32_t i = 0; i < spd->numOfCols; ++i) {
if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null
if (spd->cols[i].valStat == VAL_STAT_NONE) { // current column do not have any value to insert, set it to null
for (int32_t n = 0; n < rowNum; ++n) {
char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset;

567
src/client/src/tscSQLParser.c
View File

@ -39,6 +39,7 @@
#include "ttokendef.h"
#include "qScript.h"
#include "ttype.h"
#include "qFilter.h"
#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@ -140,6 +141,7 @@ static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlE
static bool validateDebugFlag(int32_t v);
static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo);
static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex);
static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) {
return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0;
@ -183,6 +185,9 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType,
if (var->nType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(var->nType)) {
break;
}
if (colType == TSDB_DATA_TYPE_BOOL && (var->i64 > 1 ||var->i64 < 0)) {
break;
}
tbufWriteInt64(&bw, var->i64);
} else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) {
if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) {
@ -3217,7 +3222,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
pCmd->command = TSDB_SQL_SHOW;
const char* msg1 = "invalid name";
const char* msg2 = "pattern filter string too long";
const char* msg2 = "wildcard string should be less than %d characters";
const char* msg3 = "database name too long";
const char* msg5 = "database name is empty";
const char* msg6 = "pattern string is empty";
@ -3260,8 +3265,10 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
if (!tscValidateTableNameLength(pCmd->payloadLen)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
if (pPattern->n > tsMaxWildCardsLen){
char tmp[64] = {0};
sprintf(tmp, msg2, tsMaxWildCardsLen);
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tmp);
}
}
} else if (showType == TSDB_MGMT_TABLE_VNODES) {
@ -3732,8 +3739,8 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false);
} else if (colType == TSDB_DATA_TYPE_NCHAR) {
// pRight->value.nLen + 1 is larger than the actual nchar string length
pColumnFilter->pz = (int64_t)calloc(1, bufLen * TSDB_NCHAR_SIZE);
// bufLen + 1 is larger than the actual nchar string length
pColumnFilter->pz = (int64_t)calloc(1, (bufLen + 1) * TSDB_NCHAR_SIZE);
retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false);
size_t len = twcslen((wchar_t*)pColumnFilter->pz);
pColumnFilter->len = len * TSDB_NCHAR_SIZE;
@ -3839,51 +3846,40 @@ static int32_t tablenameCondToString(tSqlExpr* pExpr, SStringBuilder* sb) {
}
enum {
TSQL_EXPR_TS = 0,
TSQL_EXPR_TAG = 1,
TSQL_EXPR_COLUMN = 2,
TSQL_EXPR_TBNAME = 3,
TSQL_EXPR_TS = 1,
TSQL_EXPR_TAG = 2,
TSQL_EXPR_COLUMN = 4,
TSQL_EXPR_TBNAME = 8,
TSQL_EXPR_JOIN = 16,
};
static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) {
#define GET_MIXED_TYPE(t) (((t) >= TSQL_EXPR_JOIN) || ((t) > TSQL_EXPR_COLUMN && (t) < TSQL_EXPR_TBNAME) || ((t) == (TSQL_EXPR_TS|TSQL_EXPR_TAG)))
static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
int32_t ret = 0;
const char* msg1 = "non binary column not support like operator";
const char* msg2 = "binary column not support this operator";
const char* msg3 = "bool column not support this operator";
const char* msg4 = "primary key not support this operator";
SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema);
SColumnFilterInfo* pColFilter = NULL;
pColumn->info.flist.numOfFilters++;
/*
* in case of TK_AND filter condition, we first find the corresponding column and build the query condition together
* the already existed condition.
*/
if (sqlOptr == TK_AND) {
// this is a new filter condition on this column
if (pColumn->info.flist.numOfFilters == 0) {
pColFilter = addColumnFilterInfo(&pColumn->info.flist);
} else { // update the existed column filter information, find the filter info here
pColFilter = &pColumn->info.flist.filterInfo[0];
}
if (pColFilter == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
} else if (sqlOptr == TK_OR) {
// TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2"
pColFilter = addColumnFilterInfo(&pColumn->info.flist);
if (pColFilter == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
} else { // error;
if (sqlOptr != TK_AND && sqlOptr != TK_OR) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
SColumnFilterInfo* pColFilter = calloc(1, sizeof(SColumnFilterInfo));
pColFilter->filterstr =
((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0);
@ -3894,17 +3890,20 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC
&& pExpr->tokenId != TK_NOTNULL
&& pExpr->tokenId != TK_LIKE
&& pExpr->tokenId != TK_IN) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
goto _err_ret;
}
} else {
if (pExpr->tokenId == TK_LIKE) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
goto _err_ret;
}
if (pSchema->type == TSDB_DATA_TYPE_BOOL) {
int32_t t = pExpr->tokenId;
if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL && t != TK_IN) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
goto _err_ret;
}
}
}
@ -3916,7 +3915,12 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC
}
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
return doExtractColumnFilterInfo(pCmd, pQueryInfo, tinfo.precision, pColFilter, pSchema->type, pExpr);
ret = doExtractColumnFilterInfo(pCmd, pQueryInfo, tinfo.precision, pColFilter, pSchema->type, pExpr);
_err_ret:
freeColumnFilterInfo(pColFilter, 1);
return ret;
}
static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pTableCond, SStringBuilder* sb) {
@ -3953,7 +3957,61 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr*
return ret;
}
static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) {
static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) {
int32_t ret = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i);
if (p1 == NULL) { // no query condition on this table
continue;
}
tExprNode* p = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex));
ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
taosArrayDestroy(colList);
SBufferWriter bw = tbufInitWriter(NULL, false);
TRY(0) {
exprTreeToBinary(&bw, p);
} CATCH(code) {
tbufCloseWriter(&bw);
UNUSED(code);
// TODO: more error handling
} END_TRY
// add to required table column list
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
int64_t uid = pTableMetaInfo->pTableMeta->id.uid;
STblCond cond = {
.uid = uid,
.idx = i,
.len = (int32_t)(tbufTell(&bw)),
.cond = tbufGetData(&bw, true)
};
if (pQueryInfo->colCond == NULL) {
pQueryInfo->colCond = taosArrayInit(2, sizeof(SCond));
}
taosArrayPush(pQueryInfo->colCond, &cond);
tSqlExprDestroy(p1);
tExprTreeDestroy(p, NULL);
if (ret) {
break;
}
}
return ret;
}
static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) {
if (pExpr == NULL) {
pQueryInfo->onlyHasTagCond &= true;
return TSDB_CODE_SUCCESS;
@ -3961,19 +4019,19 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSq
pQueryInfo->onlyHasTagCond &= false;
if (!tSqlExprIsParentOfLeaf(pExpr)) { // internal node
int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId);
int32_t ret = checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId);
return checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId);
} else { // handle leaf node
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
return extractColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr);
return checkColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr);
}
}
@ -4240,6 +4298,10 @@ static bool isValidExpr(tSqlExpr* pLeft, tSqlExpr* pRight, int32_t optr) {
return false;
}
if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && (optr == TK_NOTNULL || optr == TK_ISNULL)) {
return false;
}
return true;
}
@ -4359,7 +4421,18 @@ static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg
return TSDB_CODE_SUCCESS;
}
static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) {
static int32_t setNormalExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, int32_t parentOptr) {
if (*parent != NULL) {
*parent = tSqlExprCreate((*parent), pExpr, parentOptr);
} else {
*parent = pExpr;
}
return TSDB_CODE_SUCCESS;
}
static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
const char* msg = "only support is [not] null";
tSqlExpr* pRight = pExpr->pRight;
@ -4367,13 +4440,34 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) {
return invalidOperationMsg(msgBuf, msg);
}
if (pRight->tokenId == TK_STRING) {
SSchema* pSchema = tscGetTableSchema(pTableMeta);
if (IS_VAR_DATA_TYPE(pSchema[index].type)) {
return TSDB_CODE_SUCCESS;
}
char *v = strndup(pRight->exprToken.z, pRight->exprToken.n);
int32_t len = strRmquote(v, pRight->exprToken.n);
if (len > 0) {
uint32_t type = 0;
tGetToken(v, &type);
if (type == TK_NULL) {
free(v);
return invalidOperationMsg(msgBuf, msg);
}
}
free(v);
}
return TSDB_CODE_SUCCESS;
}
// check for like expression
static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
const char* msg1 = "wildcard string should be less than %d characters";
const char* msg2 = "illegal column name";
const char* msg2 = "illegal column type for like";
tSqlExpr* pLeft = pExpr->pLeft;
tSqlExpr* pRight = pExpr->pRight;
@ -4394,11 +4488,23 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
return TSDB_CODE_SUCCESS;
}
int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) {
tSqlExpr* left = tSqlExprClone(expr);
tSqlExpr* right = expr;
left->tokenId = TK_LT;
right->tokenId = TK_GT;
*rexpr = tSqlExprCreate(left, right, TK_OR);
return TSDB_CODE_SUCCESS;
}
static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr,
int32_t* type, int32_t parentOptr) {
int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) {
const char* msg1 = "table query cannot use tags filter";
const char* msg2 = "illegal column name";
const char* msg3 = "only one query time range allowed";
const char* msg4 = "too many join tables";
const char* msg5 = "not support ordinary column join";
const char* msg6 = "only one query condition on tbname allowed";
@ -4414,13 +4520,15 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
*tbIdx = index.tableIndex;
assert(tSqlExprIsParentOfLeaf(*pExpr));
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
// validate the null expression
int32_t code = validateNullExpr(*pExpr, tscGetErrorMsgPayload(pCmd));
int32_t code = validateNullExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -4487,12 +4595,24 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
* since this expression is used to set the join query type
*/
tSqlExprDestroy(*pExpr);
if (type) {
*type |= TSQL_EXPR_JOIN;
}
} else {
ret = setExprToCond(&pCondExpr->pTimewindow, *pExpr, msg3, parentOptr, pQueryInfo->msg);
tSqlExpr *rexpr = NULL;
if ((*pExpr)->tokenId == TK_NE) {
handleNeOptr(&rexpr, *pExpr);
} else {
rexpr = *pExpr;
}
ret = setNormalExprToCond(tsExpr, rexpr, parentOptr);
if (type) {
*type |= TSQL_EXPR_TS;
}
}
*pExpr = NULL; // remove this expression
*type = TSQL_EXPR_TS;
} else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
// query on tags, check for tag query condition
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
@ -4517,7 +4637,9 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
*type = TSQL_EXPR_TBNAME;
if (type) {
*type |= TSQL_EXPR_TAG;
}
*pExpr = NULL;
} else {
if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query
@ -4528,23 +4650,41 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY;
ret = setExprToCond(&pCondExpr->pJoinExpr, *pExpr, NULL, parentOptr, pQueryInfo->msg);
*pExpr = NULL;
if (type) {
*type |= TSQL_EXPR_JOIN;
}
} else {
// do nothing
// ret = setExprToCond(pCmd, &pCondExpr->pTagCond,
// *pExpr, NULL, parentOptr);
tSqlExpr *rexpr = NULL;
if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) {
handleNeOptr(&rexpr, *pExpr);
*pExpr = rexpr;
}
*type = TSQL_EXPR_TAG;
if (type) {
*type |= TSQL_EXPR_TAG;
}
}
}
} else { // query on other columns
*type = TSQL_EXPR_COLUMN;
if (type) {
*type |= TSQL_EXPR_COLUMN;
}
if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
ret = setExprToCond(&pCondExpr->pColumnCond, *pExpr, NULL, parentOptr, pQueryInfo->msg);
tSqlExpr *rexpr = NULL;
if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR && pSchema->type != TSDB_DATA_TYPE_BOOL)) {
handleNeOptr(&rexpr, *pExpr);
} else {
rexpr = *pExpr;
}
ret = setNormalExprToCond(columnExpr, rexpr, parentOptr);
*pExpr = NULL; // remove it from expr tree
}
@ -4552,12 +4692,20 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
}
int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr,
int32_t* type, int32_t parentOptr) {
int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr, tSqlExpr** tsExpr) {
if (pExpr == NULL) {
return TSDB_CODE_SUCCESS;
}
const char* msg1 = "query condition between different columns must use 'AND'";
tSqlExpr *columnLeft = NULL;
tSqlExpr *columnRight = NULL;
tSqlExpr *tsLeft = NULL;
tSqlExpr *tsRight = NULL;
int32_t ret = 0;
const char* msg1 = "query condition between columns/tags/timestamp/join fields must use 'AND'";
const char* msg2 = "query condition between tables must use 'AND'";
if ((*pExpr)->flags & (1 << EXPR_FLAG_TS_ERROR)) {
return TSDB_CODE_TSC_INVALID_OPERATION;
@ -4570,48 +4718,94 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
return TSDB_CODE_TSC_INVALID_OPERATION;
}
int32_t leftType = -1;
int32_t rightType = -1;
int32_t leftType = 0;
int32_t rightType = 0;
int32_t leftTbIdx = 0;
int32_t rightTbIdx = 0;
if (!tSqlExprIsParentOfLeaf(*pExpr)) {
int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId);
ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, type ? &leftType : NULL, &leftTbIdx, (*pExpr)->tokenId, &columnLeft, &tsLeft);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
goto err_ret;
}
ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId);
ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, type ? &rightType : NULL, &rightTbIdx, (*pExpr)->tokenId, &columnRight, &tsRight);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
goto err_ret;
}
/*
* if left child and right child do not belong to the same group, the sub
* expression is not valid for parent node, it must be TK_AND operator.
*/
if (leftType != rightType) {
if ((*pExpr)->tokenId == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
if (type != NULL && ((leftType != rightType) || GET_MIXED_TYPE(leftType)) && ((*pExpr)->tokenId == TK_OR)) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
goto err_ret;
}
*type = rightType;
if (((leftTbIdx != rightTbIdx) || (leftTbIdx == -1 || rightTbIdx == -1)) && ((*pExpr)->tokenId == TK_OR)) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
goto err_ret;
}
if (columnLeft && columnRight) {
setNormalExprToCond(&columnLeft, columnRight, (*pExpr)->tokenId);
*columnExpr = columnLeft;
} else {
*columnExpr = columnLeft ? columnLeft : columnRight;
}
if (tsLeft && tsRight) {
setNormalExprToCond(&tsLeft, tsRight, (*pExpr)->tokenId);
*tsExpr = tsLeft;
} else {
*tsExpr = tsLeft ? tsLeft : tsRight;
}
if (type) {
*type = leftType|rightType;
}
*tbIdx = (leftTbIdx == rightTbIdx) ? leftTbIdx : -1;
return TSDB_CODE_SUCCESS;
}
exchangeExpr(*pExpr);
if (pLeft->tokenId == TK_ID && pRight->tokenId == TK_TIMESTAMP && (pRight->flags & (1 << EXPR_FLAG_TIMESTAMP_VAR))) {
return TSDB_CODE_TSC_INVALID_OPERATION;
ret = TSDB_CODE_TSC_INVALID_OPERATION;
goto err_ret;
}
if ((pLeft->flags & (1 << EXPR_FLAG_TS_ERROR)) || (pRight->flags & (1 << EXPR_FLAG_TS_ERROR))) {
return TSDB_CODE_TSC_INVALID_OPERATION;
ret = TSDB_CODE_TSC_INVALID_OPERATION;
goto err_ret;
}
return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr);
ret = handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, tbIdx, parentOptr, columnExpr, tsExpr);
if (ret) {
goto err_ret;
}
return TSDB_CODE_SUCCESS;
err_ret:
tSqlExprDestroy(columnLeft);
tSqlExprDestroy(columnRight);
tSqlExprDestroy(tsLeft);
tSqlExprDestroy(tsRight);
return ret;
}
static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) {
if (*pExpr == NULL) {
*pOut = NULL;
return;
}
if (tSqlExprIsParentOfLeaf(*pExpr)) {
tSqlExpr* pLeft = (*pExpr)->pLeft;
@ -4733,38 +4927,73 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
return TSDB_CODE_SUCCESS;
}
static bool validateFilterExpr(SQueryInfo* pQueryInfo) {
SArray* pColList = pQueryInfo->colList;
int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) {
const char* msg0 = "only one time stamp window allowed";
size_t num = taosArrayGetSize(pColList);
#define SET_EMPTY_RANGE(w) do { (w)->skey = INT64_MAX; (w)->ekey = INT64_MIN; } while (0)
#define IS_EMPTY_RANGE(w) ((w)->skey == INT64_MAX && (w)->ekey == INT64_MIN)
for (int32_t i = 0; i < num; ++i) {
SColumn* pCol = taosArrayGetP(pColList, i);
for (int32_t j = 0; j < pCol->info.flist.numOfFilters; ++j) {
SColumnFilterInfo* pColFilter = &pCol->info.flist.filterInfo[j];
int32_t lowerOptr = pColFilter->lowerRelOptr;
int32_t upperOptr = pColFilter->upperRelOptr;
if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) &&
(upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) {
continue;
if (optr == TSDB_RELATION_AND) {
if (res->skey > win->ekey || win->skey > res->ekey) {
SET_EMPTY_RANGE(res);
return TSDB_CODE_SUCCESS;
}
// there must be at least two range, not support yet.
if (lowerOptr * upperOptr != TSDB_RELATION_INVALID) {
return false;
}
}
if (res->skey < win->skey) {
res->skey = win->skey;
}
return true;
if (res->ekey > win->ekey) {
res->ekey = win->ekey;
}
static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) {
const char* msg0 = "invalid timestamp";
const char* msg1 = "only one time stamp window allowed";
return TSDB_CODE_SUCCESS;
}
if (res->skey > win->ekey || win->skey > res->ekey) {
if (IS_EMPTY_RANGE(res)) {
res->skey = win->skey;
res->ekey = win->ekey;
return TSDB_CODE_SUCCESS;
}
if (IS_EMPTY_RANGE(win)) {
return TSDB_CODE_SUCCESS;
}
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0);
}
if (res->skey > win->skey) {
res->skey = win->skey;
}
if (res->ekey < win->ekey) {
res->ekey = win->ekey;
}
return TSDB_CODE_SUCCESS;
}
static int32_t createTimeRangeExpr(tSqlExpr** pExpr, STimeWindow* win, uint32_t tokenId) {
*pExpr = calloc(1, sizeof(tSqlExpr));
(*pExpr)->type = SQL_NODE_VALUE;
(*pExpr)->tokenId = tokenId;
(*pExpr)->value.nType = TSDB_DATA_TYPE_VALUE_ARRAY;
(*pExpr)->value.nLen = 2;
(*pExpr)->value.arr = taosArrayInit(2, sizeof(int64_t));
taosArrayPush((*pExpr)->value.arr, &win->skey);
taosArrayPush((*pExpr)->value.arr, &win->ekey);
return TSDB_CODE_SUCCESS;
}
static int32_t convertTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) {
const char* msg0 = "invalid timestamp or operator for timestamp";
int32_t code = 0;
STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
if (pExpr == NULL) {
pQueryInfo->onlyHasTagCond &= true;
@ -4774,16 +5003,15 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE
if (!tSqlExprIsParentOfLeaf(pExpr)) {
if (pExpr->tokenId == TK_OR) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
code = getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft);
code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft);
if (code) {
return code;
}
return getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight);
code = convertTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight);
if (code) {
return code;
}
} else {
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
@ -4795,19 +5023,13 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlE
tSqlExpr* pRight = pExpr->pRight;
STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg0);
}
// update the timestamp query range
if (pQueryInfo->window.skey < win.skey) {
pQueryInfo->window.skey = win.skey;
}
createTimeRangeExpr(&pExpr->pRight, &win, pRight->tokenId);
if (pQueryInfo->window.ekey > win.ekey) {
pQueryInfo->window.ekey = win.ekey;
}
tSqlExprDestroy(pRight);
}
return TSDB_CODE_SUCCESS;
@ -4973,7 +5195,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
return TSDB_CODE_SUCCESS;
}
static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSqlExpr** pExpr) {
static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) {
int32_t ret = TSDB_CODE_SUCCESS;
if (pCondExpr->pTagCond == NULL) {
@ -4981,7 +5203,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
}
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i);
tSqlExpr* p1 = extractExprForSTable(pCmd, &pCondExpr->pTagCond, pQueryInfo, i);
if (p1 == NULL) { // no query condition on this table
continue;
}
@ -4990,6 +5212,10 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
SArray* colList = taosArrayInit(10, sizeof(SColIndex));
ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
//if (ret == TSDB_CODE_SUCCESS) {
// ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList));
//}
SBufferWriter bw = tbufInitWriter(NULL, false);
TRY(0) {
@ -5016,14 +5242,14 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
}
tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw);
tSqlExprCompact(pExpr);
tSqlExprCompact(&pCondExpr->pTagCond);
if (ret == TSDB_CODE_SUCCESS) {
ret = validateTagCondExpr(pCmd, p);
}
tSqlExprDestroy(p1);
tExprTreeDestroy(p, NULL);
tExprTreeDestroy(p, NULL); //TODO
taosArrayDestroy(colList);
if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@ -5035,7 +5261,6 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
}
}
pCondExpr->pTagCond = NULL;
return ret;
}
@ -5134,6 +5359,46 @@ int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) {
return TSDB_CODE_SUCCESS;
}
static int32_t getQueryTimeRange(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) {
int32_t ret = TSDB_CODE_SUCCESS;
if (*pExpr == NULL) {
return ret;
}
//multiple tables's query time range mixed together
tExprNode* p = NULL;
SFilterInfo *filter = NULL;
SArray* colList = taosArrayInit(10, sizeof(SColIndex));
ret = exprTreeFromSqlExpr(pCmd, &p, *pExpr, pQueryInfo, colList, NULL);
taosArrayDestroy(colList);
if (ret != TSDB_CODE_SUCCESS) {
goto _ret;
}
ret = filterInitFromTree(p, &filter, FI_OPTION_NO_REWRITE|FI_OPTION_TIMESTAMP);
if (ret != TSDB_CODE_SUCCESS) {
goto _ret;
}
ret = filterGetTimeRange(filter, &pQueryInfo->window);
filterFreeInfo(filter);
_ret:
tExprTreeDestroy(p, NULL);
if (ret) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), tstrerror(ret));
}
return ret;
}
int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) {
if (pExpr == NULL) {
@ -5141,7 +5406,7 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
}
const char* msg1 = "invalid expression";
const char* msg2 = "invalid filter expression";
// const char* msg2 = "invalid filter expression";
int32_t ret = TSDB_CODE_SUCCESS;
@ -5154,14 +5419,30 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
}
int32_t type = 0;
if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId)) != TSDB_CODE_SUCCESS) {
int32_t tbIdx = 0;
int32_t *etype = &type;
#if 0
//DISABLE PARENT CONDITION GROUP TYPE CHECK
if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) {
etype = NULL;
}
#endif
if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, etype, &tbIdx, (*pExpr)->tokenId, &condExpr.pColumnCond, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 && condExpr.pTimewindow != NULL) {
setNormalExprToCond(&condExpr.pColumnCond, condExpr.pTimewindow, TK_AND);
condExpr.pTimewindow = NULL;
}
tSqlExprCompact(pExpr);
// after expression compact, the expression tree is only include tag query condition
condExpr.pTagCond = (*pExpr);
*pExpr = NULL;
// 1. check if it is a join query
if ((ret = validateJoinExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) {
@ -5169,14 +5450,18 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
}
// 2. get the query time range
if ((ret = getTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) {
return ret;
if ((ret = convertTimeRangeFromExpr(&pSql->cmd, pQueryInfo, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
if ((ret = getQueryTimeRange(&pSql->cmd, pQueryInfo, &condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
// 3. get the tag query condition
if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr, pExpr)) != TSDB_CODE_SUCCESS) {
return ret;
if ((ret = getTagQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
// 4. get the table name query condition
@ -5185,15 +5470,14 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
}
// 5. other column query condition
if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) {
if ((ret = checkColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 ) {
if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pTimewindow, TK_AND)) != TSDB_CODE_SUCCESS) {
if ((ret = getColQueryCondExpr(&pSql->cmd, pQueryInfo, &condExpr.pColumnCond)) != TSDB_CODE_SUCCESS) {
goto PARSE_WHERE_EXIT;
}
}
// 6. join condition
if ((ret = getJoinCondInfo(&pSql->cmd, pQueryInfo, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) {
@ -5209,10 +5493,10 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq
goto PARSE_WHERE_EXIT;
}
if (!validateFilterExpr(pQueryInfo)) {
ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2);
goto PARSE_WHERE_EXIT;
}
//if (!validateFilterExpr(pQueryInfo)) {
// ret = invalidOperationMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2);
// goto PARSE_WHERE_EXIT;
//}
//doAddJoinTagsColumnsIntoTagList(&pSql->cmd, pQueryInfo, &condExpr);
if (condExpr.tsJoin) {
@ -5243,7 +5527,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t
* filter primary ts filter expression like:
* where ts in ('2015-12-12 4:8:12')
*/
if (pRight->tokenId == TK_SET || optr == TK_IN) {
if (pRight->tokenId == TK_SET || optr == TK_IN || optr == TK_NE) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
@ -8115,19 +8399,13 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
char name[TSDB_TABLE_FNAME_LEN] = {0};
//if (!pSql->pBuf) {
// if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) {
// code = TSDB_CODE_TSC_OUT_OF_MEMORY;
// goto _end;
// }
//}
plist = taosArrayInit(4, POINTER_BYTES);
pVgroupList = taosArrayInit(4, POINTER_BYTES);
taosArraySort(tableNameList, tnameComparFn);
taosArrayRemoveDuplicate(tableNameList, tnameComparFn, NULL);
STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf);
size_t numOfTables = taosArrayGetSize(tableNameList);
for (int32_t i = 0; i < numOfTables; ++i) {
SName* pname = taosArrayGet(tableNameList, i);
@ -8143,7 +8421,8 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
// avoid mem leak, may should update pTableMeta
void* pVgroupIdList = NULL;
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity);
code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity, (STableMeta **)(&pSTMeta));
pSql->pBuf = (void *)pSTMeta;
// create the child table meta from super table failed, try load it from mnode
if (code != TSDB_CODE_SUCCESS) {
@ -8671,15 +8950,11 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0);
if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) {
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryInfo->colList);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumn* pCols = taosArrayGetP(pQueryInfo->colList, i);
if (pCols->info.flist.numOfFilters > 0) {
if ((pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) || IS_TSWINDOW_SPECIFIED(pQueryInfo->window)) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
}
}
}
// parse the having clause in the first place
if (validateHavingClause(pQueryInfo, pSqlNode->pHaving, pCmd, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery) !=
@ -8778,6 +9053,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, uint64_t *uid) {
tExprNode* pLeft = NULL;
tExprNode* pRight= NULL;
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if (pSqlExpr->pLeft != NULL) {
int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid);
@ -8845,13 +9121,13 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
}
}
} else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd));
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
pQueryInfo->curTableIdx = index.tableIndex;
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta;
int32_t numOfColumns = tscGetNumOfColumns(pTableMeta);
*pExpr = calloc(1, sizeof(tExprNode));
@ -8874,21 +9150,25 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
return TSDB_CODE_SUCCESS;
} else if (pSqlExpr->tokenId == TK_SET) {
int32_t colType = -1;
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
if (pCols != NULL && taosArrayGetSize(pCols) > 0) {
SColIndex* idx = taosArrayGet(pCols, taosArrayGetSize(pCols) - 1);
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx)->pTableMeta;
if (pCols != NULL) {
size_t colSize = taosArrayGetSize(pCols);
if (colSize > 0) {
SColIndex* idx = taosArrayGet(pCols, colSize - 1);
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex);
if (pSchema != NULL) {
colType = pSchema->type;
}
}
}
tVariant *pVal;
if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) {
colType = TSDB_DATA_TYPE_BIGINT;
} else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) {
colType = TSDB_DATA_TYPE_DOUBLE;
}
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pQueryInfo->curTableIdx);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (serializeExprListToVariant(pSqlExpr->Expr.paramList, &pVal, colType, tinfo.precision) == false) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression");
@ -8927,7 +9207,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
// NOTE: binary|nchar data allows the >|< type filter
if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) {
if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) {
if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
}
@ -8982,3 +9262,4 @@ void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) {
#endif

40
src/client/src/tscServer.c
View File

@ -684,7 +684,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo);
int32_t srcColFilterSize = 0;
int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo);
size_t numOfExprs = tscNumOfExprs(pQueryInfo);
@ -695,6 +695,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
int32_t tableSerialize = 0;
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
if (pTableMetaInfo->pVgroupTables != NULL) {
size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
@ -707,8 +708,15 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
tableSerialize = totalTables * sizeof(STableIdInfo);
}
return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize +
exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) {
STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0);
if (pCond != NULL && pCond->cond != NULL) {
srcColFilterSize = pCond->len;
}
}
return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + exprSize + tsBufSize +
tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
}
static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg,
@ -966,10 +974,21 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tableCols[i].colId = htons(pCol->colId);
pQueryMsg->tableCols[i].bytes = htons(pCol->bytes);
pQueryMsg->tableCols[i].type = htons(pCol->type);
pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters);
//pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters);
pQueryMsg->tableCols[i].flist.numOfFilters = 0;
// append the filter information after the basic column information
serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg);
//serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg);
}
if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) {
STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0);
if (pCond != NULL && pCond->cond != NULL) {
pQueryMsg->colCondLen = htons(pCond->len);
memcpy(pMsg, pCond->cond, pCond->len);
pMsg += pCond->len;
}
}
for (int32_t i = 0; i < query.numOfOutput; ++i) {
@ -1044,7 +1063,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
if (pCond != NULL && pCond->cond != NULL) {
pQueryMsg->tagCondLen = htonl(pCond->len);
pQueryMsg->tagCondLen = htons(pCond->len);
memcpy(pMsg, pCond->cond, pCond->len);
pMsg += pCond->len;
@ -2853,18 +2872,19 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool
tNameExtractFullName(&pTableMetaInfo->name, name);
size_t len = strlen(name);
if (pTableMetaInfo->tableMetaCapacity != 0) {
if (pTableMetaInfo->pTableMeta != NULL) {
// just make runtime happy
if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) {
memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity);
}
}
taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity);
STableMeta* pMeta = pTableMetaInfo->pTableMeta;
STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf);
if (pMeta && pMeta->id.uid > 0) {
// in case of child table, here only get the
if (pMeta->tableType == TSDB_CHILD_TABLE) {
int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity);
int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta));
pSql->pBuf = (void *)(pSTMeta);
if (code != TSDB_CODE_SUCCESS) {
return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate);
}

9
src/client/src/tscSubquery.c
View File

@ -796,6 +796,7 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
STimeWindow window = pQueryInfo->window;
tscInitQueryInfo(pQueryInfo);
pQueryInfo->colCond = pSupporter->colCond;
pQueryInfo->window = window;
TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
@ -1883,6 +1884,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
SColumnIndex colIndex = {0};
pSupporter->colCond = pNewQueryInfo->colCond;
pNewQueryInfo->colCond = NULL;
STagCond* pTagCond = &pSupporter->tagCond;
assert(pTagCond->joinInfo.hasJoin);
@ -2319,6 +2323,11 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
goto _error;
}
if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, 0) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
pNewQueryInfo->window = pQueryInfo->window;
pNewQueryInfo->interval = pQueryInfo->interval;
pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow;

360
src/client/src/tscUtil.c
View File

@ -62,11 +62,11 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
break;
case TSDB_DATA_TYPE_FLOAT:
n = sprintf(str, "%f", GET_FLOAT_VAL(buf));
n = sprintf(str, "%e", GET_FLOAT_VAL(buf));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = sprintf(str, "%f", GET_DOUBLE_VAL(buf));
n = sprintf(str, "%e", GET_DOUBLE_VAL(buf));
break;
case TSDB_DATA_TYPE_BINARY:
@ -82,6 +82,22 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
n = bufSize + 2;
break;
case TSDB_DATA_TYPE_UTINYINT:
n = sprintf(str, "%d", *(uint8_t*)buf);
break;
case TSDB_DATA_TYPE_USMALLINT:
n = sprintf(str, "%d", *(uint16_t*)buf);
break;
case TSDB_DATA_TYPE_UINT:
n = sprintf(str, "%u", *(uint32_t*)buf);
break;
case TSDB_DATA_TYPE_UBIGINT:
n = sprintf(str, "%" PRIu64, *(uint64_t*)buf);
break;
default:
tscError("unsupported type:%d", type);
return TSDB_CODE_TSC_INVALID_VALUE;
@ -118,6 +134,24 @@ SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
return NULL;
}
STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx) {
if (filters == NULL) {
return NULL;
}
size_t size = taosArrayGetSize(filters);
for (int32_t i = 0; i < size; ++i) {
STblCond* cond = taosArrayGet(filters, i);
if (uid == cond->uid && (idx >= 0 && cond->idx == idx)) {
return cond;
}
}
return NULL;
}
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
if (tbufTell(bw) == 0) {
return;
@ -753,8 +787,7 @@ typedef struct SDummyInputInfo {
SSDataBlock *block;
STableQueryInfo *pTableQueryInfo;
SSqlObj *pSql; // refactor: remove it
int32_t numOfFilterCols;
SSingleColumnFilterInfo *pFilterInfo;
SFilterInfo *pFilterInfo;
} SDummyInputInfo;
typedef struct SJoinStatus {
@ -770,38 +803,7 @@ typedef struct SJoinOperatorInfo {
SRspResultInfo resultInfo; // todo refactor, add this info for each operator
} SJoinOperatorInfo;
static void converNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t rows, bool *gotNchar) {
for (int32_t i = 0; i < numOfFilterCols; ++i) {
if (pFilterInfo[i].info.type == TSDB_DATA_TYPE_NCHAR) {
pFilterInfo[i].pData2 = pFilterInfo[i].pData;
pFilterInfo[i].pData = malloc(rows * pFilterInfo[i].info.bytes);
int32_t bufSize = pFilterInfo[i].info.bytes - VARSTR_HEADER_SIZE;
for (int32_t j = 0; j < rows; ++j) {
char* dst = (char *)pFilterInfo[i].pData + j * pFilterInfo[i].info.bytes;
char* src = (char *)pFilterInfo[i].pData2 + j * pFilterInfo[i].info.bytes;
int32_t len = 0;
taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
varDataLen(dst) = len;
}
*gotNchar = true;
}
}
}
static void freeNcharFilterColumn(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) {
for (int32_t i = 0; i < numOfFilterCols; ++i) {
if (pFilterInfo[i].info.type == TSDB_DATA_TYPE_NCHAR) {
if (pFilterInfo[i].pData2) {
tfree(pFilterInfo[i].pData);
pFilterInfo[i].pData = pFilterInfo[i].pData2;
pFilterInfo[i].pData2 = NULL;
}
}
}
}
static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) {
static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* pFilterInfo) {
int32_t offset = 0;
char* pData = pRes->data;
@ -817,14 +819,16 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SSingleColumnF
}
// filter data if needed
if (numOfFilterCols > 0) {
doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock);
if (pFilterInfo) {
//doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock);
doSetFilterColInfo(pFilterInfo, pBlock);
bool gotNchar = false;
converNcharFilterColumn(pFilterInfo, numOfFilterCols, pBlock->info.rows, &gotNchar);
filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar);
int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t));
bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p);
//bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p);
bool all = filterExecute(pFilterInfo, pBlock->info.rows, p);
if (gotNchar) {
freeNcharFilterColumn(pFilterInfo, numOfFilterCols);
filterFreeNcharColumns(pFilterInfo);
}
if (!all) {
doCompactSDataBlock(pBlock, pBlock->info.rows, p);
@ -862,7 +866,7 @@ SSDataBlock* doGetDataBlock(void* param, bool* newgroup) {
pBlock->info.rows = pRes->numOfRows;
if (pRes->numOfRows != 0) {
doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols);
doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo);
*newgroup = false;
return pBlock;
}
@ -877,7 +881,7 @@ SSDataBlock* doGetDataBlock(void* param, bool* newgroup) {
}
pBlock->info.rows = pRes->numOfRows;
doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo, pInput->numOfFilterCols);
doSetupSDataBlock(pRes, pBlock, pInput->pFilterInfo);
*newgroup = false;
return pBlock;
}
@ -925,16 +929,31 @@ SSDataBlock* doDataBlockJoin(void* param, bool* newgroup) {
SColumnInfoData* p0 = taosArrayGet(st0->pBlock->pDataBlock, 0);
int64_t* ts0 = (int64_t*) p0->pData;
if (st0->index >= st0->pBlock->info.rows) {
continue;
}
bool prefixEqual = true;
while(1) {
prefixEqual = true;
for (int32_t i = 1; i < pJoinInfo->numOfUpstream; ++i) {
SJoinStatus* st = &pJoinInfo->status[i];
ts0 = (int64_t*) p0->pData;
SColumnInfoData* p = taosArrayGet(st->pBlock->pDataBlock, 0);
int64_t* ts = (int64_t*)p->pData;
if (st->index >= st->pBlock->info.rows || st0->index >= st0->pBlock->info.rows) {
fetchNextBlockIfCompleted(pOperator, newgroup);
if (pOperator->status == OP_EXEC_DONE) {
return pJoinInfo->pRes;
}
prefixEqual = false;
break;
}
if (ts[st->index] < ts0[st0->index]) { // less than the first
prefixEqual = false;
@ -1053,22 +1072,21 @@ static void destroyDummyInputOperator(void* param, int32_t numOfOutput) {
pInfo->block = destroyOutputBuf(pInfo->block);
pInfo->pSql = NULL;
doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols);
filterFreeInfo(pInfo->pFilterInfo);
cleanupResultRowInfo(&pInfo->pTableQueryInfo->resInfo);
tfree(pInfo->pTableQueryInfo);
}
// todo this operator servers as the adapter for Operator tree and SqlRes result, remove it later
SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) {
SOperatorInfo* createDummyInputOperator(SSqlObj* pSql, SSchema* pSchema, int32_t numOfCols, SFilterInfo* pFilters) {
assert(numOfCols > 0);
STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
SDummyInputInfo* pInfo = calloc(1, sizeof(SDummyInputInfo));
pInfo->pSql = pSql;
pInfo->pFilterInfo = pFilterInfo;
pInfo->numOfFilterCols = numOfFilterCols;
pInfo->pFilterInfo = pFilters;
pInfo->pTableQueryInfo = createTmpTableQueryInfo(win);
pInfo->block = calloc(numOfCols, sizeof(SSDataBlock));
@ -1156,6 +1174,7 @@ void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, b
pRes->completed = (pRes->numOfRows == 0);
}
/*
static void createInputDataFilterInfo(SQueryInfo* px, int32_t numOfCol1, int32_t* numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo) {
SColumnInfo* tableCols = calloc(numOfCol1, sizeof(SColumnInfo));
for(int32_t i = 0; i < numOfCol1; ++i) {
@ -1173,6 +1192,7 @@ static void createInputDataFilterInfo(SQueryInfo* px, int32_t numOfCol1, int32_t
tfree(tableCols);
}
*/
void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQueryInfo* px, SSqlObj* pSql) {
SSqlRes* pOutput = &pSql->res;
@ -1201,11 +1221,17 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
// if it is a join query, create join operator here
int32_t numOfCol1 = pTableMeta->tableInfo.numOfColumns;
int32_t numOfFilterCols = 0;
SSingleColumnFilterInfo* pFilterInfo = NULL;
createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols, &pFilterInfo);
SFilterInfo *pFilters = NULL;
STblCond *pCond = NULL;
if (px->colCond) {
pCond = tsGetTableFilter(px->colCond, pTableMeta->id.uid, 0);
if (pCond && pCond->cond) {
createQueryFilter(pCond->cond, pCond->len, &pFilters);
}
//createInputDataFlterInfo(px, numOfCol1, &numOfFilterCols, &pFilterInfo);
}
SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols);
SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilters);
pOutput->precision = pSqlObjList[0]->res.precision;
@ -1222,15 +1248,21 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
for(int32_t i = 1; i < px->numOfTables; ++i) {
STableMeta* pTableMeta1 = tscGetMetaInfo(px, i)->pTableMeta;
numOfCol1 = pTableMeta1->tableInfo.numOfColumns;
SFilterInfo *pFilters1 = NULL;
SSchema* pSchema1 = tscGetTableSchema(pTableMeta1);
int32_t n = pTableMeta1->tableInfo.numOfColumns;
int32_t numOfFilterCols1 = 0;
SSingleColumnFilterInfo* pFilterInfo1 = NULL;
createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols1, &pFilterInfo1);
if (px->colCond) {
pCond = tsGetTableFilter(px->colCond, pTableMeta1->id.uid, i);
if (pCond && pCond->cond) {
createQueryFilter(pCond->cond, pCond->len, &pFilters1);
}
//createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols1, &pFilterInfo1);
}
p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilterInfo1, numOfFilterCols1);
p[i] = createDummyInputOperator(pSqlObjList[i], pSchema1, n, pFilters1);
memcpy(&schema[offset], pSchema1, n * sizeof(SSchema));
offset += n;
}
@ -1776,101 +1808,6 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i
return TSDB_CODE_SUCCESS;
}
static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) {
SSchema* pSchema = pBuilder->pSchema;
char* p = (char*)pBuilder->buf;
int toffset = 0;
uint16_t nCols = pBuilder->nCols;
uint8_t memRowType = payloadType(p);
uint16_t nColsBound = payloadNCols(p);
if (pBuilder->nCols <= 0 || nColsBound <= 0) {
return NULL;
}
char* pVals = POINTER_SHIFT(p, payloadValuesOffset(p));
SMemRow* memRow = (SMemRow)pBuilder->pDataBlock;
memRowSetType(memRow, memRowType);
// ----------------- Raw payload structure for row:
/* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
* | |<----------------- flen ------------->|<--- value part --->|
* |SMemRowType| dataTLen | nCols | colId | colType | offset | ... | value |...|...|... |
* +-----------+----------+----------+--------------------------------------|--------------------|
* | uint8_t | uint32_t | uint16_t | int16_t | uint8_t | uint16_t | ... |.......|...|...|... |
* +-----------+----------+----------+--------------------------------------+--------------------|
* 1. offset in column data tuple starts from the value part in case of uint16_t overflow.
* 2. dataTLen: total length including the header and body.
*/
if (memRowType == SMEM_ROW_DATA) {
SDataRow trow = (SDataRow)memRowDataBody(memRow);
dataRowSetLen(trow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen));
dataRowSetVersion(trow, pBuilder->sversion);
p = (char*)payloadBody(pBuilder->buf);
uint16_t i = 0, j = 0;
while (j < nCols) {
if (i >= nColsBound) {
break;
}
int16_t colId = payloadColId(p);
if (colId == pSchema[j].colId) {
// ASSERT(payloadColType(p) == pSchema[j].type);
tdAppendColVal(trow, POINTER_SHIFT(pVals, payloadColOffset(p)), pSchema[j].type, toffset);
toffset += TYPE_BYTES[pSchema[j].type];
p = payloadNextCol(p);
++i;
++j;
} else if (colId < pSchema[j].colId) {
p = payloadNextCol(p);
++i;
} else {
tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset);
toffset += TYPE_BYTES[pSchema[j].type];
++j;
}
}
while (j < nCols) {
tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset);
toffset += TYPE_BYTES[pSchema[j].type];
++j;
}
#if 0 // no need anymore
while (i < nColsBound) {
p = payloadNextCol(p);
++i;
}
#endif
} else if (memRowType == SMEM_ROW_KV) {
SKVRow kvRow = (SKVRow)memRowKvBody(memRow);
kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound));
kvRowSetNCols(kvRow, nColsBound);
memRowSetKvVersion(memRow, pBuilder->sversion);
p = (char*)payloadBody(pBuilder->buf);
int i = 0;
while (i < nColsBound) {
int16_t colId = payloadColId(p);
uint8_t colType = payloadColType(p);
tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, &toffset);
//toffset += sizeof(SColIdx);
p = payloadNextCol(p);
++i;
}
} else {
ASSERT(0);
}
int32_t rowTLen = memRowTLen(memRow);
pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + rowTLen; // next row
pBuilder->pSubmitBlk->dataLen += rowTLen;
return memRow;
}
// Erase the empty space reserved for binary data
static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam,
SBlockKeyTuple* blkKeyTuple) {
@ -1902,10 +1839,11 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI
int32_t schemaSize = sizeof(STColumn) * numOfCols;
pBlock->schemaLen = schemaSize;
} else {
if (IS_RAW_PAYLOAD(insertParam->payloadType)) {
for (int32_t j = 0; j < tinfo.numOfColumns; ++j) {
flen += TYPE_BYTES[pSchema[j].type];
}
}
pBlock->schemaLen = 0;
}
@ -1932,18 +1870,19 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI
pBlock->dataLen += memRowTLen(memRow);
}
} else {
SMemRowBuilder rowBuilder;
rowBuilder.pSchema = pSchema;
rowBuilder.sversion = pTableMeta->sversion;
rowBuilder.flen = flen;
rowBuilder.nCols = tinfo.numOfColumns;
rowBuilder.pDataBlock = pDataBlock;
rowBuilder.pSubmitBlk = pBlock;
rowBuilder.buf = p;
for (int32_t i = 0; i < numOfRows; ++i) {
rowBuilder.buf = (blkKeyTuple + i)->payloadAddr;
tdGenMemRowFromBuilder(&rowBuilder);
char* payload = (blkKeyTuple + i)->payloadAddr;
if (isNeedConvertRow(payload)) {
convertSMemRow(pDataBlock, payload, pTableDataBlock);
TDRowTLenT rowTLen = memRowTLen(pDataBlock);
pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
pBlock->dataLen += rowTLen;
} else {
TDRowTLenT rowTLen = memRowTLen(payload);
memcpy(pDataBlock, payload, rowTLen);
pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
pBlock->dataLen += rowTLen;
}
}
}
@ -2004,7 +1943,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
if (pBlocks->numOfRows > 0) {
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format
int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta);
int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
STableDataBlocks* dataBuf = NULL;
int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE,
@ -2017,7 +1956,8 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
return ret;
}
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta);
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta);
if (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = (uint32_t)(destSize * 1.5);
@ -2061,7 +2001,9 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
pBlocks->numOfRows, pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey);
}
int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta);
int32_t len = pBlocks->numOfRows *
(isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) +
sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta);
pBlocks->tid = htonl(pBlocks->tid);
pBlocks->uid = htobe64(pBlocks->uid);
@ -2268,6 +2210,11 @@ int32_t tscGetResRowLength(SArray* pExprList) {
}
static void destroyFilterInfo(SColumnFilterList* pFilterList) {
if (pFilterList->filterInfo == NULL) {
pFilterList->numOfFilters = 0;
return;
}
for(int32_t i = 0; i < pFilterList->numOfFilters; ++i) {
if (pFilterList->filterInfo[i].filterstr) {
tfree(pFilterList->filterInfo[i].pz);
@ -2970,6 +2917,64 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) {
return 0;
}
int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx) {
if (src == NULL) {
return 0;
}
size_t s = taosArrayGetSize(src);
*dest = taosArrayInit(s, sizeof(SCond));
for (int32_t i = 0; i < s; ++i) {
STblCond* pCond = taosArrayGet(src, i);
STblCond c = {0};
if (tidx > 0) {
if (!(pCond->uid == uid && pCond->idx == tidx)) {
continue;
}
c.idx = 0;
} else {
c.idx = pCond->idx;
}
c.len = pCond->len;
c.uid = pCond->uid;
if (pCond->len > 0) {
assert(pCond->cond != NULL);
c.cond = malloc(c.len);
if (c.cond == NULL) {
return -1;
}
memcpy(c.cond, pCond->cond, c.len);
}
taosArrayPush(*dest, &c);
}
return 0;
}
void tscColCondRelease(SArray** pCond) {
if (*pCond == NULL) {
return;
}
size_t s = taosArrayGetSize(*pCond);
for (int32_t i = 0; i < s; ++i) {
STblCond* p = taosArrayGet(*pCond, i);
tfree(p->cond);
}
taosArrayDestroy(*pCond);
*pCond = NULL;
}
void tscTagCondRelease(STagCond* pTagCond) {
free(pTagCond->tbnameCond.cond);
@ -3162,6 +3167,7 @@ int32_t tscAddQueryInfo(SSqlCmd* pCmd) {
static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
tscTagCondRelease(&pQueryInfo->tagCond);
tscColCondRelease(&pQueryInfo->colCond);
tscFieldInfoClear(&pQueryInfo->fieldsInfo);
tscExprDestroy(pQueryInfo->exprList);
@ -3252,6 +3258,11 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
goto _error;
}
if (tscColCondCopy(&pQueryInfo->colCond, pSrc->colCond, 0, -1) != 0) {
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
if (pSrc->fillType != TSDB_FILL_NONE) {
pQueryInfo->fillVal = calloc(1, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t));
if (pQueryInfo->fillVal == NULL) {
@ -3647,6 +3658,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
goto _error;
}
if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, tableIndex) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
//just make memory memory sanitizer happy
//refactor later
@ -4448,14 +4464,16 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) {
return cMeta;
}
int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity) {
int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta**ppSTable) {
assert(*ppChild != NULL);
STableMeta* p = NULL;
size_t sz = 0;
STableMeta* p = *ppSTable;
STableMeta* pChild = *ppChild;
size_t sz = (p != NULL) ? tscGetTableMetaSize(p) : 0; //ppSTableBuf actually capacity may larger than sz, dont care
if (p != NULL && sz != 0) {
memset((char *)p, 0, sz);
}
taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz);
*ppSTable = p;
// tableMeta exists, build child table meta according to the super table meta
// the uid need to be checked in addition to the general name of the super table.
@ -4474,10 +4492,8 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name,
memcpy(pChild->schema, p->schema, totalBytes);
*ppChild = pChild;
tfree(p);
return TSDB_CODE_SUCCESS;
} else { // super table has been removed, current tableMeta is also expired. remove it here
tfree(p);
taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
return -1;
}

93
src/common/inc/tdataformat.h
View File

@ -186,6 +186,7 @@ typedef void *SDataRow;
#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535
#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r))
#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
@ -201,14 +202,18 @@ void tdFreeDataRow(SDataRow row);
void tdInitDataRow(SDataRow row, STSchema *pSchema);
SDataRow tdDataRowDup(SDataRow row);
// offset here not include dataRow header length
static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
int32_t offset) {
ASSERT(value != NULL);
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
if (IS_VAR_DATA_TYPE(type)) {
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
if (isCopyVarData) {
memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
}
dataRowLen(row) += varDataTLen(value);
} else {
if (offset == 0) {
@ -223,6 +228,12 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t
return 0;
}
// offset here not include dataRow header length
static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
return tdAppendDataColVal(row, value, true, type, offset);
}
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
if (IS_VAR_DATA_TYPE(type)) {
@ -328,11 +339,10 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle);
void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data
static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
@ -357,10 +367,8 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
}
typedef struct {
int maxRowSize;
int maxCols; // max number of columns
int maxPoints; // max number of points
int numOfRows;
int numOfCols; // Total number of cols
int sversion; // TODO: set sversion
@ -407,7 +415,7 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
}
}
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
SDataCols *tdNewDataCols(int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
int tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
@ -475,9 +483,10 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
}
// offset here not include kvRow header length
static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t *offset) {
static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
int32_t offset) {
ASSERT(value != NULL);
int32_t toffset = *offset + TD_KV_ROW_HEAD_SIZE;
int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE;
SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
@ -485,10 +494,12 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t
pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
if (IS_VAR_DATA_TYPE(type)) {
if (isCopyValData) {
memcpy(ptr, value, varDataTLen(value));
}
kvRowLen(row) += varDataTLen(value);
} else {
if (*offset == 0) {
if (offset == 0) {
ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
@ -497,7 +508,6 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t
}
kvRowLen(row) += TYPE_BYTES[type];
}
*offset += sizeof(SColIdx);
return 0;
}
@ -592,12 +602,24 @@ typedef void *SMemRow;
#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
#define SMEM_ROW_DATA 0U // SDataRow
#define SMEM_ROW_KV 1U // SKVRow
#define SMEM_ROW_DATA 0x0U // SDataRow
#define SMEM_ROW_KV 0x01U // SKVRow
#define SMEM_ROW_CONVERT 0x80U // SMemRow convert flag
#define memRowType(r) (*(uint8_t *)(r))
#define KVRatioKV (0.2f) // all bool
#define KVRatioPredict (0.4f)
#define KVRatioData (0.75f) // all bigint
#define KVRatioConvert (0.9f)
#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory
#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT)) // highest bit
#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
#define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT)
#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag
#define memRowKvBody(r) \
@ -614,6 +636,14 @@ typedef void *SMemRow;
#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen
static FORCE_INLINE char *memRowEnd(SMemRow row) {
if (isDataRow(row)) {
return (char *)dataRowEnd(memRowDataBody(row));
} else {
return (char *)kvRowEnd(memRowKvBody(row));
}
}
#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r))
#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version
@ -631,7 +661,6 @@ typedef void *SMemRow;
} \
} while (0)
#define memRowSetType(r, t) (memRowType(r) = (t))
#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r))
@ -664,12 +693,12 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_
}
}
static FORCE_INLINE int tdAppendMemColVal(SMemRow row, const void *value, int16_t colId, int8_t type, int32_t offset,
int32_t *kvOffset) {
static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
int8_t type, int32_t offset) {
if (isDataRow(row)) {
tdAppendColVal(memRowDataBody(row), value, type, offset);
tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
} else {
tdAppendKvColVal(memRowKvBody(row), value, colId, type, kvOffset);
tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
}
return 0;
}
@ -691,6 +720,30 @@ static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value
return len;
}
/**
* 1. calculate the delta of AllNullLen for SDataRow.
* 2. calculate the real len for SKVRow.
*/
static FORCE_INLINE void tdGetColAppendDeltaLen(const void *value, int8_t colType, int32_t *dataLen, int32_t *kvLen) {
switch (colType) {
case TSDB_DATA_TYPE_BINARY: {
int32_t varLen = varDataLen(value);
*dataLen += (varLen - CHAR_BYTES);
*kvLen += (varLen + sizeof(SColIdx));
break;
}
case TSDB_DATA_TYPE_NCHAR: {
int32_t varLen = varDataLen(value);
*dataLen += (varLen - TSDB_NCHAR_SIZE);
*kvLen += (varLen + sizeof(SColIdx));
break;
}
default: {
*kvLen += (TYPE_BYTES[colType] + sizeof(SColIdx));
break;
}
}
}
typedef struct {
int16_t colId;
@ -706,7 +759,7 @@ static FORCE_INLINE void setSColInfo(SColInfo* colInfo, int16_t colId, uint8_t c
SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
#if 0
// ----------------- Raw payload structure for row:
/* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
* | |<----------------- flen ------------->|<--- value part --->|
@ -752,6 +805,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); }
#endif
#ifdef __cplusplus
}
#endif

2
src/common/inc/tvariant.h
View File

@ -53,6 +53,8 @@ int32_t tVariantToString(tVariant *pVar, char *dst);
int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix);
int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo);
int32_t tVariantTypeSetType(tVariant *pVariant, char type);
#ifdef __cplusplus

62
src/common/src/tdataformat.c
View File

@ -19,10 +19,10 @@
#include "wchar.h"
#include "tarray.h"
static void dataColSetNEleNull(SDataCol *pCol, int nEle);
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows, bool forceSetNull);
//TODO: change caller to use return val
int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
int spaceNeeded = pCol->bytes * maxPoints;
if(IS_VAR_DATA_TYPE(pCol->type)) {
@ -31,7 +31,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
if(pCol->spaceSize < spaceNeeded) {
void* ptr = realloc(pCol->pData, spaceNeeded);
if(ptr == NULL) {
uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize,
uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded,
strerror(errno));
return -1;
} else {
@ -239,20 +239,19 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
pDataCol->len = 0;
}
// value from timestamp should be TKEY here instead of TSKEY
void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
if (isAllRowsNull(pCol)) {
if (isNull(value, pCol->type)) {
// all null value yet, just return
return;
return 0;
}
if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
if (numOfRows > 0) {
// Find the first not null value, fill all previouse values as NULL
dataColSetNEleNull(pCol, numOfRows, maxPoints);
} else {
tdAllocMemForCol(pCol, maxPoints);
dataColSetNEleNull(pCol, numOfRows);
}
}
@ -268,12 +267,21 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes;
}
return 0;
}
static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
} else {
return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
}
}
bool isNEleNull(SDataCol *pCol, int nEle) {
if(isAllRowsNull(pCol)) return true;
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
}
return true;
}
@ -290,9 +298,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
}
}
void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
tdAllocMemForCol(pCol, maxPoints);
static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->len = 0;
for (int i = 0; i < nEle; i++) {
@ -318,7 +324,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) {
}
}
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
SDataCols *tdNewDataCols(int maxCols, int maxRows) {
SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols));
if (pCols == NULL) {
uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno));
@ -326,6 +332,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
}
pCols->maxPoints = maxRows;
pCols->maxCols = maxCols;
pCols->numOfRows = 0;
pCols->numOfCols = 0;
if (maxCols > 0) {
pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
@ -342,13 +351,8 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
pCols->cols[i].pData = NULL;
pCols->cols[i].dataOff = NULL;
}
pCols->maxCols = maxCols;
}
pCols->maxRowSize = maxRowSize;
return pCols;
}
@ -357,8 +361,9 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
int oldMaxCols = pCols->maxCols;
if (schemaNCols(pSchema) > oldMaxCols) {
pCols->maxCols = schemaNCols(pSchema);
pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
if (pCols->cols == NULL) return -1;
void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
if (ptr == NULL) return -1;
pCols->cols = ptr;
for(i = oldMaxCols; i < pCols->maxCols; i++) {
pCols->cols[i].pData = NULL;
pCols->cols[i].dataOff = NULL;
@ -366,10 +371,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
}
}
if (schemaTLen(pSchema) > pCols->maxRowSize) {
pCols->maxRowSize = schemaTLen(pSchema);
}
tdResetDataCols(pCols);
pCols->numOfCols = schemaNCols(pSchema);
@ -398,7 +399,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
}
SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints);
SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
if (pRet == NULL) return NULL;
pRet->numOfCols = pDataCols->numOfCols;
@ -413,7 +414,10 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
if (keepData) {
if (pDataCols->cols[i].len > 0) {
tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints);
if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
tdFreeDataCols(pRet);
return NULL;
}
pRet->cols[i].len = pDataCols->cols[i].len;
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
@ -584,9 +588,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0 && (forceSetNull || (!forceSetNull && !isNull(src2->cols[i].pData, src2->cols[i].type)))) {
if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
target->maxPoints);
} else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
target->maxPoints);
}
}
target->numOfRows++;
@ -844,7 +851,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
int16_t k;
for (k = 0; k < nKvNCols; ++k) {
SColInfo *pColInfo = taosArrayGet(stashRow, k);
tdAppendKvColVal(kvRow, pColInfo->colVal, pColInfo->colId, pColInfo->colType, &toffset);
tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset);
toffset += sizeof(SColIdx);
}
ASSERT(kvLen == memRowTLen(tRow));
}

181
src/common/src/texpr.c
View File

@ -118,7 +118,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
} else if (pNode->nodeType == TSQL_NODE_VALUE) {
tVariantDestroy(pNode->pVal);
} else if (pNode->nodeType == TSQL_NODE_COL) {
free(pNode->pSchema);
tfree(pNode->pSchema);
}
free(pNode);
@ -435,7 +435,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
expr->_node.optr = TSDB_RELATION_IN;
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
right->pVal = pVal;
pVal->nType = TSDB_DATA_TYPE_ARRAY;
pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY;
pVal->arr = taosArrayInit(2, POINTER_BYTES);
const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN;
@ -502,6 +502,183 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) {
*q = (void *)pObj;
}
void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) {
SBufferReader br = tbufInitReader(buf, len, false);
uint32_t sType = tbufReadUint32(&br);
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType));
int dummy = -1;
tVariant tmpVar = {0};
size_t t = 0;
int32_t sz = tbufReadInt32(&br);
void *pvar = NULL;
int64_t val = 0;
int32_t bufLen = 0;
if (IS_NUMERIC_TYPE(sType)) {
bufLen = 60; // The maximum length of string that a number is converted to.
} else {
bufLen = 128;
}
char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE);
for (int32_t i = 0; i < sz; i++) {
switch (sType) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
*(uint8_t *)&val = (uint8_t)tbufReadInt64(&br);
t = sizeof(val);
pvar = &val;
break;
}
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
*(uint16_t *)&val = (uint16_t)tbufReadInt64(&br);
t = sizeof(val);
pvar = &val;
break;
}
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_INT: {
*(uint32_t *)&val = (uint32_t)tbufReadInt64(&br);
t = sizeof(val);
pvar = &val;
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_BIGINT: {
*(uint64_t *)&val = (uint64_t)tbufReadInt64(&br);
t = sizeof(val);
pvar = &val;
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
*(double *)&val = tbufReadDouble(&br);
t = sizeof(val);
pvar = &val;
break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(float *)&val = (float)tbufReadDouble(&br);
t = sizeof(val);
pvar = &val;
break;
}
case TSDB_DATA_TYPE_BINARY: {
pvar = (char *)tbufReadBinary(&br, &t);
break;
}
case TSDB_DATA_TYPE_NCHAR: {
pvar = (char *)tbufReadBinary(&br, &t);
break;
}
default:
taosHashCleanup(pObj);
*q = NULL;
return;
}
tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType);
if (bufLen < t) {
tmp = realloc(tmp, t * TSDB_NCHAR_SIZE);
bufLen = (int32_t)t;
}
switch (tType) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
goto err_ret;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
goto err_ret;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_INT: {
if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
goto err_ret;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_BIGINT: {
if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
goto err_ret;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
goto err_ret;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
goto err_ret;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_BINARY: {
if (tVariantDump(&tmpVar, tmp, tType, true)) {
goto err_ret;
}
t = varDataLen(tmp);
pvar = varDataVal(tmp);
break;
}
case TSDB_DATA_TYPE_NCHAR: {
if (tVariantDump(&tmpVar, tmp, tType, true)) {
goto err_ret;
}
t = varDataLen(tmp);
pvar = varDataVal(tmp);
break;
}
default:
goto err_ret;
}
taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy));
tVariantDestroy(&tmpVar);
memset(&tmpVar, 0, sizeof(tmpVar));
}
*q = (void *)pObj;
pObj = NULL;
err_ret:
tVariantDestroy(&tmpVar);
taosHashCleanup(pObj);
tfree(tmp);
}
tExprNode* exprdup(tExprNode* pNode) {
if (pNode == NULL) {
return NULL;

2
src/common/src/tname.c
View File

@ -61,7 +61,7 @@ bool tscValidateTableNameLength(size_t len) {
// TODO refactor
SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) {
if (numOfFilters == 0) {
if (numOfFilters == 0 || src == NULL) {
assert(src == NULL);
return NULL;
}

103
src/common/src/ttypes.c
View File

@ -372,21 +372,21 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
}
tDataTypeDescriptor tDataTypes[15] = {
{TSDB_DATA_TYPE_NULL, 6,1, "NOTYPE", NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint, getStatics_i64},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", tsCompressTinyint, tsDecompressTinyint, getStatics_u8},
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", tsCompressSmallint, tsDecompressSmallint, getStatics_u16},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", tsCompressBigint, tsDecompressBigint, getStatics_u64},
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8},
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64},
};
char tTokenTypeSwitcher[13] = {
@ -405,6 +405,32 @@ char tTokenTypeSwitcher[13] = {
TSDB_DATA_TYPE_NCHAR, // TK_NCHAR
};
float floatMin = -FLT_MAX, floatMax = FLT_MAX;
double doubleMin = -DBL_MAX, doubleMax = DBL_MAX;
FORCE_INLINE void* getDataMin(int32_t type) {
switch (type) {
case TSDB_DATA_TYPE_FLOAT:
return &floatMin;
case TSDB_DATA_TYPE_DOUBLE:
return &doubleMin;
default:
return &tDataTypes[type].minValue;
}
}
FORCE_INLINE void* getDataMax(int32_t type) {
switch (type) {
case TSDB_DATA_TYPE_FLOAT:
return &floatMax;
case TSDB_DATA_TYPE_DOUBLE:
return &doubleMax;
default:
return &tDataTypes[type].maxValue;
}
}
bool isValidDataType(int32_t type) {
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT;
}
@ -566,6 +592,53 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
}
}
void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
if (optr == TSDB_BINARY_OP_ADD) {
switch (type) {
case TSDB_DATA_TYPE_TINYINT:
*((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);
break;
case TSDB_DATA_TYPE_UTINYINT:
*((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2);
break;
case TSDB_DATA_TYPE_SMALLINT:
*((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2);
break;
case TSDB_DATA_TYPE_USMALLINT:
*((uint16_t *)dst) = GET_UINT16_VAL(s1) + GET_UINT16_VAL(s2);
break;
case TSDB_DATA_TYPE_INT:
*((int32_t *)dst) = GET_INT32_VAL(s1) + GET_INT32_VAL(s2);
break;
case TSDB_DATA_TYPE_UINT:
*((uint32_t *)dst) = GET_UINT32_VAL(s1) + GET_UINT32_VAL(s2);
break;
case TSDB_DATA_TYPE_BIGINT:
*((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2);
break;
case TSDB_DATA_TYPE_UBIGINT:
*((uint64_t *)dst) = GET_UINT64_VAL(s1) + GET_UINT64_VAL(s2);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
*((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2);
break;
case TSDB_DATA_TYPE_FLOAT:
SET_FLOAT_VAL(dst, GET_FLOAT_VAL(s1) + GET_FLOAT_VAL(s2));
break;
case TSDB_DATA_TYPE_DOUBLE:
SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2));
break;
default: {
assert(0);
break;
}
}
} else {
assert(0);
}
}
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) {
switch (type) {
case TSDB_DATA_TYPE_INT:

96
src/common/src/tvariant.c
View File

@ -23,6 +23,13 @@
#include "tutil.h"
#include "tvariant.h"
#define SET_EXT_INFO(converted, res, minv, maxv, exti) do { \
if (converted == NULL || exti == NULL || *converted == false) { break; } \
if ((res) < (minv)) { *exti = -1; break; } \
if ((res) > (maxv)) { *exti = 1; break; } \
assert(0); \
} while (0)
void tVariantCreate(tVariant *pVar, SStrToken *token) {
int32_t ret = 0;
int32_t type = token->type;
@ -184,7 +191,7 @@ void tVariantDestroy(tVariant *pVar) {
}
// NOTE: this is only for string array
if (pVar->nType == TSDB_DATA_TYPE_ARRAY) {
if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) {
size_t num = taosArrayGetSize(pVar->arr);
for(size_t i = 0; i < num; i++) {
void* p = taosArrayGetP(pVar->arr, i);
@ -192,6 +199,9 @@ void tVariantDestroy(tVariant *pVar) {
}
taosArrayDestroy(pVar->arr);
pVar->arr = NULL;
} else if (pVar->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
taosArrayDestroy(pVar->arr);
pVar->arr = NULL;
}
}
@ -220,7 +230,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
pDst->i64 = pSrc->i64;
} else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array
} else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) { // this is only for string array
size_t num = taosArrayGetSize(pSrc->arr);
pDst->arr = taosArrayInit(num, sizeof(char*));
for(size_t i = 0; i < num; i++) {
@ -228,9 +238,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
char* n = strdup(p);
taosArrayPush(pDst->arr, &n);
}
} else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
size_t num = taosArrayGetSize(pSrc->arr);
pDst->arr = taosArrayInit(num, sizeof(int64_t));
pDst->nLen = pSrc->nLen;
assert(pSrc->nLen == num);
for(size_t i = 0; i < num; i++) {
int64_t *p = taosArrayGet(pSrc->arr, i);
taosArrayPush(pDst->arr, p);
}
}
if (pDst->nType != TSDB_DATA_TYPE_ARRAY) {
if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) {
pDst->nLen = tDataTypes[pDst->nType].bytes;
}
}
@ -450,7 +469,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val
return 0;
}
static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr) {
static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) {
if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
setNull((char *)result, type, tDataTypes[type].bytes);
return 0;
@ -540,6 +559,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
}
}
if (converted) {
*converted = true;
}
bool code = false;
uint64_t ui = 0;
@ -602,6 +625,18 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) {
* to column type defined in schema
*/
int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) {
return tVariantDumpEx(pVariant, payload, type, includeLengthPrefix, NULL, NULL);
}
/*
* transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType
* to column type defined in schema
*/
int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) {
if (converted) {
*converted = false;
}
if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
return -1;
}
@ -620,7 +655,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_TINYINT: {
if (convertToInteger(pVariant, &result, type, true, false) < 0) {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo);
return -1;
}
*((int8_t *)payload) = (int8_t) result;
@ -628,7 +664,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_UTINYINT: {
if (convertToInteger(pVariant, &result, type, false, false) < 0) {
if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo);
return -1;
}
*((uint8_t *)payload) = (uint8_t) result;
@ -636,7 +673,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_SMALLINT: {
if (convertToInteger(pVariant, &result, type, true, false) < 0) {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo);
return -1;
}
*((int16_t *)payload) = (int16_t)result;
@ -644,7 +682,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_USMALLINT: {
if (convertToInteger(pVariant, &result, type, false, false) < 0) {
if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
SET_EXT_INFO(converted, result, 0, UINT16_MAX - 1, extInfo);
return -1;
}
*((uint16_t *)payload) = (uint16_t)result;
@ -652,7 +691,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_INT: {
if (convertToInteger(pVariant, &result, type, true, false) < 0) {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo);
return -1;
}
*((int32_t *)payload) = (int32_t)result;
@ -660,7 +700,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_UINT: {
if (convertToInteger(pVariant, &result, type, false, false) < 0) {
if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
SET_EXT_INFO(converted, result, 0, UINT32_MAX - 1, extInfo);
return -1;
}
*((uint32_t *)payload) = (uint32_t)result;
@ -668,7 +709,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_BIGINT: {
if (convertToInteger(pVariant, &result, type, true, false) < 0) {
if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo);
return -1;
}
*((int64_t *)payload) = (int64_t)result;
@ -676,7 +718,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
}
case TSDB_DATA_TYPE_UBIGINT: {
if (convertToInteger(pVariant, &result, type, false, false) < 0) {
if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo);
return -1;
}
*((uint64_t *)payload) = (uint64_t)result;
@ -696,11 +739,37 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
return -1;
}
if (converted) {
*converted = true;
}
if (value > FLT_MAX || value < -FLT_MAX) {
SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo);
return -1;
}
SET_FLOAT_VAL(payload, value);
}
} else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
if (converted) {
*converted = true;
}
if (pVariant->i64 > FLT_MAX || pVariant->i64 < -FLT_MAX) {
SET_EXT_INFO(converted, pVariant->i64, -FLT_MAX, FLT_MAX, extInfo);
return -1;
}
SET_FLOAT_VAL(payload, pVariant->i64);
} else if (IS_FLOAT_TYPE(pVariant->nType)) {
if (converted) {
*converted = true;
}
if (pVariant->dKey > FLT_MAX || pVariant->dKey < -FLT_MAX) {
SET_EXT_INFO(converted, pVariant->dKey, -FLT_MAX, FLT_MAX, extInfo);
return -1;
}
SET_FLOAT_VAL(payload, pVariant->dKey);
} else if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
*((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL;
@ -824,6 +893,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
return 0;
}
/*
* In variant, bool/smallint/tinyint/int/bigint share the same attribution of
* structure, also ignore the convert the type required
@ -848,7 +918,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT: {
convertToInteger(pVariant, &(pVariant->i64), type, true, true);
convertToInteger(pVariant, &(pVariant->i64), type, true, true, NULL);
pVariant->nType = TSDB_DATA_TYPE_BIGINT;
break;
}

1
src/connector/jdbc/pom.xml
View File

@ -113,7 +113,6 @@
</includes>
<excludes>
<exclude>**/AppMemoryLeakTest.java</exclude>
<exclude>**/AuthenticationTest.java</exclude>
<exclude>**/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java</exclude>
<exclude>**/DatetimeBefore1970Test.java</exclude>
<exclude>**/FailOverTest.java</exclude>

7
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
View File

@ -14,6 +14,8 @@
*****************************************************************************/
package com.taosdata.jdbc;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.sql.*;
import java.util.*;
import java.util.logging.Logger;
@ -127,6 +129,11 @@ public class TSDBDriver extends AbstractDriver {
return null;
}
if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED);
if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED);
try {
TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE),
(String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE));

2
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java
View File

@ -33,6 +33,8 @@ public class TSDBError {
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type in tdengine");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PRECISION, "unknown timestamp precision");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, "user is required");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, "password is required");
TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error");

5
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java
View File

@ -29,6 +29,9 @@ public class TSDBErrorNumbers {
public static final int ERROR_UNKNOWN_TIMESTAMP_PRECISION = 0x2316; // unknown timestamp precision
public static final int ERROR_RESTFul_Client_Protocol_Exception = 0x2317;
public static final int ERROR_RESTFul_Client_IOException = 0x2318;
public static final int ERROR_USER_IS_REQUIRED = 0x2319; // user is required
public static final int ERROR_PASSWORD_IS_REQUIRED = 0x231a; // password is required
public static final int ERROR_UNKNOWN = 0x2350; //unknown error
@ -67,6 +70,8 @@ public class TSDBErrorNumbers {
errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE);
errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PRECISION);
errorNumbers.add(ERROR_RESTFul_Client_IOException);
errorNumbers.add(ERROR_USER_IS_REQUIRED);
errorNumbers.add(ERROR_PASSWORD_IS_REQUIRED);
errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception);

1
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
View File

@ -36,7 +36,6 @@ public class TSDBJNIConnector {
static {
System.loadLibrary("taos");
System.out.println("java.library.path:" + System.getProperty("java.library.path"));
}
public boolean isClosed() {

10
src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java
View File

@ -7,6 +7,7 @@ import com.taosdata.jdbc.utils.HttpClientPoolUtil;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.sql.*;
import java.util.Properties;
import java.util.logging.Logger;
@ -40,8 +41,13 @@ public class RestfulDriver extends AbstractDriver {
String loginUrl = "http://" + host + ":" + port + "/rest/login/" + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + "/" + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD) + "";
try {
String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), "UTF-8");
String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), "UTF-8");
if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED);
if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED);
String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), StandardCharsets.UTF_8.displayName());
String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), StandardCharsets.UTF_8.displayName());
loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + "";
} catch (UnsupportedEncodingException e) {
e.printStackTrace();

57
src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java
View File

@ -7,6 +7,7 @@ import com.taosdata.jdbc.AbstractStatement;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.TSDBError;
import com.taosdata.jdbc.TSDBErrorNumbers;
import com.taosdata.jdbc.enums.TimestampFormat;
import com.taosdata.jdbc.utils.HttpClientPoolUtil;
import com.taosdata.jdbc.utils.SqlSyntaxValidator;
@ -45,9 +46,7 @@ public class RestfulStatement extends AbstractStatement {
if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql);
final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
return executeOneUpdate(url, sql);
return executeOneUpdate(sql);
}
@Override
@ -62,34 +61,25 @@ public class RestfulStatement extends AbstractStatement {
public boolean execute(String sql) throws SQLException {
if (isClosed())
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
if (!SqlSyntaxValidator.isValidForExecute(sql))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: " + sql);
//如果执行了use操作应该将当前Statement的catalog设置为新的database
boolean result = true;
String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("TIMESTAMP")) {
url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
}
if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("UTC")) {
url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
}
if (SqlSyntaxValidator.isUseSql(sql)) {
HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
this.database = sql.trim().replace("use", "").trim();
this.conn.setCatalog(this.database);
result = false;
} else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) {
executeOneQuery(sql);
} else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) {
executeOneUpdate(url, sql);
executeOneUpdate(sql);
result = false;
} else {
if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) {
executeQuery(sql);
executeOneQuery(sql);
} else {
executeUpdate(sql);
executeOneUpdate(sql);
result = false;
}
}
@ -97,19 +87,25 @@ public class RestfulStatement extends AbstractStatement {
return result;
}
private ResultSet executeOneQuery(String sql) throws SQLException {
if (!SqlSyntaxValidator.isValidForExecuteQuery(sql))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql);
// row data
String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
String timestampFormat = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT);
if ("TIMESTAMP".equalsIgnoreCase(timestampFormat))
private String getUrl() throws SQLException {
TimestampFormat timestampFormat = TimestampFormat.valueOf(conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).trim().toUpperCase());
String url;
switch (timestampFormat) {
case TIMESTAMP:
url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
if ("UTC".equalsIgnoreCase(timestampFormat))
break;
case UTC:
url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
break;
default:
url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
}
return url;
}
String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
private ResultSet executeOneQuery(String sql) throws SQLException {
// row data
String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
JSONObject resultJson = JSON.parseObject(result);
if (resultJson.getString("status").equals("error")) {
throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc"));
@ -119,11 +115,8 @@ public class RestfulStatement extends AbstractStatement {
return resultSet;
}
private int executeOneUpdate(String url, String sql) throws SQLException {
if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql);
String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
private int executeOneUpdate(String sql) throws SQLException {
String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
JSONObject jsonObject = JSON.parseObject(result);
if (jsonObject.getString("status").equals("error")) {
throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc"));
@ -134,7 +127,7 @@ public class RestfulStatement extends AbstractStatement {
}
private int getAffectedRows(JSONObject jsonObject) throws SQLException {
// create ... SQLs should return 0 , and Restful result is this:
// create ... SQLs should return 0 , and Restful result like this:
// {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1}
JSONArray head = jsonObject.getJSONArray("head");
if (head.size() != 1 || !"affected_rows".equals(head.getString(0)))

15
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java
View File

@ -16,8 +16,7 @@ package com.taosdata.jdbc.utils;
public class SqlSyntaxValidator {
private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe", "reset"};
private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"};
private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set", "reset"};
private static final String[] querySQL = {"select", "show", "describe"};
private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"};
@ -38,14 +37,6 @@ public class SqlSyntaxValidator {
return false;
}
public static boolean isValidForExecute(String sql) {
for (String prefix : SQL) {
if (sql.trim().toLowerCase().startsWith(prefix))
return true;
}
return false;
}
public static boolean isDatabaseUnspecifiedQuery(String sql) {
for (String databaseObj : databaseUnspecifiedShow) {
if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*"))
@ -63,9 +54,5 @@ public class SqlSyntaxValidator {
return sql.trim().toLowerCase().startsWith("use");
}
public static boolean isSelectSql(String sql) {
return sql.trim().toLowerCase().startsWith("select");
}
}

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java
View File

@ -69,6 +69,8 @@ public class SubscribeTest {
@Before
public void createDatabase() throws SQLException {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

44
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java
View File

@ -1,6 +1,9 @@
package com.taosdata.jdbc.cases;
import com.taosdata.jdbc.TSDBErrorNumbers;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.sql.*;
@ -12,6 +15,47 @@ public class AuthenticationTest {
private static final String password = "taos?data";
private Connection conn;
@Test
public void connectWithoutUserByJni() {
try {
DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?");
} catch (SQLException e) {
Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode());
Assert.assertEquals("ERROR (2319): user is required", e.getMessage());
}
}
@Test
public void connectWithoutUserByRestful() {
try {
DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?");
} catch (SQLException e) {
Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode());
Assert.assertEquals("ERROR (2319): user is required", e.getMessage());
}
}
@Test
public void connectWithoutPasswordByJni() {
try {
DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root");
} catch (SQLException e) {
Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode());
Assert.assertEquals("ERROR (231a): password is required", e.getMessage());
}
}
@Test
public void connectWithoutPasswordByRestful() {
try {
DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root");
} catch (SQLException e) {
Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode());
Assert.assertEquals("ERROR (231a): password is required", e.getMessage());
}
}
@Ignore
@Test
public void test() {
// change password

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java
View File

@ -29,6 +29,8 @@ public class BatchInsertTest {
public void before() {
try {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java
View File

@ -21,6 +21,8 @@ public class ImportTest {
public static void before() {
try {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

41
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java
View File

@ -270,6 +270,41 @@ public class InsertSpecialCharacterJniTest {
}
}
@Ignore
@Test
public void testSingleQuotaEscape() throws SQLException {
final long now = System.currentTimeMillis();
final String sql = "insert into t? using ? tags(?) values(?, ?, ?) t? using " + tbname2 + " tags(?) values(?,?,?) ";
try (PreparedStatement pstmt = conn.prepareStatement(sql)) {
// t1
pstmt.setInt(1, 1);
pstmt.setString(2, tbname2);
pstmt.setString(3, special_character_str_5);
pstmt.setTimestamp(4, new Timestamp(now));
pstmt.setBytes(5, special_character_str_5.getBytes());
// t2
pstmt.setInt(7, 2);
pstmt.setString(8, special_character_str_5);
pstmt.setTimestamp(9, new Timestamp(now));
pstmt.setString(11, special_character_str_5);
int ret = pstmt.executeUpdate();
Assert.assertEquals(2, ret);
}
String query = "select * from ?.t? where ? < ? and ts >= ? and f1 is not null";
try (PreparedStatement pstmt = conn.prepareStatement(query)) {
pstmt.setString(1, dbName);
pstmt.setInt(2, 1);
pstmt.setString(3, "ts");
pstmt.setTimestamp(4, new Timestamp(System.currentTimeMillis()));
pstmt.setTimestamp(5, new Timestamp(0));
ResultSet rs = pstmt.executeQuery();
Assert.assertNotNull(rs);
}
}
@Test
public void testCase10() throws SQLException {
final long now = System.currentTimeMillis();
@ -293,13 +328,12 @@ public class InsertSpecialCharacterJniTest {
Assert.assertEquals(2, ret);
}
//query t1
String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null";
String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null";
try (PreparedStatement pstmt = conn.prepareStatement(query)) {
pstmt.setString(1, dbName);
pstmt.setInt(2, 1);
pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis()));
pstmt.setTimestamp(4, new Timestamp(0));
pstmt.setString(5, "f1");
ResultSet rs = pstmt.executeQuery();
rs.next();
@ -311,12 +345,11 @@ public class InsertSpecialCharacterJniTest {
Assert.assertNull(f2);
}
// query t2
query = "select * from t? where ts < ? and ts >= ? and ? is not null";
query = "select * from t? where ts < ? and ts >= ? and f2 is not null";
try (PreparedStatement pstmt = conn.prepareStatement(query)) {
pstmt.setInt(1, 2);
pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
pstmt.setTimestamp(3, new Timestamp(0));
pstmt.setString(4, "f2");
ResultSet rs = pstmt.executeQuery();
rs.next();

6
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java
View File

@ -293,13 +293,12 @@ public class InsertSpecialCharacterRestfulTest {
Assert.assertEquals(2, ret);
}
//query t1
String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null";
String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null";
try (PreparedStatement pstmt = conn.prepareStatement(query)) {
pstmt.setString(1, dbName);
pstmt.setInt(2, 1);
pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis()));
pstmt.setTimestamp(4, new Timestamp(0));
pstmt.setString(5, "f1");
ResultSet rs = pstmt.executeQuery();
rs.next();
@ -311,12 +310,11 @@ public class InsertSpecialCharacterRestfulTest {
Assert.assertNull(f2);
}
// query t2
query = "select * from t? where ts < ? and ts >= ? and ? is not null";
query = "select * from t? where ts < ? and ts >= ? and f2 is not null";
try (PreparedStatement pstmt = conn.prepareStatement(query)) {
pstmt.setInt(1, 2);
pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
pstmt.setTimestamp(3, new Timestamp(0));
pstmt.setString(4, "f2");
ResultSet rs = pstmt.executeQuery();
rs.next();

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java
View File

@ -22,6 +22,8 @@ public class QueryDataTest {
public void createDatabase() {
try {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

62
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java
View File

@ -1,51 +1,49 @@
package com.taosdata.jdbc.cases;
import com.taosdata.jdbc.TSDBDriver;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.sql.*;
import java.util.Properties;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
public class ResetQueryCacheTest {
static Connection connection;
static Statement statement;
static String host = "127.0.0.1";
@Test
public void jni() throws SQLException {
// given
Connection connection = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:0/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8");
Statement statement = connection.createStatement();
@Before
public void init() {
try {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
statement = connection.createStatement();
} catch (SQLException e) {
return;
}
// when
boolean execute = statement.execute("reset query cache");
// then
assertFalse(execute);
assertEquals(0, statement.getUpdateCount());
statement.close();
connection.close();
}
@Test
public void testResetQueryCache() throws SQLException {
String resetSql = "reset query cache";
statement.execute(resetSql);
}
public void restful() throws SQLException {
// given
Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://127.0.0.1:6041/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8");
Statement statement = connection.createStatement();
// when
boolean execute = statement.execute("reset query cache");
// then
assertFalse(execute);
assertEquals(0, statement.getUpdateCount());
@After
public void close() {
try {
if (statement != null)
statement.close();
if (connection != null)
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java
View File

@ -20,6 +20,8 @@ public class SelectTest {
public void createDatabaseAndTable() {
try {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

2
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java
View File

@ -24,6 +24,8 @@ public class StableTest {
public static void createDatabase() {
try {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

21
src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java
View File

@ -1,21 +0,0 @@
package com.taosdata.jdbc.utils;
import org.junit.Assert;
import org.junit.Test;
public class SqlSyntaxValidatorTest {
@Test
public void isSelectSQL() {
Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather"));
Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather"));
Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather "));
Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)"));
}
@Test
public void isUseSQL() {
Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test"));
}
}

1
src/inc/taosdef.h
View File

@ -34,6 +34,7 @@ extern "C" {
#define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX})
#define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN})
#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
#define TSKEY_INITIAL_VAL INT64_MIN

3
src/inc/taoserror.h
View File

@ -272,7 +272,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query")
#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached")
#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica")
#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) //"System error")
#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition")
#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error")
// grant

3
src/inc/taosmsg.h
View File

@ -489,7 +489,8 @@ typedef struct {
int16_t numOfCols; // the number of columns will be load from vnode
SInterval interval;
SSessionWindow sw; // session window
uint32_t tagCondLen; // tag length in current query
uint16_t tagCondLen; // tag length in current query
uint16_t colCondLen; // column length in current query
uint32_t tbnameCondLen; // table name filter condition string length
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;

8
src/inc/ttype.h
View File

@ -47,7 +47,8 @@ typedef struct {
// this data type is internally used only in 'in' query to hold the values
#define TSDB_DATA_TYPE_ARRAY (1000)
#define TSDB_DATA_TYPE_POINTER_ARRAY (1000)
#define TSDB_DATA_TYPE_VALUE_ARRAY (1001)
#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
do { \
@ -181,6 +182,8 @@ typedef struct tDataTypeDescriptor {
int16_t nameLen;
int32_t bytes;
char * name;
int64_t minValue;
int64_t maxValue;
int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize);
int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
@ -200,6 +203,9 @@ const void *getNullValue(int32_t type);
void assignVal(char *val, const char *src, int32_t len, int32_t type);
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf);
void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
void* getDataMin(int32_t type);
void* getDataMax(int32_t type);
int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned);

32
src/plugins/http/src/httpParser.c
View File

@ -101,13 +101,17 @@ char *httpGetStatusDesc(int32_t statusCode) {
}
static void httpCleanupString(HttpString *str) {
if (str->str) {
free(str->str);
str->str = NULL;
str->pos = 0;
str->size = 0;
}
}
static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) {
char *new_str = NULL;
if (str->size == 0) {
str->pos = 0;
str->size = len + 1;
@ -115,7 +119,16 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) {
} else if (str->pos + len + 1 >= str->size) {
str->size += len;
str->size *= 4;
str->str = realloc(str->str, str->size);
new_str = realloc(str->str, str->size);
if (new_str == NULL && str->str) {
// if str->str was not NULL originally,
// the old allocated memory was left unchanged,
// see man 3 realloc
free(str->str);
}
str->str = new_str;
} else {
}
@ -326,6 +339,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) {
}
int32_t newSize = buf->pos + len + 1;
char *newStr = NULL;
if (newSize >= buf->size) {
if (buf->size >= HTTP_BUFFER_SIZE) {
httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size);
@ -336,7 +350,12 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) {
newSize = MAX(newSize, HTTP_BUFFER_INIT);
newSize *= 4;
newSize = MIN(newSize, HTTP_BUFFER_SIZE);
buf->str = realloc(buf->str, newSize);
newStr = realloc(buf->str, newSize);
if (newStr == NULL && buf->str) {
free(buf->str);
}
buf->str = newStr;
buf->size = newSize;
if (buf->str == NULL) {
@ -374,13 +393,20 @@ static HTTP_PARSER_STATE httpTopStack(HttpParser *parser) {
static int32_t httpPushStack(HttpParser *parser, HTTP_PARSER_STATE state) {
HttpStack *stack = &parser->stacks;
int8_t *newStacks = NULL;
if (stack->size == 0) {
stack->pos = 0;
stack->size = 32;
stack->stacks = malloc(stack->size * sizeof(int8_t));
} else if (stack->pos + 1 > stack->size) {
stack->size *= 2;
stack->stacks = realloc(stack->stacks, stack->size * sizeof(int8_t));
newStacks = realloc(stack->stacks, stack->size * sizeof(int8_t));
if (newStacks == NULL && stack->stacks) {
free(stack->stacks);
}
stack->stacks = newStacks;
} else {
}

16
src/plugins/http/src/httpUtil.c
View File

@ -188,13 +188,17 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (cmdSize > HTTP_MAX_CMD_SIZE) {
if (cmdSize <= 0 || cmdSize > HTTP_MAX_CMD_SIZE) {
httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user,
cmdSize, HTTP_MAX_CMD_SIZE);
return false;
}
multiCmds->cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd));
HttpSqlCmd *new_cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd));
if (new_cmds == NULL && multiCmds->cmds) {
free(multiCmds->cmds);
}
multiCmds->cmds = new_cmds;
if (multiCmds->cmds == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize);
return false;
@ -208,13 +212,17 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) {
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (bufferSize > HTTP_MAX_BUFFER_SIZE) {
if (bufferSize <= 0 || bufferSize > HTTP_MAX_BUFFER_SIZE) {
httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd,
pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE);
return false;
}
multiCmds->buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize);
char *new_buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize);
if (new_buffer == NULL && multiCmds->buffer) {
free(multiCmds->buffer);
}
multiCmds->buffer = new_buffer;
if (multiCmds->buffer == NULL) {
httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize);
return false;

10
src/query/inc/qExecutor.h
View File

@ -252,7 +252,9 @@ typedef struct SQueryAttr {
int32_t numOfFilterCols;
int64_t* fillVal;
SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query.
SSingleColumnFilterInfo* pFilterInfo;
SFilterInfo *pFilters;
void* tsdb;
SMemRef memRef;
@ -382,6 +384,7 @@ typedef struct SQInfo {
typedef struct SQueryParam {
char *sql;
char *tagCond;
char *colCond;
char *tbnameCond;
char *prevResult;
SArray *pTableIdList;
@ -390,6 +393,8 @@ typedef struct SQueryParam {
SExprInfo *pExprs;
SExprInfo *pSecExprs;
SFilterInfo *pFilters;
SColIndex *pGroupColIndex;
SColumnInfo *pTagColumnInfo;
SGroupbyExpr *pGroupbyExpr;
@ -573,6 +578,7 @@ SSDataBlock* doSLimit(void* param, bool* newgroup);
int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
void doSetFilterColInfo(SFilterInfo *pFilters, SSDataBlock* pBlock);
bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
@ -594,9 +600,11 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo);
int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters);
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
int32_t prevResultLen, void* merger);

324
src/query/inc/qFilter.h Normal file
View File

@ -0,0 +1,324 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QFILTER_H
#define TDENGINE_QFILTER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "texpr.h"
#include "hash.h"
#include "tname.h"
#define FILTER_DEFAULT_GROUP_SIZE 4
#define FILTER_DEFAULT_UNIT_SIZE 4
#define FILTER_DEFAULT_FIELD_SIZE 4
#define FILTER_DEFAULT_VALUE_SIZE 4
#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2
#define FILTER_DUMMY_EMPTY_OPTR 127
#define FILTER_DUMMY_RANGE_OPTR 126
#define MAX_NUM_STR_SIZE 40
enum {
FLD_TYPE_COLUMN = 1,
FLD_TYPE_VALUE = 2,
FLD_TYPE_MAX = 3,
FLD_DESC_NO_FREE = 4,
FLD_DATA_NO_FREE = 8,
FLD_DATA_IS_HASH = 16,
};
enum {
MR_ST_START = 1,
MR_ST_FIN = 2,
MR_ST_ALL = 4,
MR_ST_EMPTY = 8,
};
enum {
RANGE_FLG_EXCLUDE = 1,
RANGE_FLG_INCLUDE = 2,
RANGE_FLG_NULL = 4,
};
enum {
FI_OPTION_NO_REWRITE = 1,
FI_OPTION_TIMESTAMP = 2,
FI_OPTION_NEED_UNIQE = 4,
};
enum {
FI_STATUS_ALL = 1,
FI_STATUS_EMPTY = 2,
FI_STATUS_REWRITE = 4,
FI_STATUS_CLONED = 8,
};
enum {
RANGE_TYPE_UNIT = 1,
RANGE_TYPE_VAR_HASH = 2,
RANGE_TYPE_MR_CTX = 3,
};
typedef struct OptrStr {
uint16_t optr;
char *str;
} OptrStr;
typedef struct SFilterRange {
int64_t s;
int64_t e;
char sflag;
char eflag;
} SFilterRange;
typedef struct SFilterColRange {
uint16_t idx; //column field idx
bool isNull;
bool notNull;
bool isRange;
SFilterRange ra;
} SFilterColRange;
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t*);
typedef struct SFilterRangeCompare {
int64_t s;
int64_t e;
rangeCompFunc func;
} SFilterRangeCompare;
typedef struct SFilterRangeNode {
struct SFilterRangeNode* prev;
struct SFilterRangeNode* next;
union {
SFilterRange ra;
SFilterRangeCompare rc;
};
} SFilterRangeNode;
typedef struct SFilterRangeCtx {
int32_t type;
int32_t options;
int8_t status;
bool isnull;
bool notnull;
bool isrange;
int16_t colId;
__compar_fn_t pCompareFunc;
SFilterRangeNode *rf; //freed
SFilterRangeNode *rs;
} SFilterRangeCtx ;
typedef struct SFilterVarCtx {
int32_t type;
int32_t options;
int8_t status;
bool isnull;
bool notnull;
bool isrange;
SHashObj *wild;
SHashObj *value;
} SFilterVarCtx;
typedef struct SFilterField {
uint16_t flag;
void* desc;
void* data;
} SFilterField;
typedef struct SFilterFields {
uint16_t size;
uint16_t num;
SFilterField *fields;
} SFilterFields;
typedef struct SFilterFieldId {
uint16_t type;
uint16_t idx;
} SFilterFieldId;
typedef struct SFilterGroup {
uint16_t unitSize;
uint16_t unitNum;
uint16_t *unitIdxs;
uint8_t *unitFlags; // !unit result
} SFilterGroup;
typedef struct SFilterColInfo {
uint8_t type;
int32_t dataType;
void *info;
} SFilterColInfo;
typedef struct SFilterGroupCtx {
uint16_t colNum;
uint16_t *colIdx;
SFilterColInfo *colInfo;
} SFilterGroupCtx;
typedef struct SFilterColCtx {
uint16_t colIdx;
void* ctx;
} SFilterColCtx;
typedef struct SFilterCompare {
uint8_t type;
uint8_t optr;
uint8_t optr2;
} SFilterCompare;
typedef struct SFilterUnit {
SFilterCompare compare;
SFilterFieldId left;
SFilterFieldId right;
SFilterFieldId right2;
} SFilterUnit;
typedef struct SFilterComUnit {
void *colData;
void *valData;
void *valData2;
uint16_t dataSize;
uint8_t dataType;
uint8_t optr;
int8_t func;
int8_t rfunc;
} SFilterComUnit;
typedef struct SFilterPCtx {
SHashObj *valHash;
SHashObj *unitHash;
} SFilterPCtx;
typedef struct SFilterInfo {
uint32_t options;
uint32_t status;
uint16_t unitSize;
uint16_t unitNum;
uint16_t groupNum;
uint16_t colRangeNum;
SFilterFields fields[FLD_TYPE_MAX];
SFilterGroup *groups;
uint16_t *cgroups;
SFilterUnit *units;
SFilterComUnit *cunits;
uint8_t *unitRes; // result
uint8_t *unitFlags; // got result
SFilterRangeCtx **colRange;
filter_exec_func func;
SFilterPCtx pctx;
} SFilterInfo;
#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0)
#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0)
#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true)
#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true)))
#define FILTER_GET_FLAG(st, f) (st & f)
#define FILTER_SET_FLAG(st, f) st |= (f)
#define FILTER_CLR_FLAG(st, f) st &= (~f)
#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0)
#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0)
#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0)
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#define CHK_RETV(c) do { if (c) { return; } } while (0)
#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
#define CHK_LRET(c, r,...) do { if (c) { qError(__VA_ARGS__); return r; } } while (0)
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)
#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags))
#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1
#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx])
#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx])
#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v)
#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0)
#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL)
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p);
extern int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QFILTER_H

17
src/query/inc/qTableMeta.h
View File

@ -3,6 +3,7 @@
#include "tsdb.h" //todo tsdb should not be here
#include "qSqlparser.h"
#include "qFilter.h"
typedef struct SFieldInfo {
int16_t numOfOutput; // number of column in result
@ -16,6 +17,14 @@ typedef struct SCond {
char * cond;
} SCond;
typedef struct STblCond {
uint64_t uid;
int16_t idx; //table index
int32_t len; // length of tag query condition data
char * cond;
} STblCond;
typedef struct SJoinNode {
uint64_t uid;
int16_t tagColId;
@ -89,6 +98,11 @@ typedef struct STableMetaInfo {
struct SQInfo; // global merge operator
struct SQueryAttr; // query object
typedef struct STableFilter {
uint64_t uid;
SFilterInfo info;
} STableFilter;
typedef struct SQueryInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately.
uint32_t type; // query/insert type
@ -106,8 +120,11 @@ typedef struct SQueryInfo {
SLimitVal slimit;
STagCond tagCond;
SArray * colCond;
SOrderVal order;
int16_t numOfTables;
int16_t curTableIdx;
STableMetaInfo **pTableMetaInfo;
struct STSBuf *tsBuf;

186
src/query/src/qExecutor.c
View File

@ -2545,77 +2545,14 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i
#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR)
static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) {
static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) {
SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) {
return true;
}
bool ret = true;
for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) {
SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k];
int32_t index = -1;
for(int32_t i = 0; i < pQueryAttr->numOfCols; ++i) {
if (pDataStatis[i].colId == pFilterInfo->info.colId) {
index = i;
break;
}
}
// no statistics data, load the true data block
if (index == -1) {
if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) {
return true;
}
// not support pre-filter operation on binary/nchar data type
if (!IS_PREFILTER_TYPE(pFilterInfo->info.type)) {
return true;
}
// all data in current column are NULL, no need to check its boundary value
if (pDataStatis[index].numOfNull == numOfRows) {
// if isNULL query exists, load the null data column
for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) {
SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j];
if (pFilterElem->fp == isNullOperator) {
return true;
}
}
continue;
}
SDataStatis* pDataBlockst = &pDataStatis[index];
if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) {
float minval = (float)(*(double *)(&pDataBlockst->min));
float maxval = (float)(*(double *)(&pDataBlockst->max));
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
if (ret == false) {
return false;
}
}
} else {
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
if (ret == false) {
return false;
}
}
}
}
return ret;
return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows);
}
static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {
@ -2834,6 +2771,49 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf
tfree(p);
}
void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) {
int32_t numOfRows = pBlock->info.rows;
int8_t *p = calloc(numOfRows, sizeof(int8_t));
bool all = true;
if (pRuntimeEnv->pTsBuf != NULL) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
TSKEY* k = (TSKEY*) pColInfoData->pData;
for (int32_t i = 0; i < numOfRows; ++i) {
int32_t offset = ascQuery? i:(numOfRows - i - 1);
int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery);
if (ret == TS_JOIN_TAG_NOT_EQUALS) {
break;
} else if (ret == TS_JOIN_TS_NOT_EQUALS) {
all = false;
continue;
} else {
assert(ret == TS_JOIN_TS_EQUAL);
p[offset] = true;
}
if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) {
break;
}
}
// save the cursor status
pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
} else {
all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, p);
}
if (!all) {
doCompactSDataBlock(pBlock, numOfRows, p);
}
tfree(p);
}
static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId);
static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes);
@ -2875,6 +2855,15 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi
}
}
void doSetFilterColInfo(SFilterInfo * pFilters, SSDataBlock* pBlock) {
for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j);
filterSetColFieldData(pFilters, pColInfo->info.colId, pColInfo->pData);
}
}
int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) {
*status = BLK_DATA_NO_NEEDED;
@ -2911,7 +2900,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
// Calculate all time windows that are overlapping or contain current data block.
// If current data block is contained by all possible time window, do not load current data block.
if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 ||
if (pQueryAttr->pFilters || pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 ||
(QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) {
(*status) = BLK_DATA_ALL_NEEDED;
}
@ -3013,9 +3002,12 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
return terrno;
}
doSetFilterColumnInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols, pBlock);
if (pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) {
filterRowsInDataBlock(pRuntimeEnv, pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols, pBlock, ascQuery);
if (pQueryAttr->pFilters != NULL) {
doSetFilterColInfo(pQueryAttr->pFilters, pBlock);
}
if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery);
}
}
@ -6899,7 +6891,8 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols);
pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput);
pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols);
pQueryMsg->tagCondLen = htonl(pQueryMsg->tagCondLen);
pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen);
pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen);
pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset);
pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen);
pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks);
@ -6930,7 +6923,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
pColInfo->colId = htons(pColInfo->colId);
pColInfo->type = htons(pColInfo->type);
pColInfo->bytes = htons(pColInfo->bytes);
pColInfo->flist.numOfFilters = htons(pColInfo->flist.numOfFilters);
pColInfo->flist.numOfFilters = 0;
if (!isValidDataType(pColInfo->type)) {
qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type);
@ -6938,6 +6931,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
goto _cleanup;
}
/*
int32_t numOfFilters = pColInfo->flist.numOfFilters;
if (numOfFilters > 0) {
pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo));
@ -6951,8 +6945,21 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
if (code != TSDB_CODE_SUCCESS) {
goto _cleanup;
}
*/
}
if (pQueryMsg->colCondLen > 0) {
param->colCond = calloc(1, pQueryMsg->colCondLen);
if (param->colCond == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _cleanup;
}
memcpy(param->colCond, pMsg, pQueryMsg->colCondLen);
pMsg += pQueryMsg->colCondLen;
}
param->tableScanOperator = pQueryMsg->tableScanOperator;
param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES);
if (param->pExpr == NULL) {
@ -7526,6 +7533,28 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
return TSDB_CODE_SUCCESS;
}
int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) {
tExprNode* expr = NULL;
TRY(TSDB_MAX_TAG_CONDITIONS) {
expr = exprTreeFromBinary(data, len);
} CATCH( code ) {
CLEANUP_EXECUTE();
return code;
} END_TRY
if (expr == NULL) {
qError("failed to create expr tree");
return TSDB_CODE_QRY_APP_ERROR;
}
int32_t ret = filterInitFromTree(expr, pFilters, 0);
tExprTreeDestroy(expr, NULL);
return ret;
}
// todo refactor
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExpr, SExprInfo* prevExpr, SUdfInfo *pUdfInfo) {
@ -7675,7 +7704,7 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil
int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) {
for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) {
if (pQueryAttr->tableCols[i].flist.numOfFilters > 0) {
if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) {
pQueryAttr->numOfFilterCols++;
}
}
@ -7758,7 +7787,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) {
}
SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs,
SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId,
SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId,
char* sql, uint64_t qId, SUdfInfo* pUdfInfo) {
int16_t numOfCols = pQueryMsg->numOfCols;
int16_t numOfOutput = pQueryMsg->numOfOutput;
@ -7810,6 +7839,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
pQueryAttr->needReverseScan = pQueryMsg->needReverseScan;
pQueryAttr->stateWindow = pQueryMsg->stateWindow;
pQueryAttr->vgId = vgId;
pQueryAttr->pFilters = pFilters;
pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo));
if (pQueryAttr->tableCols == NULL) {
@ -7843,10 +7873,6 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
}
doUpdateExprColumnIndex(pQueryAttr);
int32_t ret = createFilterInfo(pQueryAttr, pQInfo->qId);
if (ret != TSDB_CODE_SUCCESS) {
goto _cleanup;
}
if (pSecExprs != NULL) {
int32_t resultRowSize = 0;
@ -7961,6 +7987,8 @@ _cleanup_qinfo:
tfree(pExprs);
filterFreeInfo(pFilters);
_cleanup:
freeQInfo(pQInfo);
return NULL;
@ -8316,6 +8344,8 @@ void freeQueryAttr(SQueryAttr* pQueryAttr) {
taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo);
tfree(pQueryAttr->pGroupbyExpr);
}
filterFreeInfo(pQueryAttr->pFilters);
}
}

3076
src/query/src/qFilter.c Normal file
View File

File diff suppressed because it is too large Load Diff

5
src/query/src/qSqlParser.c
View File

@ -423,6 +423,9 @@ tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) {
pExpr->pRight = tSqlExprClone(pSrc->pRight);
}
memset(&pExpr->value, 0, sizeof(pExpr->value));
tVariantAssign(&pExpr->value, &pSrc->value);
//we don't clone paramList now because clone is only used for between/and
assert(pSrc->Expr.paramList == NULL);
return pExpr;
@ -478,9 +481,7 @@ static void doDestroySqlExprNode(tSqlExpr *pExpr) {
return;
}
if (pExpr->tokenId == TK_STRING) {
tVariantDestroy(&pExpr->value);
}
tSqlExprListDestroy(pExpr->Expr.paramList);
free(pExpr);

18
src/query/src/queryMain.c
View File

@ -103,6 +103,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
}
}
if (param.colCond != NULL) {
if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, &param.pFilters)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code);
if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
goto _over;
@ -162,13 +168,19 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
assert(pQueryMsg->stableQuery == isSTableQuery);
(*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, vgId, param.sql, qId, param.pUdfInfo);
param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo);
param.sql = NULL;
param.pExprs = NULL;
param.pSecExprs = NULL;
param.pGroupbyExpr = NULL;
param.pTagColumnInfo = NULL;
param.pFilters = NULL;
if ((*pQInfo) == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _over;
}
param.pUdfInfo = NULL;
code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
@ -178,6 +190,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
taosArrayDestroy(param.pGroupbyExpr->columnInfo);
}
tfree(param.colCond);
destroyUdfInfo(param.pUdfInfo);
taosArrayDestroy(param.pTableIdList);
@ -190,6 +204,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters);
}
filterFreeInfo(param.pFilters);
//pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
if (code != TSDB_CODE_SUCCESS) {
*pQInfo = NULL;

1
src/query/tests/CMakeLists.txt
View File

@ -27,3 +27,4 @@ SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w)

367
src/query/tests/rangeMergeTest.cpp Normal file
View File

@ -0,0 +1,367 @@
#include <gtest/gtest.h>
#include <iostream>
#include "qResultbuf.h"
#include "taos.h"
#include "taosdef.h"
#include "qFilter.h"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
extern "C" {
extern void* filterInitRangeCtx(int32_t type, int32_t options);
extern int32_t filterGetRangeNum(void* h, int32_t* num);
extern int32_t filterGetRangeRes(void* h, SFilterRange *ra);
extern int32_t filterFreeRangeCtx(void* h);
extern int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr);
}
namespace {
void intDataTest() {
printf("running %s\n", __FUNCTION__);
int32_t asize = 0;
SFilterRange ra[10] = {0};
int64_t *s =NULL;
int64_t *e =NULL;
int64_t s0[3] = {-100, 1, 3};
int64_t e0[3] = {0 , 2, 4};
int64_t s1[3] = {INT64_MIN, 0 , 3};
int64_t e1[3] = {100 , 50, 4};
int64_t s2[5] = {1 , 3 , 10,30,70};
int64_t e2[5] = {10, 100, 20,50,120};
int64_t s3[3] = {1 , 20 , 5};
int64_t e3[3] = {10, 100, 25};
int64_t s4[2] = {10, 0};
int64_t e4[2] = {20, 5};
int64_t s5[3] = {0, 6 ,7};
int64_t e5[3] = {4, 10,20};
int64_t rs[10];
int64_t re[10];
int32_t num = 0;
void *h = NULL;
s = s0;
e = e0;
asize = sizeof(s0)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 3);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, -100);
ASSERT_EQ(ra[0].e, 0);
ASSERT_EQ(ra[1].s, 1);
ASSERT_EQ(ra[1].e, 2);
ASSERT_EQ(ra[2].s, 3);
ASSERT_EQ(ra[2].e, 4);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, FI_OPTION_TIMESTAMP);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, -100);
ASSERT_EQ(ra[0].e, 4);
filterFreeRangeCtx(h);
s = s1;
e = e1;
asize = sizeof(s1)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 3);
ASSERT_EQ(ra[0].e, 4);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, INT64_MIN);
ASSERT_EQ(ra[0].e, 100);
filterFreeRangeCtx(h);
s = s2;
e = e2;
asize = sizeof(s2)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 1);
ASSERT_EQ(ra[0].e, 120);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, i % 2 ? TSDB_RELATION_OR : TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, i % 2 ? TSDB_RELATION_AND : TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 70);
ASSERT_EQ(ra[0].e, 120);
filterFreeRangeCtx(h);
s = s3;
e = e3;
asize = sizeof(s3)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 1);
ASSERT_EQ(ra[0].e, 100);
filterFreeRangeCtx(h);
s = s4;
e = e4;
asize = sizeof(s4)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 2);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 0);
ASSERT_EQ(ra[0].e, 5);
ASSERT_EQ(ra[1].s, 10);
ASSERT_EQ(ra[1].e, 20);
filterFreeRangeCtx(h);
s = s5;
e = e5;
asize = sizeof(s5)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 2);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 0);
ASSERT_EQ(ra[0].e, 4);
ASSERT_EQ(ra[1].s, 6);
ASSERT_EQ(ra[1].e, 20);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].s = s[i];
ra[0].e = e[i];
filterAddRange(h, ra, (i == (asize -1)) ? TSDB_RELATION_AND : TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 1);
filterGetRangeRes(h, ra);
ASSERT_EQ(ra[0].s, 7);
ASSERT_EQ(ra[0].e, 10);
filterFreeRangeCtx(h);
int64_t s6[2] = {0, 4};
int64_t e6[2] = {4, 6};
s = s6;
e = e6;
asize = sizeof(s6)/sizeof(s[0]);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].eflag = 1;
ra[1].sflag = 4;
ra[i].s = s[i];
ra[i].e = e[i];
filterAddRange(h, ra + i, TSDB_RELATION_AND);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 0);
filterFreeRangeCtx(h);
memset(ra, 0, sizeof(ra));
h = filterInitRangeCtx(TSDB_DATA_TYPE_BIGINT, 0);
for (int32_t i = 0; i < asize; ++i) {
ra[0].eflag = 1;
ra[1].sflag = 1;
ra[i].s = s[i];
ra[i].e = e[i];
filterAddRange(h, ra + i, TSDB_RELATION_OR);
}
filterGetRangeNum(h, &num);
ASSERT_EQ(num, 2);
ASSERT_EQ(ra[0].s, 0);
ASSERT_EQ(ra[0].e, 4);
ASSERT_EQ(ra[0].eflag, 1);
ASSERT_EQ(ra[1].s, 4);
ASSERT_EQ(ra[1].e, 6);
ASSERT_EQ(ra[1].sflag, 1);
filterFreeRangeCtx(h);
}
} // namespace
TEST(testCase, rangeMergeTest) {
intDataTest();
}

5
src/tsdb/src/tsdbCommit.c
View File

@ -722,7 +722,7 @@ static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) {
return -1;
}
pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
if (pCommith->pDataCols == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyCommitH(pCommith);
@ -920,7 +920,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo
SDataCol * pDataCol = pDataCols->cols + ncol;
SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull;
// if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it
if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it
continue;
}
@ -1277,6 +1276,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
if (key1 < key2) {
for (int i = 0; i < pDataCols->numOfCols; i++) {
//TODO: dataColAppendVal may fail
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
@ -1308,6 +1308,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
ASSERT(!isRowDel);
for (int i = 0; i < pDataCols->numOfCols; i++) {
//TODO: dataColAppendVal may fail
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}

2
src/tsdb/src/tsdbCompact.c
View File

@ -296,7 +296,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
return -1;
}
pComph->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
if (pComph->pDataCols == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyCompactH(pComph);

22
src/tsdb/src/tsdbMemTable.c
View File

@ -702,11 +702,12 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
}
//row1 has higher priority
static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, STSchema **ppSchema1, STSchema **ppSchema2, STable* pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) {
static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo,
STSchema **ppSchema1, STSchema **ppSchema2,
STable* pTable, int32_t* pPoints, SMemRow* pLastRow) {
//for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows!
if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) {
(*pAffectedRows)++;
(*pPoints)++;
return NULL;
}
@ -715,7 +716,6 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep
void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1));
if(pMem == NULL) return NULL;
memRowCpy(pMem, row1);
(*pAffectedRows)++;
(*pPoints)++;
*pLastRow = pMem;
return pMem;
@ -750,18 +750,16 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep
if(pMem == NULL) return NULL;
memRowCpy(pMem, tmp);
(*pAffectedRows)++;
(*pPoints)++;
*pLastRow = pMem;
return pMem;
}
static void* tsdbInsertDupKeyMergePacked(void** args) {
return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7], args[8]);
return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]);
}
static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) {
static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) {
if(pSkipList->insertHandleFn == NULL) {
tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9);
@ -769,17 +767,16 @@ static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STa
dupHandleSavedFunc->args[3] = NULL;
dupHandleSavedFunc->args[4] = NULL;
dupHandleSavedFunc->args[5] = pTable;
dupHandleSavedFunc->args[6] = pAffectedRows;
dupHandleSavedFunc->args[7] = pPoints;
dupHandleSavedFunc->args[8] = pLastRow;
pSkipList->insertHandleFn = dupHandleSavedFunc;
}
pSkipList->insertHandleFn->args[6] = pPoints;
pSkipList->insertHandleFn->args[7] = pLastRow;
}
static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
int64_t points = 0;
int32_t points = 0;
STable *pTable = NULL;
SSubmitBlkIter blkIter = {0};
SMemTable *pMemTable = NULL;
@ -830,9 +827,10 @@ static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *
SMemRow lastRow = NULL;
int64_t osize = SL_SIZE(pTableData->pData);
tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, pAffectedRows, &points, &lastRow);
tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow);
tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext);
int64_t dsize = SL_SIZE(pTableData->pData) - osize;
(*pAffectedRows) += points;
if(lastRow != NULL) {

1
src/tsdb/src/tsdbMeta.c
View File

@ -17,7 +17,6 @@
#define TSDB_SUPER_TABLE_SL_LEVEL 5
#define DEFAULT_TAG_INDEX_COLUMN 0
static int tsdbCompareSchemaVersion(const void *key1, const void *key2);
static char * getTagIndexKey(const void *pData);
static STable *tsdbNewTable();
static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable);

6
src/tsdb/src/tsdbRead.c
View File

@ -466,7 +466,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL);
pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock);
pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock);
if (pQueryHandle->pDataCols == NULL) {
tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@ -1446,7 +1446,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
return midPos;
}
int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
char* pData = NULL;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
@ -1481,7 +1481,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
}
if (pColInfo->info.colId == src->colId) {
if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) {
if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
memmove(pData, (char*)src->pData + bytes * start, bytes * num);
} else { // handle the var-string

10
src/tsdb/src/tsdbReadImpl.c
View File

@ -42,14 +42,14 @@ int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) {
return -1;
}
pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
if (pReadh->pDCols[0] == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyReadH(pReadh);
return -1;
}
pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
if (pReadh->pDCols[1] == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbDestroyReadH(pReadh);
@ -463,7 +463,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
SDataCol *pDataCol = &(pDataCols->cols[dcol]);
if (dcol != 0 && ccol >= pBlockData->numOfCols) {
// Set current column as NULL and forward
dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
dataColReset(pDataCol);
dcol++;
continue;
}
@ -503,7 +503,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
ccol++;
} else {
// Set current column as NULL and forward
dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
dataColReset(pDataCol);
dcol++;
}
}
@ -608,7 +608,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
}
if (pBlockCol == NULL) {
dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
dataColReset(pDataCol);
continue;
}

6
src/util/inc/hash.h
View File

@ -42,7 +42,7 @@ typedef struct SHashNode {
#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode));
#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode)))
typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0,
@ -170,6 +170,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p);
void taosHashCancelIterate(SHashObj *pHashObj, void *p);
void *taosHashGetDataKey(SHashObj *pHashObj, void *data);
uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data);
#ifdef __cplusplus
}
#endif

32
src/util/inc/tcompare.h
View File

@ -53,6 +53,38 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr);
int32_t taosArrayCompareString(const void* a, const void* b);
int32_t setCompareBytes1(const void *pLeft, const void *pRight);
int32_t setCompareBytes2(const void *pLeft, const void *pRight);
int32_t setCompareBytes4(const void *pLeft, const void *pRight);
int32_t setCompareBytes8(const void *pLeft, const void *pRight);
int32_t compareInt32Val(const void *pLeft, const void *pRight);
int32_t compareInt64Val(const void *pLeft, const void *pRight);
int32_t compareInt16Val(const void *pLeft, const void *pRight);
int32_t compareInt8Val(const void *pLeft, const void *pRight);
int32_t compareUint32Val(const void *pLeft, const void *pRight);
int32_t compareUint64Val(const void *pLeft, const void *pRight);
int32_t compareUint16Val(const void *pLeft, const void *pRight);
int32_t compareUint8Val(const void* pLeft, const void* pRight);
int32_t compareFloatVal(const void *pLeft, const void *pRight);
int32_t compareDoubleVal(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
#ifdef __cplusplus
}
#endif

11
src/util/src/hash.c
View File

@ -776,6 +776,17 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
}
FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) {
SHashNode * node = GET_HASH_PNODE(data);
return GET_HASH_NODE_KEY(node);
}
FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) {
SHashNode * node = GET_HASH_PNODE(data);
return node->keyLen;
}
// release the pNode, return next pNode, and lock the current entry
static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {

53
src/util/src/tcompare.c
View File

@ -19,6 +19,22 @@
#include "tarray.h"
#include "hash.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
}
int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
}
int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
}
int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
}
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
if (left > right) return 1;
@ -48,21 +64,21 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) {
}
int32_t compareUint32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint64Val(const void *pLeft, const void *pRight) {
int64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint16Val(const void *pLeft, const void *pRight) {
int16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
@ -262,7 +278,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
char pattern[128] = {0};
@ -290,11 +306,11 @@ int32_t taosArrayCompareString(const void* a, const void* b) {
// const SArray* arr = (const SArray*) pRight;
// return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1;
//}
static int32_t compareFindItemInSet(const void *pLeft, const void* pRight) {
int32_t compareFindItemInSet(const void *pLeft, const void* pRight) {
return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
}
static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'};
wchar_t pattern[128] = {0};
@ -310,6 +326,29 @@ static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
__compar_fn_t comparFn = NULL;
if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT:
return setCompareBytes1;
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT:
return setCompareBytes2;
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_FLOAT:
return setCompareBytes4;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_TIMESTAMP:
return setCompareBytes8;
default:
assert(0);
}
}
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break;
@ -334,6 +373,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_NCHAR: {
if (optr == TSDB_RELATION_LIKE) {
comparFn = compareWStrPatternComp;
} else if (optr == TSDB_RELATION_IN) {
comparFn = compareFindItemInSet;
} else {
comparFn = compareLenPrefixedWStr;
}

1
src/util/src/terror.c
View File

@ -280,6 +280,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION, "One valid time range condition expected")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error")

2
tests/pytest/query/queryBetweenAnd.py
View File

@ -101,7 +101,7 @@ class TDTestCase:
# tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}")
# tdSql.checkRows(1)
tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}")
tdSql.checkRows(0)
tdSql.checkRows(2)
tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}")
# tdSql.checkRows(3)

217
tests/pytest/query/queryWildcardLength.py Normal file
View File

@ -0,0 +1,217 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from copy import deepcopy
import string
import random
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def cleanTb(self):
query_sql = "show stables"
res_row_list = tdSql.query(query_sql, True)
stb_list = map(lambda x: x[0], res_row_list)
for stb in stb_list:
tdSql.execute(f'drop table if exists {stb}')
query_sql = "show tables"
res_row_list = tdSql.query(query_sql, True)
tb_list = map(lambda x: x[0], res_row_list)
for tb in tb_list:
tdSql.execute(f'drop table if exists {tb}')
def getLongWildcardStr(self, len=None):
"""
generate long wildcard str
"""
maxWildCardsLength = int(tdSql.getVariable('maxWildCardsLength')[0])
if len:
chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len))
else:
chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(maxWildCardsLength+1))
return chars
def genTableName(self):
'''
generate table name
hp_name--->'%str'
lp_name--->'str%'
ul_name--->'st_r'
'''
table_name = self.getLongWildcardStr()
table_name_list = list(table_name)
table_name_list.pop(-1)
if len(table_name_list) > 1:
lp_name = deepcopy(table_name_list)
lp_name[-1] = '%'
lp_name = ''.join(lp_name)
ul_name = list(lp_name)
ul_name[int(len(ul_name)/2)] = '_'
ul_name = ''.join(ul_name)
table_name_list = list(table_name)
hp_name = deepcopy(table_name_list)
hp_name.pop(1)
hp_name[0] = '%'
hp_name = ''.join(hp_name)
else:
hp_name = '%'
lp_name = '%'
ul_name = '_'
return table_name, hp_name, lp_name, ul_name
def checkRegularTableWildcardLength(self):
'''
check regular table wildcard length with % and _
'''
self.cleanTb()
table_name, hp_name, lp_name, ul_name = self.genTableName()
tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, a1 int)")
sql_list = [f'show tables like "{hp_name}"', f'show tables like "{lp_name}"', f'show tables like "{ul_name}"']
for sql in sql_list:
tdSql.query(sql)
if len(table_name) >= 1:
tdSql.checkRows(1)
else:
tdSql.error(sql)
exceed_sql_list = [f'show tables like "%{hp_name}"', f'show tables like "{lp_name}%"', f'show tables like "{ul_name}%"']
for sql in exceed_sql_list:
tdSql.error(sql)
def checkSuperTableWildcardLength(self):
'''
check super table wildcard length with % and _
'''
self.cleanTb()
table_name, hp_name, lp_name, ul_name = self.genTableName()
tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, c1 int) tags (t1 int)")
sql_list = [f'show stables like "{hp_name}"', f'show stables like "{lp_name}"', f'show stables like "{ul_name}"']
for sql in sql_list:
tdSql.query(sql)
if len(table_name) >= 1:
tdSql.checkRows(1)
else:
tdSql.error(sql)
exceed_sql_list = [f'show stables like "%{hp_name}"', f'show stables like "{lp_name}%"', f'show stables like "{ul_name}%"']
for sql in exceed_sql_list:
tdSql.error(sql)
def checkRegularWildcardSelectLength(self):
'''
check regular table wildcard select length with % and _
'''
self.cleanTb()
table_name, hp_name, lp_name, ul_name = self.genTableName()
tdSql.execute(f"CREATE TABLE {table_name} (ts timestamp, bi1 binary(200), nc1 nchar(200))")
tdSql.execute(f'insert into {table_name} values (now, "{table_name}", "{table_name}")')
sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"',
f'select * from {table_name} where bi1 like "{lp_name}"',
f'select * from {table_name} where bi1 like "{ul_name}"',
f'select * from {table_name} where nc1 like "{hp_name}"',
f'select * from {table_name} where nc1 like "{lp_name}"',
f'select * from {table_name} where nc1 like "{ul_name}"']
for sql in sql_list:
tdSql.query(sql)
if len(table_name) >= 1:
tdSql.checkRows(1)
else:
tdSql.error(sql)
exceed_sql_list = [f'select * from {table_name} where bi1 like "%{hp_name}"',
f'select * from {table_name} where bi1 like "{lp_name}%"',
f'select * from {table_name} where bi1 like "{ul_name}%"',
f'select * from {table_name} where nc1 like "%{hp_name}"',
f'select * from {table_name} where nc1 like "{lp_name}%"',
f'select * from {table_name} where nc1 like "{ul_name}%"']
for sql in exceed_sql_list:
tdSql.error(sql)
def checkStbWildcardSelectLength(self):
'''
check stb wildcard select length with % and _
'''
self.cleanTb()
table_name, hp_name, lp_name, ul_name = self.genTableName()
tdSql.execute(f'CREATE TABLE {table_name} (ts timestamp, bi1 binary(200), nc1 nchar(200)) tags (si1 binary(200), sc1 nchar(200))')
tdSql.execute(f'create table {table_name}_sub1 using {table_name} tags ("{table_name}", "{table_name}")')
tdSql.execute(f'insert into {table_name}_sub1 values (now, "{table_name}", "{table_name}");')
# TODO sc1 leave a bug ---> TD-5918
# sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"',
# f'select * from {table_name} where bi1 like "{lp_name}"',
# f'select * from {table_name} where bi1 like "{ul_name}"',
# f'select * from {table_name} where nc1 like "{hp_name}"',
# f'select * from {table_name} where nc1 like "{lp_name}"',
# f'select * from {table_name} where nc1 like "{ul_name}"',
# f'select * from {table_name} where si1 like "{hp_name}"',
# f'select * from {table_name} where si1 like "{lp_name}"',
# f'select * from {table_name} where si1 like "{ul_name}"',
# f'select * from {table_name} where sc1 like "{hp_name}"',
# f'select * from {table_name} where sc1 like "{lp_name}"',
# f'select * from {table_name} where sc1 like "{ul_name}"']
sql_list = [f'select * from {table_name} where bi1 like "{hp_name}"',
f'select * from {table_name} where bi1 like "{lp_name}"',
f'select * from {table_name} where bi1 like "{ul_name}"',
f'select * from {table_name} where nc1 like "{hp_name}"',
f'select * from {table_name} where nc1 like "{lp_name}"',
f'select * from {table_name} where nc1 like "{ul_name}"',
f'select * from {table_name} where si1 like "{hp_name}"',
f'select * from {table_name} where si1 like "{lp_name}"',
f'select * from {table_name} where si1 like "{ul_name}"']
for sql in sql_list:
tdSql.query(sql)
if len(table_name) >= 1:
tdSql.checkRows(1)
else:
tdSql.error(sql)
exceed_sql_list = [f'select * from {table_name} where bi1 like "%{hp_name}"',
f'select * from {table_name} where bi1 like "{lp_name}%"',
f'select * from {table_name} where bi1 like "{ul_name}%"',
f'select * from {table_name} where nc1 like "%{hp_name}"',
f'select * from {table_name} where nc1 like "{lp_name}%"',
f'select * from {table_name} where nc1 like "{ul_name}%"',
f'select * from {table_name} where si1 like "%{hp_name}"',
f'select * from {table_name} where si1 like "{lp_name}%"',
f'select * from {table_name} where si1 like "{ul_name}%"',
f'select * from {table_name} where sc1 like "%{hp_name}"',
f'select * from {table_name} where sc1 like "{lp_name}%"',
f'select * from {table_name} where sc1 like "{ul_name}%"']
for sql in exceed_sql_list:
tdSql.error(sql)
def run(self):
tdSql.prepare()
self.checkRegularTableWildcardLength()
self.checkSuperTableWildcardLength()
self.checkRegularWildcardSelectLength()
self.checkStbWildcardSelectLength()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

16
tests/pytest/util/sql.py
View File

@ -81,6 +81,22 @@ class TDSql:
return self.queryResult
return self.queryRows
def getVariable(self, search_attr):
'''
get variable of search_attr access "show variables"
'''
try:
sql = 'show variables'
param_list = self.query(sql, row_tag=True)
for param in param_list:
if param[0] == search_attr:
return param[1], param_list
except Exception as e:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
raise Exception(repr(e))
def getColNameList(self, sql, col_tag=None):
self.sql = sql
try:

111
tests/script/general/parser/condition.sim Normal file
View File

@ -0,0 +1,111 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
system sh/exec.sh -n dnode1 -s start
sleep 100
sql connect
sql drop database if exists cdb
sql create database if not exists cdb
sql use cdb
sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double)
sql create table tb1 using stb1 tags(1,'1',1.0)
sql create table tb2 using stb1 tags(2,'2',2.0)
sql create table tb3 using stb1 tags(3,'3',3.0)
sql create table tb4 using stb1 tags(4,'4',4.0)
sql create table tb5 using stb1 tags(5,'5',5.0)
sql create table tb6 using stb1 tags(6,'6',6.0)
sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1')
sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2')
sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3')
sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4')
sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11')
sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12')
sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13')
sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14')
sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21')
sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22')
sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23')
sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24')
sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31')
sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32')
sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33')
sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34')
sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41')
sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42')
sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43')
sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44')
sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51')
sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52')
sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53')
sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54')
sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61')
sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62')
sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63')
sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64')
sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)
sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int)
sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1)
sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2)
sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01')
sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02')
sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03')
sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04')
sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05')
sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06')
sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07')
sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08')
sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09')
sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10')
sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11')
sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12')
sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13')
sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14')
sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15')
sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16')
sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double)
sql create table tb3_1 using stb3 tags(1,'1',1.0)
sql create table tb3_2 using stb3 tags(2,'2',2.0)
sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1')
sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2')
sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3')
sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4')
sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5')
sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL)
sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)
sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11')
sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12')
sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13')
sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14')
sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15')
sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL)
sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)
sleep 100
sql connect
run general/parser/condition_query.sim
print ================== restart server to commit data into disk
system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 100
system sh/exec.sh -n dnode1 -s start
print ================== server restart completed
sql connect
sleep 100
run general/parser/condition_query.sim

2628
tests/script/general/parser/condition_query.sim Normal file
View File

File diff suppressed because it is too large Load Diff

5
tests/script/general/parser/set_tag_vals.sim
View File

@ -83,10 +83,7 @@ while $i < $tbNum
endw
print ================== all tags have been changed!
sql select tbname from $stb where t3 = 'NULL'
if $rows != 0 then
return -1
endi
sql_error select tbname from $stb where t3 = 'NULL'
print ================== set tag to NULL
sql create table stb1_tg (ts timestamp, c1 int) tags(t1 int,t2 bigint,t3 double,t4 float,t5 smallint,t6 tinyint)

38
tests/script/general/parser/where.sim
View File

@ -65,7 +65,14 @@ $tb = $tbPrefix . $i
sql_error select * from $tb where c7
# TBASE-654 : invalid filter expression cause server crashed
sql_error select count(*) from $tb where c1<10 and c1<>2
sql select count(*) from $tb where c1<10 and c1<>2
if $rows != 1 then
return -1
endi
if $data00 != 900 then
return -1
endi
sql select * from $tb where c7 = false
$val = $rowNum / 100
@ -253,30 +260,11 @@ sql insert into tb_where_NULL values ('2019-01-01 09:00:02.000', 2, 'val2')
sql_error select * from tb_where_NULL where c1 = NULL
sql_error select * from tb_where_NULL where c1 <> NULL
sql_error select * from tb_where_NULL where c1 < NULL
sql select * from tb_where_NULL where c1 = "NULL"
if $rows != 0 then
return -1
endi
sql select * from tb_where_NULL where c1 <> "NULL"
if $rows != 2 then
return -1
endi
sql select * from tb_where_NULL where c1 <> "nulL"
if $rows != 2 then
return -1
endi
sql select * from tb_where_NULL where c1 > "NULL"
if $rows != 0 then
return -1
endi
sql select * from tb_where_NULL where c1 >= "NULL"
if $rows != 0 then
return -1
endi
sql_error select * from tb_where_NULL where c1 = "NULL"
sql_error select * from tb_where_NULL where c1 <> "NULL"
sql_error select * from tb_where_NULL where c1 <> "nulL"
sql_error select * from tb_where_NULL where c1 > "NULL"
sql_error select * from tb_where_NULL where c1 >= "NULL"
sql select * from tb_where_NULL where c2 = "NULL"
if $rows != 0 then
return -1

1
tests/script/test.sh
View File

@ -121,6 +121,7 @@ echo "rpcDebugFlag 143" >> $TAOS_CFG
echo "tmrDebugFlag 131" >> $TAOS_CFG
echo "cDebugFlag 143" >> $TAOS_CFG
echo "udebugFlag 143" >> $TAOS_CFG
echo "debugFlag 143" >> $TAOS_CFG
echo "wal 0" >> $TAOS_CFG
echo "asyncLog 0" >> $TAOS_CFG
echo "locale en_US.UTF-8" >> $TAOS_CFG