xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into test/TD-4366

This commit is contained in:
bryanchang0603 2021-06-15 17:00:44 +08:00
parent 90cc682dd3 5242a597dc
commit 73f277c9dc
15 changed files with 149 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
documentation20/cn/12.taos-sql/docs.md
View File

@ -681,9 +681,10 @@ Query OK, 1 row(s) in set (0.001091s)
| % | match with any char sequences | **`binary`** **`nchar`** |
| _ | match with a single char | **`binary`** **`nchar`** |
1. 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。
2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。
3. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。
1. <> 算子也可以写为 != ，请注意，这个算子不能用于数据表第一列的 timestamp 字段。
2. 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。
3. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。
4. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。
<!--
<a class="anchor" id="having"></a>
@ -1178,9 +1179,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
应用字段：不能应用在timestamp、binary、nchar、bool类型字段。
适用于：**表**。
适用于：**表、（超级表）**。
说明：输出结果行数是范围内总行数减一，第一行没有结果输出。
说明：输出结果行数是范围内总行数减一，第一行没有结果输出。从 2.1.3.0 版本开始，DIFF 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。
示例：
```mysql

5
src/client/src/tscPrepare.c
View File

@ -1528,13 +1528,14 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) {
STscStmt* pStmt = (STscStmt*)stmt;
SSqlObj* pSql = pStmt->pSql;
SSqlCmd* pCmd = &pSql->cmd;
if (stmt == NULL || pStmt->pSql == NULL || pStmt->taos == NULL) {
STMT_RET(TSDB_CODE_TSC_DISCONNECTED);
}
SSqlObj* pSql = pStmt->pSql;
SSqlCmd* pCmd = &pSql->cmd;
if (name == NULL) {
tscError("0x%"PRIx64" name is NULL", pSql->self);
STMT_RET(invalidOperationMsg(tscGetErrorMsgPayload(&pStmt->pSql->cmd), "name is NULL"));

22
src/client/src/tscSQLParser.c
View File

@ -150,12 +150,9 @@ int16_t getNewResColId(SSqlCmd* pCmd) {
// formate "type | size | value"
bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType) {
bool ret = false;
if (!pList || pList->size <= 0) {
if (!pList || pList->size <= 0 || colType < 0) {
return ret;
}
if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) {
return ret;
}
tSqlExprItem* item = (tSqlExprItem *)taosArrayGet(pList, 0);
int32_t firstTokenType = item->pNode->token.type;
@ -4562,11 +4559,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
free(tmp);
} else {
double tmp;
if (p->_node.optr == TSDB_RELATION_IN) {
retVal = validateParamOfRelationIn(vVariant, schemaType);
} else {
retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false);
}
retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false);
}
if (retVal != TSDB_CODE_SUCCESS) {
@ -8065,18 +8058,23 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
return TSDB_CODE_SUCCESS;
} else if (pSqlExpr->tokenId == TK_SET) {
int32_t type = -1;
int32_t colType = -1;
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
if (pCols != NULL) {
SColIndex* idx = taosArrayGet(pCols, 0);
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex);
if (pSchema != NULL) {
type = pSchema->type;
colType = pSchema->type;
}
}
tVariant *pVal;
if (serializeExprListToVariant(pSqlExpr->pParam, &pVal, type) == false) {
if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) {
colType = TSDB_DATA_TYPE_BIGINT;
} else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) {
colType = TSDB_DATA_TYPE_DOUBLE;
}
if (serializeExprListToVariant(pSqlExpr->pParam, &pVal, colType) == false) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression");
}
*pExpr = calloc(1, sizeof(tExprNode));

3
src/common/src/texpr.c
View File

@ -470,6 +470,9 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) {
SBufferReader br = tbufInitReader(buf, len, false);
uint32_t type = tbufReadUint32(&br);
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type));
int dummy = -1;
int32_t sz = tbufReadInt32(&br);
for (int32_t i = 0; i < sz; i++) {

2
src/inc/taosdef.h
View File

@ -166,7 +166,7 @@ do { \
#define TSDB_BINARY_OP_REMAINDER 34
#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN))
#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
#define TS_PATH_DELIMITER_LEN 1

2
src/mnode/src/mnodeTable.c
View File

@ -1501,7 +1501,7 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) {
}
// update
SSchema *schema = (SSchema *) (pStable->schema + col);
SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns);
ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);
schema->bytes = pAlter->schema[0].bytes;
mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,

24
src/query/src/qExecutor.c
View File

@ -2282,7 +2282,7 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) {
return true;
}
bool ret = true;
for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) {
SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k];
@ -2319,26 +2319,34 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
}
SDataStatis* pDataBlockst = &pDataStatis[index];
if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) {
float minval = (float)(*(double *)(&pDataBlockst->min));
float maxval = (float)(*(double *)(&pDataBlockst->max));
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT)) {
return true;
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
if (ret == false) {
return false;
}
}
} else {
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type)) {
return true;
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
if (ret == false) {
return false;
}
}
}
}
return false;
return ret;
}
static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {

2
src/query/src/qFilterfunc.c
View File

@ -262,7 +262,7 @@ bool inOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxv
return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv));
}
return true;
} else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_DOUBLE) {
} else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) {
double v;
GET_TYPED_DATA(v, double, type, minval);
return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&v, sizeof(v));

10
src/util/inc/hash.h
View File

@ -61,6 +61,7 @@ typedef struct SHashObj {
size_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_hash_free_fn_t freeFp; // hash node free callback function
_equal_fn_t equalFp; // equal function
SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type
@ -78,6 +79,15 @@ typedef struct SHashObj {
*/
SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type);
/**
* set equal func of the hash table
* @param pHashObj
* @param equalFp
* @return
*/
void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
/**
* return the size of hash table
* @param pHashObj

9
src/util/inc/hashfunc.h
View File

@ -20,6 +20,8 @@
typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
typedef int32_t (*_equal_fn_t)(const void *a, const void *b, size_t sz);
/**
* murmur hash algorithm
* @key usually string
@ -36,9 +38,14 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len);
* @return
*/
uint32_t taosIntHash_32(const char *key, uint32_t len);
uint32_t taosIntHash_64(const char *key, uint32_t len);
int32_t taosFloatEqual(const void *a, const void *b, size_t sz);
int32_t taosDoubleEqual(const void *a,const void *b, size_t sz);
_hash_fn_t taosGetDefaultHashFunction(int32_t type);
_equal_fn_t taosGetDefaultEqualFunction(int32_t type);
#endif //TDENGINE_HASHUTIL_H

7
src/util/inc/tcompare.h
View File

@ -27,6 +27,13 @@ extern "C" {
#define TSDB_PATTERN_NOWILDCARDMATCH 2
#define TSDB_PATTERN_STRING_MAX_LEN 20
#define FLT_COMPAR_TOL_FACTOR 4
#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y)))
#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y)))
#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' }
typedef struct SPatternCompareInfo {

20
src/util/src/hash.c
View File

@ -73,10 +73,10 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
return i;
}
static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
SHashNode *pNode = pe->next;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
assert(pNode->hashVal == hashVal);
break;
}
@ -165,8 +165,8 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
// the max slots is not defined by user
pHashObj->capacity = taosHashCapacity((int32_t)capacity);
assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
pHashObj->hashFp = fn;
pHashObj->equalFp = memcmp;
pHashObj->hashFp = fn;
pHashObj->type = type;
pHashObj->enableUpdate = update;
@ -189,6 +189,12 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
return pHashObj;
}
void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) {
if (pHashObj != NULL && fp != NULL) {
pHashObj->equalFp = fp;
}
}
int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); }
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
@ -223,7 +229,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
SHashNode* prev = NULL;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
assert(pNode->hashVal == hashVal);
break;
}
@ -306,7 +312,7 @@ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void
assert(pe->next == NULL);
}
SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal);
SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal);
if (pNode != NULL) {
if (fp != NULL) {
fp(GET_HASH_NODE_DATA(pNode));
@ -362,7 +368,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
SHashNode *prevNode = NULL;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0)
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0)
break;
prevNode = pNode;

18
src/util/src/tcompare.c
View File

@ -90,13 +90,10 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) {
if (isnan(p2)) {
return 1;
}
float ret = p1 - p2;
if (fabs(ret) < FLT_EPSILON) {
if (FLT_EQUAL(p1, p2)) {
return 0;
} else {
return ret > 0? 1 : -1;
}
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
@ -114,13 +111,10 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
if (isnan(p2)) {
return 1;
}
double ret = p1 - p2;
if (fabs(ret) < FLT_EPSILON) {
if (FLT_EQUAL(p1, p2)) {
return 0;
} else {
return ret > 0? 1 : -1;
}
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {

70
src/util/src/thashutil.c
View File

@ -16,17 +16,20 @@
#include "os.h"
#include "hashfunc.h"
#include "tutil.h"
#include "tcompare.h"
#define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))
#define DLT (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)
#define BASE 1000
#define FMIX32(h) \
do { \
(h) ^= (h) >> 16; \
(h) *= 0x85ebca6b; \
(h) ^= (h) >> 13; \
(h) *= 0xc2b2ae35; \
(h) ^= (h) >> 16; \
} while (0)
(h) ^= (h) >> 16; } while (0)
uint32_t MurmurHash3_32(const char *key, uint32_t len) {
const uint8_t *data = (const uint8_t *)key;
@ -78,7 +81,38 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) {
uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; }
uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; }
uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; }
uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) {
float f = GET_FLOAT_VAL(key);
if (isnan(f)) {
return 0x7fc00000;
}
if (FLT_EQUAL(f, 0.0)) {
return 0;
}
if (fabs(f) < FLT_MAX/BASE - DLT) {
int t = (int)(round(BASE * (f + DLT)));
return (uint32_t)t;
} else {
return 0x7fc00000;
}
}
uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) {
double f = GET_DOUBLE_VAL(key);
if (isnan(f)) {
return 0x7fc00000;
}
if (FLT_EQUAL(f, 0.0)) {
return 0;
}
if (fabs(f) < DBL_MAX/BASE - DLT) {
int t = (int)(round(BASE * (f + DLT)));
return (uint32_t)t;
} else {
return 0x7fc00000;
}
}
uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) {
uint64_t val = *(uint64_t *)key;
@ -92,13 +126,35 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) {
_hash_fn_t fn = NULL;
switch(type) {
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64;break;
case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32;break;
case TSDB_DATA_TYPE_INT: fn = taosIntHash_32; break;
case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64;break;
case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32;break;
case TSDB_DATA_TYPE_NCHAR: fn = MurmurHash3_32;break;
case TSDB_DATA_TYPE_INT: fn = taosIntHash_32; break;
case TSDB_DATA_TYPE_SMALLINT: fn = taosIntHash_16; break;
case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break;
case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break;
case TSDB_DATA_TYPE_FLOAT: fn = taosFloatHash; break;
case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleHash; break;
default: fn = taosIntHash_32;break;
}
return fn;
}
}
int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b);
}
int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b);
}
_equal_fn_t taosGetDefaultEqualFunction(int32_t type) {
_equal_fn_t fn = NULL;
switch (type) {
case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break;
case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break;
default: fn = memcmp; break;
}
return fn;
}

2
tests/pytest/fulltest.sh
View File

@ -342,7 +342,7 @@ python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py
python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py
python3 ./test.py -f tag_lite/drop_auto_create.py
python3 test.py -f insert/insert_before_use_db.py
python3 test.py -f alter/alter_keep.py
#python3 test.py -f alter/alter_keep.py
python3 test.py -f alter/alter_cacheLastRow.py
python3 ./test.py -f query/querySession.py
python3 test.py -f alter/alter_create_exception.py