enh(index): fix tag query error

This commit is contained in:
yihaoDeng 2022-05-15 23:29:13 +08:00
parent 54d0e3c360
commit e8a637699b
6 changed files with 148 additions and 39 deletions

View File

@ -39,6 +39,12 @@ _cache_range_compare indexGetCompare(RangeType ty);
int32_t indexConvertData(void* src, int8_t type, void** dst); int32_t indexConvertData(void* src, int8_t type, void** dst);
int32_t indexGetDataByteLen(int8_t type);
int32_t indexMayFillNumbericData(void* number, int32_t tlen);
int32_t indexMayUnfillNumbericData(void* number, int32_t tlen);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -282,8 +282,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
if (0 != strncmp(c->colVal, pCt->colVal, skip)) { if (0 != strncmp(c->colVal, pCt->colVal, skip)) {
break; break;
} }
char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1);
memcpy(p, c->colVal, strlen(c->colVal));
TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType); TExeCond cond = cmpFn(p + skip, term->colVal, dType);
if (cond == MATCH) { if (cond == MATCH) {
if (c->operaType == ADD_VALUE) { if (c->operaType == ADD_VALUE) {
INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
@ -297,6 +299,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
} else if (cond == BREAK) { } else if (cond == BREAK) {
break; break;
} }
taosMemoryFree(p);
} }
taosMemoryFree(pCt); taosMemoryFree(pCt);

View File

@ -31,44 +31,34 @@ static __compar_fn_t indexGetCompar(int8_t type) {
} }
static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
__compar_fn_t func = indexGetCompar(type); __compar_fn_t func = indexGetCompar(type);
int32_t tlen = indexGetDataByteLen(type);
indexMayUnfillNumbericData(a, tlen);
indexMayUnfillNumbericData(b, tlen);
return tDoCommpare(func, QUERY_LESS_THAN, a, b); return tDoCommpare(func, QUERY_LESS_THAN, a, b);
} }
static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = indexGetCompar(type); __compar_fn_t func = indexGetCompar(type);
int32_t tlen = indexGetDataByteLen(type);
indexMayUnfillNumbericData(a, tlen);
indexMayUnfillNumbericData(b, tlen);
return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
} }
static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
__compar_fn_t func = indexGetCompar(type); __compar_fn_t func = indexGetCompar(type);
if (type == TSDB_DATA_TYPE_INT) {
char* v1 = (char*)a; int32_t tlen = indexGetDataByteLen(type);
char* v2 = (char*)b; indexMayUnfillNumbericData(a, tlen);
for (int i = 0; i < sizeof(int32_t); i++) { indexMayUnfillNumbericData(b, tlen);
if (v1[i] == '0') {
v1[i] = 0;
}
if (v2[i] == '0') {
v2[i] = 0;
}
}
return tDoCommpare(func, QUERY_GREATER_THAN, v1, v2);
}
return tDoCommpare(func, QUERY_GREATER_THAN, a, b); return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
} }
static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = indexGetCompar(type); __compar_fn_t func = indexGetCompar(type);
if (type == TSDB_DATA_TYPE_INT) {
char* v1 = (char*)a; int32_t tlen = indexGetDataByteLen(type);
char* v2 = (char*)b; indexMayUnfillNumbericData(a, tlen);
for (int i = 0; i < sizeof(int32_t); i++) { indexMayUnfillNumbericData(b, tlen);
if (v1[i] == '0') {
v1[i] = 0;
}
if (v2[i] == '0') {
v2[i] = 0;
}
}
return tDoCommpare(func, QUERY_GREATER_EQUAL, v1, v2);
}
return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
} }
@ -200,9 +190,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
tlen = taosEncodeFixedU32(dst, *(uint32_t*)src); tlen = taosEncodeFixedU32(dst, *(uint32_t*)src);
break; break;
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
tlen = taosEncodeFixedI64(NULL, *(uint32_t*)src); tlen = taosEncodeFixedI64(NULL, *(int64_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1); *dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedI64(dst, *(uint32_t*)src); tlen = taosEncodeFixedI64(dst, *(int64_t*)src);
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
tlen = taosEncodeBinary(NULL, src, sizeof(double)); tlen = taosEncodeBinary(NULL, src, sizeof(double));
@ -210,9 +200,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
tlen = taosEncodeBinary(dst, src, sizeof(double)); tlen = taosEncodeBinary(dst, src, sizeof(double));
break; break;
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
tlen = taosEncodeFixedU64(NULL, *(uint32_t*)src); tlen = taosEncodeFixedU64(NULL, *(uint64_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1); *dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedU64(dst, *(uint32_t*)src); tlen = taosEncodeFixedU64(dst, *(uint64_t*)src);
break; break;
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
@ -241,14 +231,78 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
break; break;
} }
*dst = *dst - tlen; *dst = *dst - tlen;
if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY &&
type != TSDB_DATA_TYPE_VARCHAR) { indexMayFillNumbericData(*dst, tlen);
uint8_t* p = *dst;
for (int i = 0; i < tlen; i++) { // if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY &&
if (p[i] == 0) { // type != TSDB_DATA_TYPE_VARCHAR) { uint8_t* p = *dst;
p[i] = (uint8_t)'0'; // for (int i = 0; i < tlen; i++) {
} // if (p[i] == 0) {
} // p[i] = (uint8_t)'0';
// }
// }
//}
return tlen;
}
int32_t indexGetDataByteLen(int8_t type) {
int32_t tlen = -1;
switch (type) {
case TSDB_DATA_TYPE_TIMESTAMP:
tlen = sizeof(int64_t);
break;
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
tlen = sizeof(uint8_t);
break;
case TSDB_DATA_TYPE_TINYINT:
tlen = sizeof(uint8_t);
break;
case TSDB_DATA_TYPE_SMALLINT:
tlen = sizeof(int16_t);
break;
case TSDB_DATA_TYPE_USMALLINT:
tlen = sizeof(uint16_t);
break;
case TSDB_DATA_TYPE_INT:
tlen = sizeof(int32_t);
break;
case TSDB_DATA_TYPE_UINT:
tlen = sizeof(uint32_t);
break;
case TSDB_DATA_TYPE_BIGINT:
tlen = sizeof(int64_t);
break;
case TSDB_DATA_TYPE_UBIGINT:
tlen = sizeof(uint64_t);
break;
case TSDB_DATA_TYPE_FLOAT:
tlen = sizeof(float);
break;
case TSDB_DATA_TYPE_DOUBLE:
tlen = sizeof(double);
break;
default:
break;
} }
return tlen; return tlen;
} }
int32_t indexMayFillNumbericData(void* number, int32_t tlen) {
for (int i = 0; i < tlen; i++) {
int8_t* p = number;
if (p[i] == 0) {
p[i] = (uint8_t)'0';
}
}
return 0;
}
int32_t indexMayUnfillNumbericData(void* number, int32_t tlen) {
for (int i = 0; i < tlen; i++) {
int8_t* p = number;
if (p[i] == (uint8_t)'0') {
p[i] = 0;
}
}
return 0;
}

Binary file not shown.

View File

@ -456,4 +456,32 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) {
EXPECT_EQ(0, taosArrayGetSize(result)); EXPECT_EQ(0, taosArrayGetSize(result));
indexMultiTermQueryDestroy(mq); indexMultiTermQueryDestroy(mq);
} }
{
std::string colName("test1");
int val = 15;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
{
std::string colName("test1");
int val = 8;
// std::string colVal("10");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(),
(const char*)&val, sizeof(val));
SArray* result = taosArrayInit(1, sizeof(uint64_t));
indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL);
tIndexJsonSearch(index, mq, result);
EXPECT_EQ(2000, taosArrayGetSize(result));
indexMultiTermQueryDestroy(mq);
}
} }

View File

@ -6,12 +6,14 @@
#include <vector> #include <vector>
#include "index.h" #include "index.h"
#include "indexCache.h" #include "indexCache.h"
#include "indexComm.h"
#include "indexFst.h" #include "indexFst.h"
#include "indexFstCountingWriter.h" #include "indexFstCountingWriter.h"
#include "indexFstUtil.h" #include "indexFstUtil.h"
#include "indexInt.h" #include "indexInt.h"
#include "indexTfile.h" #include "indexTfile.h"
#include "indexUtil.h" #include "indexUtil.h"
#include "tcoding.h"
#include "tglobal.h" #include "tglobal.h"
#include "tskiplist.h" #include "tskiplist.h"
#include "tutil.h" #include "tutil.h"
@ -305,3 +307,19 @@ TEST_F(UtilEnv, 01Except) {
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1); ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100); ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100);
} }
TEST_F(UtilEnv, testFill) {
for (int i = 0; i < 10000; i++) {
char buf[10] = {0};
void *pBuf = (void *)buf;
int val = i;
int v;
taosEncodeFixedI32((void **)(&pBuf), val);
// memcpy(buf, &val, sizeof(int));
indexMayFillNumbericData((void *)buf, sizeof(val));
indexMayUnfillNumbericData((void *)buf, sizeof(val));
taosDecodeFixedI32(buf, &v);
ASSERT_EQ(val, v);
}
assert(0);
}