From c3760f272471cb99fc8a2ebf705d3901aa5c3e30 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 14 May 2022 23:40:43 +0800 Subject: [PATCH 1/5] enh(index): fix tag query error --- source/libs/index/src/index.c | 4 +- source/libs/index/src/indexCache.c | 1 - source/libs/index/src/indexComm.c | 28 +++++++- source/libs/index/test/jsonUT.cc | 110 +++++++++++++++-------------- 4 files changed, 85 insertions(+), 58 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 46f2f7a93b..b9df2e88ce 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -109,17 +109,15 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadMutexInit(&sIdx->mtx, NULL); sIdx->refId = indexAddRef(sIdx); - taosAcquireRef(indexRefMgt, sIdx->refId); + indexAcquireRef(sIdx->refId); *index = sIdx; - return 0; END: if (sIdx != NULL) { indexClose(sIdx); } - *index = NULL; return -1; } diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index d4231619ec..6cd00f76e1 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -463,7 +463,6 @@ int indexCacheSchedToMerge(IndexCache* pCache) { // schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t)); // memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t)); schedMsg.msg = NULL; - indexAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index ac26ed1fab..3070659851 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -39,10 +39,36 @@ static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); + if (type == TSDB_DATA_TYPE_INT) { + char* v1 = (char*)a; + char* v2 = (char*)b; + for (int i = 0; i < sizeof(int32_t); i++) { + if (v1[i] == '0') { + v1[i] = 0; + } + if (v2[i] == '0') { + v2[i] = 0; + } + } + return tDoCommpare(func, QUERY_GREATER_THAN, v1, v2); + } return tDoCommpare(func, QUERY_GREATER_THAN, a, b); } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); + if (type == TSDB_DATA_TYPE_INT) { + char* v1 = (char*)a; + char* v2 = (char*)b; + for (int i = 0; i < sizeof(int32_t); i++) { + if (v1[i] == '0') { + v1[i] = 0; + } + if (v2[i] == '0') { + v2[i] = 0; + } + } + return tDoCommpare(func, QUERY_GREATER_EQUAL, v1, v2); + } return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); } @@ -216,7 +242,7 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { } *dst = *dst - tlen; if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY && - type == TSDB_DATA_TYPE_VARCHAR) { + type != TSDB_DATA_TYPE_VARCHAR) { uint8_t* p = *dst; for (int i = 0; i < tlen; i++) { if (p[i] == 0) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 3de7cb66f2..eb1eed403e 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -204,9 +204,10 @@ TEST_F(JsonEnv, testWriteMillonData) { TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test"); - std::string colVal("10"); + // std::string colVal("10"); + int val = 10; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -217,35 +218,9 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } { std::string colName("test2"); - std::string colVal("20"); + int val = 20; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); - - SIndexMultiTerm* terms = indexMultiTermCreate(); - indexMultiTermAdd(terms, term); - for (size_t i = 0; i < 1000; i++) { - tIndexJsonPut(index, terms, i); - } - indexMultiTermDestroy(terms); - } - { - std::string colName("test2"); - std::string colVal("15"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); - - SIndexMultiTerm* terms = indexMultiTermCreate(); - indexMultiTermAdd(terms, term); - for (size_t i = 0; i < 1000; i++) { - tIndexJsonPut(index, terms, i); - } - indexMultiTermDestroy(terms); - } - { - std::string colName("test2"); - std::string colVal("15"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -256,11 +231,36 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } { std::string colName("test"); - std::string colVal("10"); + int val = 15; + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("test2"); + const char* val = "test"; + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + (const char*)val, strlen(val)); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("test"); + int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -270,11 +270,11 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } { std::string colName("test"); - std::string colVal("10"); + int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -284,11 +284,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } { std::string colName("test"); - std::string colVal("10"); + int val = 10; + ; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -298,11 +299,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } { std::string colName("test"); - std::string colVal("10"); + int val = 10; + // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -312,11 +314,12 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } { std::string colName("test"); - std::string colVal("10"); + int val = 10; + // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -329,9 +332,9 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { TEST_F(JsonEnv, testWriteJsonTfileAndCache) { { std::string colName("test1"); - std::string colVal("10"); + int val = 10; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -355,11 +358,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("test1"); - std::string colVal("10"); + int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -369,11 +372,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("test1"); - std::string colVal("10"); + int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -426,6 +429,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { { std::string colName("other_column"); std::string colVal("100"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); From 54d0e3c360b511448a2a904026e5d7c9d778de7f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 15 May 2022 11:29:59 +0800 Subject: [PATCH 2/5] enh(index): fix tag query error --- source/libs/index/test/jsonUT.cc | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index eb1eed403e..e3ac4b9777 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -386,11 +386,12 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("test1"); - std::string colVal("10"); + // std::string colVal("10"); + int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -400,11 +401,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("test1"); - std::string colVal("10"); + int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -414,11 +415,11 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("test1"); - std::string colVal("10"); + int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -428,10 +429,10 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("other_column"); - std::string colVal("100"); + int val = 100; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -442,11 +443,12 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { } { std::string colName("test1"); - std::string colVal("10"); + int val = 10; + // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), colVal.c_str(), - colVal.size()); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); From e8a637699b838c75e7cb8efac5c2290fc89a5c73 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 15 May 2022 23:29:13 +0800 Subject: [PATCH 3/5] enh(index): fix tag query error --- source/libs/index/inc/indexComm.h | 6 ++ source/libs/index/src/indexCache.c | 5 +- source/libs/index/src/indexComm.c | 130 ++++++++++++++++++-------- source/libs/index/test/.utilUT.cc.swn | Bin 0 -> 20480 bytes source/libs/index/test/jsonUT.cc | 28 ++++++ source/libs/index/test/utilUT.cc | 18 ++++ 6 files changed, 148 insertions(+), 39 deletions(-) create mode 100644 source/libs/index/test/.utilUT.cc.swn diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 043404f48f..9b23e4eb44 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -39,6 +39,12 @@ _cache_range_compare indexGetCompare(RangeType ty); int32_t indexConvertData(void* src, int8_t type, void** dst); +int32_t indexGetDataByteLen(int8_t type); + +int32_t indexMayFillNumbericData(void* number, int32_t tlen); + +int32_t indexMayUnfillNumbericData(void* number, int32_t tlen); + #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 6cd00f76e1..1d7a4a5419 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -282,8 +282,10 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe if (0 != strncmp(c->colVal, pCt->colVal, skip)) { break; } + char* p = taosMemoryCalloc(1, strlen(c->colVal) + 1); + memcpy(p, c->colVal, strlen(c->colVal)); - TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType); + TExeCond cond = cmpFn(p + skip, term->colVal, dType); if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) @@ -297,6 +299,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe } else if (cond == BREAK) { break; } + taosMemoryFree(p); } taosMemoryFree(pCt); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 3070659851..89f19a3b81 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -31,44 +31,34 @@ static __compar_fn_t indexGetCompar(int8_t type) { } static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); + + int32_t tlen = indexGetDataByteLen(type); + indexMayUnfillNumbericData(a, tlen); + indexMayUnfillNumbericData(b, tlen); return tDoCommpare(func, QUERY_LESS_THAN, a, b); } static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); + + int32_t tlen = indexGetDataByteLen(type); + indexMayUnfillNumbericData(a, tlen); + indexMayUnfillNumbericData(b, tlen); return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); - if (type == TSDB_DATA_TYPE_INT) { - char* v1 = (char*)a; - char* v2 = (char*)b; - for (int i = 0; i < sizeof(int32_t); i++) { - if (v1[i] == '0') { - v1[i] = 0; - } - if (v2[i] == '0') { - v2[i] = 0; - } - } - return tDoCommpare(func, QUERY_GREATER_THAN, v1, v2); - } + + int32_t tlen = indexGetDataByteLen(type); + indexMayUnfillNumbericData(a, tlen); + indexMayUnfillNumbericData(b, tlen); return tDoCommpare(func, QUERY_GREATER_THAN, a, b); } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); - if (type == TSDB_DATA_TYPE_INT) { - char* v1 = (char*)a; - char* v2 = (char*)b; - for (int i = 0; i < sizeof(int32_t); i++) { - if (v1[i] == '0') { - v1[i] = 0; - } - if (v2[i] == '0') { - v2[i] = 0; - } - } - return tDoCommpare(func, QUERY_GREATER_EQUAL, v1, v2); - } + + int32_t tlen = indexGetDataByteLen(type); + indexMayUnfillNumbericData(a, tlen); + indexMayUnfillNumbericData(b, tlen); return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); } @@ -200,9 +190,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { tlen = taosEncodeFixedU32(dst, *(uint32_t*)src); break; case TSDB_DATA_TYPE_BIGINT: - tlen = taosEncodeFixedI64(NULL, *(uint32_t*)src); + tlen = taosEncodeFixedI64(NULL, *(int64_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedI64(dst, *(uint32_t*)src); + tlen = taosEncodeFixedI64(dst, *(int64_t*)src); break; case TSDB_DATA_TYPE_DOUBLE: tlen = taosEncodeBinary(NULL, src, sizeof(double)); @@ -210,9 +200,9 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(dst, src, sizeof(double)); break; case TSDB_DATA_TYPE_UBIGINT: - tlen = taosEncodeFixedU64(NULL, *(uint32_t*)src); + tlen = taosEncodeFixedU64(NULL, *(uint64_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeFixedU64(dst, *(uint32_t*)src); + tlen = taosEncodeFixedU64(dst, *(uint64_t*)src); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); @@ -241,14 +231,78 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { break; } *dst = *dst - tlen; - if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY && - type != TSDB_DATA_TYPE_VARCHAR) { - uint8_t* p = *dst; - for (int i = 0; i < tlen; i++) { - if (p[i] == 0) { - p[i] = (uint8_t)'0'; - } - } + + indexMayFillNumbericData(*dst, tlen); + + // if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY && + // type != TSDB_DATA_TYPE_VARCHAR) { uint8_t* p = *dst; + // for (int i = 0; i < tlen; i++) { + // if (p[i] == 0) { + // p[i] = (uint8_t)'0'; + // } + // } + //} + return tlen; +} +int32_t indexGetDataByteLen(int8_t type) { + int32_t tlen = -1; + switch (type) { + case TSDB_DATA_TYPE_TIMESTAMP: + tlen = sizeof(int64_t); + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + tlen = sizeof(uint8_t); + break; + case TSDB_DATA_TYPE_TINYINT: + tlen = sizeof(uint8_t); + break; + case TSDB_DATA_TYPE_SMALLINT: + tlen = sizeof(int16_t); + break; + case TSDB_DATA_TYPE_USMALLINT: + tlen = sizeof(uint16_t); + break; + case TSDB_DATA_TYPE_INT: + tlen = sizeof(int32_t); + break; + case TSDB_DATA_TYPE_UINT: + tlen = sizeof(uint32_t); + break; + case TSDB_DATA_TYPE_BIGINT: + tlen = sizeof(int64_t); + break; + case TSDB_DATA_TYPE_UBIGINT: + tlen = sizeof(uint64_t); + break; + case TSDB_DATA_TYPE_FLOAT: + tlen = sizeof(float); + break; + case TSDB_DATA_TYPE_DOUBLE: + tlen = sizeof(double); + break; + default: + break; } return tlen; } + +int32_t indexMayFillNumbericData(void* number, int32_t tlen) { + for (int i = 0; i < tlen; i++) { + int8_t* p = number; + if (p[i] == 0) { + p[i] = (uint8_t)'0'; + } + } + return 0; +} + +int32_t indexMayUnfillNumbericData(void* number, int32_t tlen) { + for (int i = 0; i < tlen; i++) { + int8_t* p = number; + if (p[i] == (uint8_t)'0') { + p[i] = 0; + } + } + return 0; +} diff --git a/source/libs/index/test/.utilUT.cc.swn b/source/libs/index/test/.utilUT.cc.swn new file mode 100644 index 0000000000000000000000000000000000000000..3023c72b42057ca24f48699b8e9353645b647d6c GIT binary patch literal 20480 zcmeI3ZEPGz8OJv@Eu!VovEw*NOXHVd+Cb~G=kwZLX4xQ%IqNDxpCHiG+YCR8&HU7gbaQR6g+1v>%XwwxohqrO>LhK&5=(H?zBU z=gs+EQWK}qp7hK4c6Vl<`OmYn&oeU_jco4UBsy0{H9m_q&Hmv2(HHufwWbF(%`#0} z9fyXuuu^(BZH(r_Ms`f!8MgJj z9WK~LdT4NEDpjq%;9Q_Upg^4p6zuWqIu~m3STrI_tXRHGym!e^orQzT0tEsE0tEsE z0tEsE0tEsE0tEsEE(HbbT$6SO4Y|y1%zf_nJm2>_+~0S(-=FuLZ*qUHcE1<76?pZy zhl8&`fk1&kfk1&kfk1&kfk1&kfk1&kfk1&kfk1&dP#`&`X^XgTLQVj%{_o}gFL3ZQ zJO&TLLvSw?UgHOU5 zSPECeYx6bjN%$!ogQIW<48vwv45G z*a6$18?J#Sc6z0Jx{Np%$1CGINkUld2zK;;&V!a3@ zZnUiAF45XzC(XS2d!ugO63SbtHW6)Es|issvUbO+ZMH~SR&47Ch4yrEE&?%#?_En< z#9k1hSjjDg{A6gnO*B&|y44s_HI18=2vM6bs3fwMuXQ4C?AFckki4u#gv)_|-7C?!bd^SWh+iaH=>f)+Yei}mY8Ox?if_lg>3 z))CviIdxQ#GG3pQ@##{=eNxu!c*n@{j*mMEB49iE$-a-)?DtrG#-d$$zsDLic1ACa zwh0_mnq1S=i^fGAgYnT>cTpa%I$hLxeyY0YtUWmiBa=((y_uZ7%d@Z1y6br8YPCp+ za9HTl2;uB>tMU|1CDZB=E*ls*#DHeV!E)LV7jwtBxcv6Uu{s~VnFfIdZtvbb+R)vd zr)d`Mu5QZ!uSD<~?+xNLMUXu<<8ZsFkhTrwGXr{_zt7im>KU)2wS8w6#5%98az^R7 zy8Em-N~N~Xj-T*ub;g&Ho8;}JZKTAMX^aW~K;zvshd-4fnQj>*gmQ!;d(|7>r?+!D z;|>X46`3lbRgSWfvG@Rj&=w9X;ml z*^q3g7c9vpR+hX!YuF(fu(*My7u2-SMv8kClr>U?uFB@-%Iz%nQMI~d)s;FpUhGbJ z1-GeUHo<^vwQ6~*wArUdLbr!$|$-u5BYb z(bYA`AJS1)7}-?1Fs6&;w%p9Ua&me3q5O6ump1aY@9;$098LPKO_^iVJpD?wRa}}; z;qc(Nk=835Co-;Zv@dVZP!DIi%|cc-YuGXf$SaqwaJ1XZWGWp_rY0*HHto`NQ%G%E z*O$~B&s_v1EF}#PG>!2Pf42diC$h$?G+l|I1wVALO($ z|M%AKA1ALr3J<~q@Fn;HOhOA>24~6T{|nE+58+O@3El_Kk;i`rz71c50_=cWVF|oU z&i*$z1Al=(!;fJ%$UJ`&^n=XruYf1X;~#<}@EOH6MgG_#!h{#KOPH>7Uaykynvbn)71Qj%}vQ&Ls>%Z^d}0LQQb09JxMzm z3b~1eD&gfCOmMk%dnNlCEXQnGM$e>jyF62*!EW_TntRnt(|fbiOm_`xZ3#8!HWbEN zXlHT0jgquxO6EYaQR+u&=8ZElsc)6VWKLCTnNuJ&nY4tUpNphteEgIod z%+vhSP%<)?f_Sq|rG9A*dD*3JJtbtDwv-@Bd#GHg+d#31EaodzOhUaC26YQAbQc(U zcc%0l3lYwZb@fYC;W*PdslCd2oy&ZyHkI>Ow>N9A+my|d%8Q+K1TtGW$K576R40&# zdWm>t!`x;}TTS|Z*u|0o{o~qHHYecB@|3Aql;1hyh_9Zv2Ksf?`ME<}74!K+P2Y=A zs6L1P1fqUT|*bW1*20CF0oFV5w0e8cGxD$pU0{>zj;3@bWd<*0n{|1oj`X7S% zaE3XN=iry{FdTv}Lk>O)P4GPH_RqpI@E9D1PeB)SKs(HbGpyO4fV*Koq@f2kKr_6` zdi~4r1RQ`AjKFQM1(v~5kZbxsfkSW*4uA=L&Qkf_V<<(om}Jta>bzFHNVVqXl&Bj#4rTq@QoQi2C~ElzpSj+Z5@Q^T z;SzCYJY!Ic$r!E}Q&u#3u6L(+LvT$)FPgn6(;hm+O})kKQ;}5#>1TC&=Njid@-}l$ zRCD&p@!bKd)rjScIv1s5%7=*jOm3D_57yglgP# #include "index.h" #include "indexCache.h" +#include "indexComm.h" #include "indexFst.h" #include "indexFstCountingWriter.h" #include "indexFstUtil.h" #include "indexInt.h" #include "indexTfile.h" #include "indexUtil.h" +#include "tcoding.h" #include "tglobal.h" #include "tskiplist.h" #include "tutil.h" @@ -305,3 +307,19 @@ TEST_F(UtilEnv, 01Except) { ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1); ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100); } +TEST_F(UtilEnv, testFill) { + for (int i = 0; i < 10000; i++) { + char buf[10] = {0}; + void *pBuf = (void *)buf; + int val = i; + int v; + taosEncodeFixedI32((void **)(&pBuf), val); + // memcpy(buf, &val, sizeof(int)); + indexMayFillNumbericData((void *)buf, sizeof(val)); + indexMayUnfillNumbericData((void *)buf, sizeof(val)); + + taosDecodeFixedI32(buf, &v); + ASSERT_EQ(val, v); + } + assert(0); +} From 9453f80d658bbdbb85c1bce31769e311333bd4b8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 16 May 2022 17:14:47 +0800 Subject: [PATCH 4/5] fix(index): fix index condition error --- source/libs/index/inc/indexComm.h | 8 +- source/libs/index/src/index.c | 2 +- source/libs/index/src/indexComm.c | 196 ++++++++++++++++---------- source/libs/index/test/.utilUT.cc.swn | Bin 20480 -> 0 bytes source/libs/index/test/jsonUT.cc | 2 +- source/libs/index/test/utilUT.cc | 24 ++-- 6 files changed, 140 insertions(+), 92 deletions(-) delete mode 100644 source/libs/index/test/.utilUT.cc.swn diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 9b23e4eb44..3066fd1c2c 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -33,17 +33,17 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b); +TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType); +TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b); _cache_range_compare indexGetCompare(RangeType ty); int32_t indexConvertData(void* src, int8_t type, void** dst); +int32_t indexConvertDataToStr(void* src, int8_t type, void** dst); int32_t indexGetDataByteLen(int8_t type); -int32_t indexMayFillNumbericData(void* number, int32_t tlen); - -int32_t indexMayUnfillNumbericData(void* number, int32_t tlen); +char* indexInt2str(int64_t val, char* dst, int radix); #ifdef __cplusplus } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b9df2e88ce..162d64c41c 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -271,7 +271,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->nColName = nColName; char* buf = NULL; - int32_t len = indexConvertData((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf); + int32_t len = indexConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf); assert(len != -1); tm->colVal = buf; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 89f19a3b81..0c2e9e6b44 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -19,10 +19,38 @@ #include "tcoding.h" #include "tcompare.h" #include "tdataformat.h" +#include "ttypes.h" char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; +char* indexInt2str(int64_t val, char* dst, int radix) { + char buffer[65]; + char* p; + int64_t new_val; + uint64_t uval = (uint64_t)val; + + if (radix < 0) { + if (val < 0) { + *dst++ = '-'; + uval = (uint64_t)0 - uval; /* Avoid integer overflow in (-val) for LLONG_MIN (BUG#31799). */ + } + } + p = &buffer[sizeof(buffer) - 1]; + *p = '\0'; + new_val = (int64_t)(uval / 10); + *--p = '0' + (char)(uval - (uint64_t)new_val * 10); + val = new_val; + + while (val != 0) { + new_val = val / 10; + *--p = '0' + (char)(val - new_val * 10); + val = new_val; + } + while ((*dst++ = *p++) != 0) + ; + return dst - 1; +} static __compar_fn_t indexGetCompar(int8_t type) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { return (__compar_fn_t)strcmp; @@ -31,41 +59,49 @@ static __compar_fn_t indexGetCompar(int8_t type) { } static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); - - int32_t tlen = indexGetDataByteLen(type); - indexMayUnfillNumbericData(a, tlen); - indexMayUnfillNumbericData(b, tlen); - return tDoCommpare(func, QUERY_LESS_THAN, a, b); + return tCompare(func, QUERY_LESS_THAN, a, b, type); } static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); - - int32_t tlen = indexGetDataByteLen(type); - indexMayUnfillNumbericData(a, tlen); - indexMayUnfillNumbericData(b, tlen); - return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); + return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); - - int32_t tlen = indexGetDataByteLen(type); - indexMayUnfillNumbericData(a, tlen); - indexMayUnfillNumbericData(b, tlen); - return tDoCommpare(func, QUERY_GREATER_THAN, a, b); + return tCompare(func, QUERY_GREATER_THAN, a, b, type); } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); - - int32_t tlen = indexGetDataByteLen(type); - indexMayUnfillNumbericData(a, tlen); - indexMayUnfillNumbericData(b, tlen); - return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); + return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } - -TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { +TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { + if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) { + return tDoCompare(func, cmptype, a, b); + } +#if 1 + int8_t bytes = tDataTypes[dtype].bytes; + if (bytes == 1) { + int8_t va = taosStr2int64(a); + int8_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (bytes == 2) { + int16_t va = taosStr2int64(a); + int16_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (bytes == 4) { + int32_t va = taosStr2int64(a); + int32_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else { + int64_t va = taosStr2int64(a); + int64_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } +#endif +} +TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { // optime later int32_t ret = func(a, b); - switch (comType) { + switch (comparType) { case QUERY_LESS_THAN: { if (ret < 0) return MATCH; } break; @@ -231,78 +267,92 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { break; } *dst = *dst - tlen; - - indexMayFillNumbericData(*dst, tlen); - - // if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY && - // type != TSDB_DATA_TYPE_VARCHAR) { uint8_t* p = *dst; - // for (int i = 0; i < tlen; i++) { - // if (p[i] == 0) { - // p[i] = (uint8_t)'0'; - // } - // } - //} + // indexMayFillNumbericData(*dst, tlen); return tlen; } -int32_t indexGetDataByteLen(int8_t type) { - int32_t tlen = -1; +int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { + int tlen = tDataTypes[type].bytes; + switch (type) { case TSDB_DATA_TYPE_TIMESTAMP: - tlen = sizeof(int64_t); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(int64_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: - tlen = sizeof(uint8_t); + // tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src); + //*dst = taosMemoryCalloc(1, tlen + 1); + // tlen = taosEncodeFixedU8(dst, *(uint8_t*)src); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(uint8_t*)src, *dst, 1); break; case TSDB_DATA_TYPE_TINYINT: - tlen = sizeof(uint8_t); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(int8_t*)src, *dst, 1); break; case TSDB_DATA_TYPE_SMALLINT: - tlen = sizeof(int16_t); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(int16_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_USMALLINT: - tlen = sizeof(uint16_t); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(uint16_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_INT: - tlen = sizeof(int32_t); - break; - case TSDB_DATA_TYPE_UINT: - tlen = sizeof(uint32_t); - break; - case TSDB_DATA_TYPE_BIGINT: - tlen = sizeof(int64_t); - break; - case TSDB_DATA_TYPE_UBIGINT: - tlen = sizeof(uint64_t); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(int32_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_FLOAT: - tlen = sizeof(float); + tlen = taosEncodeBinary(NULL, src, sizeof(float)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, sizeof(float)); + break; + case TSDB_DATA_TYPE_UINT: + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(uint32_t*)src, *dst, 1); + break; + case TSDB_DATA_TYPE_BIGINT: + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(int64_t*)src, *dst, 1); break; case TSDB_DATA_TYPE_DOUBLE: - tlen = sizeof(double); + tlen = taosEncodeBinary(NULL, src, sizeof(double)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, sizeof(double)); break; + case TSDB_DATA_TYPE_UBIGINT: + assert(0); + *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + indexInt2str(*(uint64_t*)src, *dst, 1); + break; + case TSDB_DATA_TYPE_NCHAR: { + tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); + *dst = *dst - tlen; + + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY +#if 1 + tlen = taosEncodeBinary(NULL, src, strlen(src)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, strlen(src)); + *dst = *dst - tlen; + break; +#endif + } + case TSDB_DATA_TYPE_VARBINARY: +#if 1 + tlen = taosEncodeBinary(NULL, src, strlen(src)); + *dst = taosMemoryCalloc(1, tlen + 1); + tlen = taosEncodeBinary(dst, src, strlen(src)); + *dst = *dst - tlen; + break; +#endif default: + TASSERT(0); break; } return tlen; } - -int32_t indexMayFillNumbericData(void* number, int32_t tlen) { - for (int i = 0; i < tlen; i++) { - int8_t* p = number; - if (p[i] == 0) { - p[i] = (uint8_t)'0'; - } - } - return 0; -} - -int32_t indexMayUnfillNumbericData(void* number, int32_t tlen) { - for (int i = 0; i < tlen; i++) { - int8_t* p = number; - if (p[i] == (uint8_t)'0') { - p[i] = 0; - } - } - return 0; -} diff --git a/source/libs/index/test/.utilUT.cc.swn b/source/libs/index/test/.utilUT.cc.swn deleted file mode 100644 index 3023c72b42057ca24f48699b8e9353645b647d6c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeI3ZEPGz8OJv@Eu!VovEw*NOXHVd+Cb~G=kwZLX4xQ%IqNDxpCHiG+YCR8&HU7gbaQR6g+1v>%XwwxohqrO>LhK&5=(H?zBU z=gs+EQWK}qp7hK4c6Vl<`OmYn&oeU_jco4UBsy0{H9m_q&Hmv2(HHufwWbF(%`#0} z9fyXuuu^(BZH(r_Ms`f!8MgJj z9WK~LdT4NEDpjq%;9Q_Upg^4p6zuWqIu~m3STrI_tXRHGym!e^orQzT0tEsE0tEsE z0tEsE0tEsE0tEsEE(HbbT$6SO4Y|y1%zf_nJm2>_+~0S(-=FuLZ*qUHcE1<76?pZy zhl8&`fk1&kfk1&kfk1&kfk1&kfk1&kfk1&kfk1&dP#`&`X^XgTLQVj%{_o}gFL3ZQ zJO&TLLvSw?UgHOU5 zSPECeYx6bjN%$!ogQIW<48vwv45G z*a6$18?J#Sc6z0Jx{Np%$1CGINkUld2zK;;&V!a3@ zZnUiAF45XzC(XS2d!ugO63SbtHW6)Es|issvUbO+ZMH~SR&47Ch4yrEE&?%#?_En< z#9k1hSjjDg{A6gnO*B&|y44s_HI18=2vM6bs3fwMuXQ4C?AFckki4u#gv)_|-7C?!bd^SWh+iaH=>f)+Yei}mY8Ox?if_lg>3 z))CviIdxQ#GG3pQ@##{=eNxu!c*n@{j*mMEB49iE$-a-)?DtrG#-d$$zsDLic1ACa zwh0_mnq1S=i^fGAgYnT>cTpa%I$hLxeyY0YtUWmiBa=((y_uZ7%d@Z1y6br8YPCp+ za9HTl2;uB>tMU|1CDZB=E*ls*#DHeV!E)LV7jwtBxcv6Uu{s~VnFfIdZtvbb+R)vd zr)d`Mu5QZ!uSD<~?+xNLMUXu<<8ZsFkhTrwGXr{_zt7im>KU)2wS8w6#5%98az^R7 zy8Em-N~N~Xj-T*ub;g&Ho8;}JZKTAMX^aW~K;zvshd-4fnQj>*gmQ!;d(|7>r?+!D z;|>X46`3lbRgSWfvG@Rj&=w9X;ml z*^q3g7c9vpR+hX!YuF(fu(*My7u2-SMv8kClr>U?uFB@-%Iz%nQMI~d)s;FpUhGbJ z1-GeUHo<^vwQ6~*wArUdLbr!$|$-u5BYb z(bYA`AJS1)7}-?1Fs6&;w%p9Ua&me3q5O6ump1aY@9;$098LPKO_^iVJpD?wRa}}; z;qc(Nk=835Co-;Zv@dVZP!DIi%|cc-YuGXf$SaqwaJ1XZWGWp_rY0*HHto`NQ%G%E z*O$~B&s_v1EF}#PG>!2Pf42diC$h$?G+l|I1wVALO($ z|M%AKA1ALr3J<~q@Fn;HOhOA>24~6T{|nE+58+O@3El_Kk;i`rz71c50_=cWVF|oU z&i*$z1Al=(!;fJ%$UJ`&^n=XruYf1X;~#<}@EOH6MgG_#!h{#KOPH>7Uaykynvbn)71Qj%}vQ&Ls>%Z^d}0LQQb09JxMzm z3b~1eD&gfCOmMk%dnNlCEXQnGM$e>jyF62*!EW_TntRnt(|fbiOm_`xZ3#8!HWbEN zXlHT0jgquxO6EYaQR+u&=8ZElsc)6VWKLCTnNuJ&nY4tUpNphteEgIod z%+vhSP%<)?f_Sq|rG9A*dD*3JJtbtDwv-@Bd#GHg+d#31EaodzOhUaC26YQAbQc(U zcc%0l3lYwZb@fYC;W*PdslCd2oy&ZyHkI>Ow>N9A+my|d%8Q+K1TtGW$K576R40&# zdWm>t!`x;}TTS|Z*u|0o{o~qHHYecB@|3Aql;1hyh_9Zv2Ksf?`ME<}74!K+P2Y=A zs6L1P1fqUT|*bW1*20CF0oFV5w0e8cGxD$pU0{>zj;3@bWd<*0n{|1oj`X7S% zaE3XN=iry{FdTv}Lk>O)P4GPH_RqpI@E9D1PeB)SKs(HbGpyO4fV*Koq@f2kKr_6` zdi~4r1RQ`AjKFQM1(v~5kZbxsfkSW*4uA=L&Qkf_V<<(om}Jta>bzFHNVVqXl&Bj#4rTq@QoQi2C~ElzpSj+Z5@Q^T z;SzCYJY!Ic$r!E}Q&u#3u6L(+LvT$)FPgn6(;hm+O})kKQ;}5#>1TC&=Njid@-}l$ zRCD&p@!bKd)rjScIv1s5%7=*jOm3D_57yglgP# Date: Mon, 16 May 2022 23:12:02 +0800 Subject: [PATCH 5/5] fix tag query error --- source/libs/index/src/indexComm.c | 2 + source/libs/index/src/indexFst.c | 2 +- source/libs/index/src/indexTfile.c | 7 ++- source/libs/index/test/jsonUT.cc | 95 +++++++++++++++++++++++++++++- 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 0c2e9e6b44..eea30bfb03 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -306,6 +306,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, src, sizeof(float)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, sizeof(float)); + *dst = *dst - tlen; break; case TSDB_DATA_TYPE_UINT: *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); @@ -319,6 +320,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, src, sizeof(double)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, sizeof(double)); + *dst = *dst - tlen; break; case TSDB_DATA_TYPE_UBIGINT: assert(0); diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index bc3ecea7a5..e2975fb7bc 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -1324,7 +1324,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } - streamStateDestroy(p); + // streamStateDestroy(p); continue; } FstTransition trn; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b787da117d..6c59986744 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -410,8 +410,9 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); cost = taosGetTimestampUs() - et; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, - tem->colName, tem->colVal, cost); + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, offset: %" PRIu64 + ", size: %d, time cost: %" PRIu64 "us", + tem->suid, tem->colName, tem->colVal, offset, (int)taosArrayGetSize(tr->total), cost); } fstSliceDestroy(&key); return 0; @@ -941,7 +942,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* // TODO(yihao): opt later WriterCtx* ctx = reader->ctx; // add block cache - char block[1024] = {0}; + char block[4096] = {0}; int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset); assert(nread >= sizeof(uint32_t)); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index d6ff7264d7..ff349b9b24 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -56,6 +56,29 @@ class JsonEnv : public ::testing::Test { SIndexJson* index; }; +static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, + int8_t operType = ADD_VALUE) { + SIndexTerm* term = + indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + tIndexJsonPut(index, terms, (int64_t)tableId); + + indexMultiTermDestroy(terms); +} +static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType, + SArray** result) { + std::string colName(colNam); + + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + + SArray* res = taosArrayInit(1, sizeof(uint64_t)); + indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType); + tIndexJsonSearch(index, mq, res); + indexMultiTermQueryDestroy(mq); + *result = res; +} TEST_F(JsonEnv, testWrite) { { std::string colName("test"); @@ -329,7 +352,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { } } -TEST_F(JsonEnv, testWriteJsonTfileAndCache) { +TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test1"); int val = 10; @@ -485,3 +508,73 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache) { indexMultiTermQueryDestroy(mq); } } +TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT2) { + { + int val = 10; + std::string colName("test1"); + for (int i = 0; i < 10000; i++) { + val += 1; + WriteData(index, colName, TSDB_DATA_TYPE_INT, &val, sizeof(val), i); + } + } + { + int val = 10; + std::string colName("test2xxx"); + std::string colVal("xxxxxxxxxxxxxxx"); + for (int i = 0; i < 100000; i++) { + val += 1; + WriteData(index, colName, TSDB_DATA_TYPE_BINARY, (void*)(colVal.c_str()), colVal.size(), i); + } + } + { + SArray* res = NULL; + std::string colName("test1"); + int val = 9; + Search(index, colName, TSDB_DATA_TYPE_INT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(10000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + std::string colName("test2xxx"); + std::string colVal("xxxxxxxxxxxxxxx"); + Search(index, colName, TSDB_DATA_TYPE_BINARY, (void*)(colVal.c_str()), colVal.size(), QUERY_TERM, &res); + EXPECT_EQ(100000, taosArrayGetSize(res)); + } +} +TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) { + { + float val = 10.0; + std::string colName("test1"); + for (int i = 0; i < 1000; i++) { + WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i); + } + } + { + float val = 2.0; + std::string colName("test1"); + for (int i = 0; i < 1000; i++) { + WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i); + } + } + { + SArray* res = NULL; + std::string colName("test1"); + float val = 1.9; + Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(2000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + std::string colName("test1"); + float val = 2.1; + Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } + { + std::string colName("test1"); + SArray* res = NULL; + float val = 2.1; + Search(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } +}