From e96ff087f2ea91a58e229c1916f6062cb7de7300 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 17 May 2022 22:22:39 +0800 Subject: [PATCH 1/2] fix: index failed to filter float/double data --- source/libs/index/src/indexComm.c | 131 +++++++++++++++++------------ source/libs/index/src/indexTfile.c | 7 +- source/libs/index/test/jsonUT.cc | 2 +- 3 files changed, 83 insertions(+), 57 deletions(-) diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index eea30bfb03..9d3e0a5cc9 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -20,12 +20,13 @@ #include "tcompare.h" #include "tdataformat.h" #include "ttypes.h" +#include "tvariant.h" char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; char* indexInt2str(int64_t val, char* dst, int radix) { - char buffer[65]; + char buffer[65] = {0}; char* p; int64_t new_val; uint64_t uval = (uint64_t)val; @@ -74,28 +75,70 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { - if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) { + if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) { return tDoCompare(func, cmptype, a, b); } #if 1 - int8_t bytes = tDataTypes[dtype].bytes; - if (bytes == 1) { - int8_t va = taosStr2int64(a); - int8_t vb = taosStr2int64(b); - return tDoCompare(func, cmptype, &va, &vb); - } else if (bytes == 2) { - int16_t va = taosStr2int64(a); - int16_t vb = taosStr2int64(b); - return tDoCompare(func, cmptype, &va, &vb); - } else if (bytes == 4) { - int32_t va = taosStr2int64(a); - int32_t vb = taosStr2int64(b); - return tDoCompare(func, cmptype, &va, &vb); - } else { + if (dtype == TSDB_DATA_TYPE_TIMESTAMP) { int64_t va = taosStr2int64(a); int64_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) { + uint8_t va = taosStr2int64(a); + uint8_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_TINYINT) { + int8_t va = taosStr2int64(a); + int8_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_SMALLINT) { + int16_t va = taosStr2int64(a); + int16_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_USMALLINT) { + uint16_t va = taosStr2int64(a); + uint16_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_INT) { + int32_t va = taosStr2int64(a); + int32_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_UINT) { + uint32_t va = taosStr2int64(a); + uint32_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_BIGINT) { + int64_t va = taosStr2int64(a); + int64_t vb = taosStr2int64(b); + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_UBIGINT) { + uint64_t va, vb; + if (0 != toUInteger(a, strlen(a), 10, &va) || 0 != toUInteger(b, strlen(b), 10, &vb)) { + return CONTINUE; + } + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_FLOAT) { + float va = strtod(a, NULL); + if (errno == ERANGE && va == -1) { + return CONTINUE; + } + float vb = strtod(b, NULL); + if (errno == ERANGE && va == -1) { + return CONTINUE; + } + return tDoCompare(func, cmptype, &va, &vb); + } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { + double va = strtod(a, NULL); + if (errno == ERANGE && va == -1) { + return CONTINUE; + } + double vb = strtod(b, NULL); + if (errno == ERANGE && va == -1) { + return CONTINUE; + } + return tDoCompare(func, cmptype, &va, &vb); } + assert(0); #endif } TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { @@ -248,20 +291,16 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY -#if 1 tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); break; -#endif } case TSDB_DATA_TYPE_VARBINARY: -#if 1 tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); break; -#endif default: TASSERT(0); break; @@ -271,87 +310,73 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) { return tlen; } int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { - int tlen = tDataTypes[type].bytes; - + int tlen = tDataTypes[type].bytes; + int32_t bufSize = 64; switch (type) { case TSDB_DATA_TYPE_TIMESTAMP: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int64_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: - // tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src); - //*dst = taosMemoryCalloc(1, tlen + 1); - // tlen = taosEncodeFixedU8(dst, *(uint8_t*)src); - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint8_t*)src, *dst, 1); break; case TSDB_DATA_TYPE_TINYINT: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int8_t*)src, *dst, 1); break; case TSDB_DATA_TYPE_SMALLINT: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int16_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_USMALLINT: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint16_t*)src, *dst, -1); break; case TSDB_DATA_TYPE_INT: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int32_t*)src, *dst, -1); break; - case TSDB_DATA_TYPE_FLOAT: - tlen = taosEncodeBinary(NULL, src, sizeof(float)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, sizeof(float)); - *dst = *dst - tlen; - break; case TSDB_DATA_TYPE_UINT: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint32_t*)src, *dst, 1); break; case TSDB_DATA_TYPE_BIGINT: - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); - indexInt2str(*(int64_t*)src, *dst, 1); - break; - case TSDB_DATA_TYPE_DOUBLE: - tlen = taosEncodeBinary(NULL, src, sizeof(double)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, sizeof(double)); - *dst = *dst - tlen; + *dst = taosMemoryCalloc(1, bufSize + 1); + sprintf(*dst, "%" PRIu64, *(uint64_t*)src); break; case TSDB_DATA_TYPE_UBIGINT: - assert(0); - *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1); + *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint64_t*)src, *dst, 1); + case TSDB_DATA_TYPE_FLOAT: + *dst = taosMemoryCalloc(1, bufSize + 1); + sprintf(*dst, "%.9lf", *(float*)src); + break; + case TSDB_DATA_TYPE_DOUBLE: + *dst = taosMemoryCalloc(1, bufSize + 1); + sprintf(*dst, "%.9lf", *(double*)src); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = *dst - tlen; - break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY -#if 1 tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); *dst = *dst - tlen; break; -#endif } case TSDB_DATA_TYPE_VARBINARY: -#if 1 tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); *dst = *dst - tlen; break; -#endif default: TASSERT(0); break; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 6c59986744..83ffe13f8c 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -20,6 +20,7 @@ p * #include "indexFstCountingWriter.h" #include "indexUtil.h" #include "taosdef.h" +#include "taoserror.h" #include "tcoding.h" #include "tcompare.h" @@ -533,10 +534,12 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); - indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size); if (wc == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr()); return NULL; } + indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size); TFileReader* reader = tfileReaderCreate(wc); return reader; @@ -613,9 +616,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { if (tfileWriteData(tw, v) != 0) { indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); - // printf("write faile\n"); } else { - // printf("write sucee\n"); // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, // (int)taosArrayGetSize(v->tableId)); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index ff349b9b24..135ae61e83 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -553,7 +553,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) { float val = 2.0; std::string colName("test1"); for (int i = 0; i < 1000; i++) { - WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i); + WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i + 1000); } } { From 3a76f8303251157edc78d9cf601cee0ea4873da8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 17 May 2022 22:31:11 +0800 Subject: [PATCH 2/2] fix: index failed to filter float/double data --- source/libs/index/src/indexTfile.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 83ffe13f8c..9533d3429e 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -473,16 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR int32_t sz = 0; char* ch = (char*)fstSliceData(s, &sz); - char* tmp = taosMemoryCalloc(1, sz + 1); - memcpy(tmp, ch, sz); + // char* tmp = taosMemoryCalloc(1, sz + 1); + // memcpy(tmp, ch, sz); - if (0 != strncmp(tmp, p, skip)) { + if (0 != strncmp(ch, p, skip)) { swsResultDestroy(rt); - taosMemoryFree(tmp); + // taosMemoryFree(tmp); break; } - TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); + TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); if (MATCH == cond) { tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); @@ -491,7 +491,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR swsResultDestroy(rt); break; } - taosMemoryFree(tmp); + // taosMemoryFree(tmp); swsResultDestroy(rt); } streamWithStateDestroy(st);