xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #12613 from taosdata/fix/filterFloat 

Fix/filter float
This commit is contained in:
Yihao Deng 2022-05-17 23:05:41 +08:00 committed by GitHub
parent 81ea237b4f 48cb91562e
commit 20f843bd6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 92 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
source/libs/index/src/indexComm.c
View File

@ -20,12 +20,13 @@
#include "tcompare.h"
#include "tdataformat.h"
#include "ttypes.h"
#include "tvariant.h"
char JSON_COLUMN[] = "JSON";
char JSON_VALUE_DELIM = '&';
char* indexInt2str(int64_t val, char* dst, int radix) {
char buffer[65];
char buffer[65] = {0};
char* p;
int64_t new_val;
uint64_t uval = (uint64_t)val;
@ -74,28 +75,70 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
}
TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) {
if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) {
return tDoCompare(func, cmptype, a, b);
}
#if 1
int8_t bytes = tDataTypes[dtype].bytes;
if (bytes == 1) {
int8_t va = taosStr2int64(a);
int8_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (bytes == 2) {
int16_t va = taosStr2int64(a);
int16_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (bytes == 4) {
int32_t va = taosStr2int64(a);
int32_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else {
if (dtype == TSDB_DATA_TYPE_TIMESTAMP) {
int64_t va = taosStr2int64(a);
int64_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) {
uint8_t va = taosStr2int64(a);
uint8_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_TINYINT) {
int8_t va = taosStr2int64(a);
int8_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_SMALLINT) {
int16_t va = taosStr2int64(a);
int16_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_USMALLINT) {
uint16_t va = taosStr2int64(a);
uint16_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_INT) {
int32_t va = taosStr2int64(a);
int32_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_UINT) {
uint32_t va = taosStr2int64(a);
uint32_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_BIGINT) {
int64_t va = taosStr2int64(a);
int64_t vb = taosStr2int64(b);
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_UBIGINT) {
uint64_t va, vb;
if (0 != toUInteger(a, strlen(a), 10, &va) || 0 != toUInteger(b, strlen(b), 10, &vb)) {
return CONTINUE;
}
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_FLOAT) {
float va = strtod(a, NULL);
if (errno == ERANGE && va == -1) {
return CONTINUE;
}
float vb = strtod(b, NULL);
if (errno == ERANGE && va == -1) {
return CONTINUE;
}
return tDoCompare(func, cmptype, &va, &vb);
} else if (dtype == TSDB_DATA_TYPE_DOUBLE) {
double va = strtod(a, NULL);
if (errno == ERANGE && va == -1) {
return CONTINUE;
}
double vb = strtod(b, NULL);
if (errno == ERANGE && va == -1) {
return CONTINUE;
}
return tDoCompare(func, cmptype, &va, &vb);
}
assert(0);
#endif
}
TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
@ -248,20 +291,16 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
break;
}
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
#if 1
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
#endif
}
case TSDB_DATA_TYPE_VARBINARY:
#if 1
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
#endif
default:
TASSERT(0);
break;
@ -272,86 +311,72 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
}
int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
int tlen = tDataTypes[type].bytes;
int32_t bufSize = 64;
switch (type) {
case TSDB_DATA_TYPE_TIMESTAMP:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int64_t*)src, *dst, -1);
break;
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
// tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
//*dst = taosMemoryCalloc(1, tlen + 1);
// tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint8_t*)src, *dst, 1);
break;
case TSDB_DATA_TYPE_TINYINT:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int8_t*)src, *dst, 1);
break;
case TSDB_DATA_TYPE_SMALLINT:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int16_t*)src, *dst, -1);
break;
case TSDB_DATA_TYPE_USMALLINT:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint16_t*)src, *dst, -1);
break;
case TSDB_DATA_TYPE_INT:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int32_t*)src, *dst, -1);
break;
case TSDB_DATA_TYPE_FLOAT:
tlen = taosEncodeBinary(NULL, src, sizeof(float));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, sizeof(float));
*dst = (char*) * dst - tlen;
break;
case TSDB_DATA_TYPE_UINT:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint32_t*)src, *dst, 1);
break;
case TSDB_DATA_TYPE_BIGINT:
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
indexInt2str(*(int64_t*)src, *dst, 1);
break;
case TSDB_DATA_TYPE_DOUBLE:
tlen = taosEncodeBinary(NULL, src, sizeof(double));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, sizeof(double));
*dst = (char*) * dst - tlen;
*dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%" PRIu64, *(uint64_t*)src);
break;
case TSDB_DATA_TYPE_UBIGINT:
assert(0);
*dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
*dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint64_t*)src, *dst, 1);
case TSDB_DATA_TYPE_FLOAT:
*dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%.9lf", *(float*)src);
break;
case TSDB_DATA_TYPE_DOUBLE:
*dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%.9lf", *(double*)src);
break;
case TSDB_DATA_TYPE_NCHAR: {
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
*dst = (char*) * dst - tlen;
*dst = *dst - tlen;
break;
}
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
#if 1
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
*dst = (char*) * dst - tlen;
*dst = (char*)*dst - tlen;
break;
#endif
}
case TSDB_DATA_TYPE_VARBINARY:
#if 1
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
*dst = (char*) * dst - tlen;
*dst = (char*)*dst - tlen;
break;
#endif
default:
TASSERT(0);
break;

19
source/libs/index/src/indexTfile.c
View File

@ -20,6 +20,7 @@ p *
#include "indexFstCountingWriter.h"
#include "indexUtil.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tcoding.h"
#include "tcompare.h"
@ -472,16 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
int32_t sz = 0;
char* ch = (char*)fstSliceData(s, &sz);
char* tmp = taosMemoryCalloc(1, sz + 1);
memcpy(tmp, ch, sz);
// char* tmp = taosMemoryCalloc(1, sz + 1);
// memcpy(tmp, ch, sz);
if (0 != strncmp(tmp, p, skip)) {
if (0 != strncmp(ch, p, skip)) {
swsResultDestroy(rt);
taosMemoryFree(tmp);
// taosMemoryFree(tmp);
break;
}
TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
if (MATCH == cond) {
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
@ -490,7 +491,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
swsResultDestroy(rt);
break;
}
taosMemoryFree(tmp);
// taosMemoryFree(tmp);
swsResultDestroy(rt);
}
streamWithStateDestroy(st);
@ -533,10 +534,12 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
tfileGenFileFullName(fullname, path, suid, colName, version);
WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
if (wc == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr());
return NULL;
}
indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
TFileReader* reader = tfileReaderCreate(wc);
return reader;
@ -613,9 +616,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
if (tfileWriteData(tw, v) != 0) {
indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
(int)taosArrayGetSize(v->tableId));
// printf("write faile\n");
} else {
// printf("write sucee\n");
// indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
// (int)taosArrayGetSize(v->tableId));

2
source/libs/index/test/jsonUT.cc
View File

@ -553,7 +553,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
float val = 2.0;
std::string colName("test1");
for (int i = 0; i < 1000; i++) {
WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i + 1000);
}
}
{