xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update charset conv

This commit is contained in:
yihaoDeng 2022-12-23 22:21:28 +08:00
parent 00370ddccf
commit 71bbc4fdd0
4 changed files with 132 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
source/libs/index/inc/indexFstFile.h
View File

@ -44,6 +44,10 @@ typedef struct IFileCtx {
bool readOnly; bool readOnly;
char buf[256]; char buf[256];
int64_t size; int64_t size;
char wBuf[4096];
int32_t wBufOffset;
#ifdef USE_MMAP #ifdef USE_MMAP
char* ptr; char* ptr;
#endif #endif

63
source/libs/index/src/indexFstFile.c
View File

@ -38,14 +38,41 @@ static FORCE_INLINE void idxGenLRUKey(char* buf, const char* path, int32_t block
return; return;
} }
static FORCE_INLINE int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) { static FORCE_INLINE int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) {
int tlen = len;
if (ctx->type == TFILE) { if (ctx->type == TFILE) {
int nwr = taosWriteFile(ctx->file.pFile, buf, len); int32_t cap = sizeof(ctx->file.wBuf);
assert(nwr == len); if (len + ctx->file.wBufOffset >= cap) {
int32_t nw = cap - ctx->file.wBufOffset;
memcpy(ctx->file.wBuf + ctx->file.wBufOffset, buf, nw);
taosWriteFile(ctx->file.pFile, ctx->file.wBuf, cap);
memset(ctx->file.wBuf, 0, cap);
ctx->file.wBufOffset = 0;
len -= nw;
buf += nw;
nw = (len / cap) * cap;
if (nw != 0) {
taosWriteFile(ctx->file.pFile, buf, nw);
}
len -= nw;
buf += nw;
if (len != 0) {
memcpy(ctx->file.wBuf, buf, len);
}
ctx->file.wBufOffset += len;
} else {
memcpy(ctx->file.wBuf + ctx->file.wBufOffset, buf, len);
ctx->file.wBufOffset += len;
}
} else { } else {
memcpy(ctx->mem.buf + ctx->offset, buf, len); memcpy(ctx->mem.buf + ctx->offset, buf, len);
} }
ctx->offset += len; ctx->offset += tlen;
return len; return tlen;
} }
static FORCE_INLINE int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) { static FORCE_INLINE int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) {
int nRead = 0; int nRead = 0;
@ -127,14 +154,22 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
} }
static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) { static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) {
if (ctx->type == TFILE) { if (ctx->type == TFILE) {
int64_t file_size = 0; if (ctx->file.readOnly == false) {
taosStatFile(ctx->file.buf, &file_size, NULL); return ctx->offset;
return (int)file_size; } else {
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
return (int)file_size;
}
} }
return 0; return 0;
} }
static FORCE_INLINE int idxFileCtxDoFlush(IFileCtx* ctx) { static FORCE_INLINE int idxFileCtxDoFlush(IFileCtx* ctx) {
if (ctx->type == TFILE) { if (ctx->type == TFILE) {
if (ctx->file.wBufOffset > 0) {
int32_t nw = taosWriteFile(ctx->file.pFile, ctx->file.wBuf, ctx->file.wBufOffset);
ctx->file.wBufOffset = 0;
}
taosFsyncFile(ctx->file.pFile); taosFsyncFile(ctx->file.pFile);
} else { } else {
// do nothing // do nothing
@ -157,10 +192,14 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
taosFtruncateFile(ctx->file.pFile, 0); taosFtruncateFile(ctx->file.pFile, 0);
taosStatFile(path, &ctx->file.size, NULL); taosStatFile(path, &ctx->file.size, NULL);
memset(ctx->file.wBuf, 0, sizeof(ctx->file.wBuf));
ctx->file.wBufOffset = 0;
} else { } else {
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL); taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL);
ctx->file.wBufOffset = 0;
#ifdef USE_MMAP #ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif #endif
@ -195,6 +234,10 @@ void idxFileCtxDestroy(IFileCtx* ctx, bool remove) {
if (ctx->type == TMEMORY) { if (ctx->type == TMEMORY) {
taosMemoryFree(ctx->mem.buf); taosMemoryFree(ctx->mem.buf);
} else { } else {
if (ctx->file.wBufOffset > 0) {
int32_t nw = taosWriteFile(ctx->file.pFile, ctx->file.wBuf, ctx->file.wBufOffset);
ctx->file.wBufOffset = 0;
}
ctx->flush(ctx); ctx->flush(ctx);
taosCloseFile(&ctx->file.pFile); taosCloseFile(&ctx->file.pFile);
if (ctx->file.readOnly) { if (ctx->file.readOnly) {
@ -202,10 +245,6 @@ void idxFileCtxDestroy(IFileCtx* ctx, bool remove) {
munmap(ctx->file.ptr, ctx->file.size); munmap(ctx->file.ptr, ctx->file.size);
#endif #endif
} }
if (ctx->file.readOnly == false) {
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
}
if (remove) { if (remove) {
unlink(ctx->file.buf); unlink(ctx->file.buf);
} }

10
source/os/src/osEnv.c
View File

@ -37,11 +37,11 @@ float tsNumOfCores = 0;
int64_t tsTotalMemoryKB = 0; int64_t tsTotalMemoryKB = 0;
char *tsProcPath = NULL; char *tsProcPath = NULL;
char tsSIMDBuiltins = 0; char tsSIMDBuiltins = 0;
char tsSSE42Enable = 0; char tsSSE42Enable = 0;
char tsAVXEnable = 0; char tsAVXEnable = 0;
char tsAVX2Enable = 0; char tsAVX2Enable = 0;
char tsFMAEnable = 0; char tsFMAEnable = 0;
void osDefaultInit() { void osDefaultInit() {
taosSeedRand(taosSafeRand()); taosSeedRand(taosSafeRand());

119
source/os/src/osString.c
View File

@ -116,39 +116,36 @@ TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4)
return memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4));
} }
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1;
#else
iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len;
if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
iconv_close(cd);
return -1;
}
iconv_close(cd);
return (int32_t)(ucs4_max_len - outLen);
#endif
}
typedef struct { typedef struct {
iconv_t conv; iconv_t conv;
int8_t inUse; int8_t inUse;
} SConv; } SConv;
SConv *gConv = NULL; typedef enum { M2C = 0, C2M } ConvType;
int32_t convUsed = 0;
int32_t gConvMaxNum = 0; // 0: Mbs --> Ucs4
// 1: Ucs4--> Mbs
SConv *gConv[2] = {NULL, NULL};
int32_t convUsed[2] = {0, 0};
int32_t gConvMaxNum[2] = {0, 0};
int32_t taosConvInit(void) { int32_t taosConvInit(void) {
gConvMaxNum = 512; int8_t M2C = 0;
gConv = taosMemoryCalloc(gConvMaxNum, sizeof(SConv)); gConvMaxNum[M2C] = 512;
for (int32_t i = 0; i < gConvMaxNum; ++i) { gConvMaxNum[1 - M2C] = 512;
gConv[i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
if ((iconv_t)-1 == gConv[i].conv || (iconv_t)0 == gConv[i].conv) { gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv));
gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv));
for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) {
gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
if ((iconv_t)-1 == gConv[M2C][i].conv || (iconv_t)0 == gConv[M2C][i].conv) {
return -1;
}
}
for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) {
gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == gConv[1 - M2C][i].conv || (iconv_t)0 == gConv[1 - M2C][i].conv) {
return -1; return -1;
} }
} }
@ -157,23 +154,33 @@ int32_t taosConvInit(void) {
} }
void taosConvDestroy() { void taosConvDestroy() {
for (int32_t i = 0; i < gConvMaxNum; ++i) { int8_t M2C = 0;
iconv_close(gConv[i].conv); for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) {
iconv_close(gConv[M2C][i].conv);
} }
taosMemoryFreeClear(gConv); for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) {
gConvMaxNum = -1; iconv_close(gConv[1 - M2C][i].conv);
}
taosMemoryFreeClear(gConv[M2C]);
taosMemoryFreeClear(gConv[1 - M2C]);
gConvMaxNum[M2C] = -1;
gConvMaxNum[1 - M2C] = -1;
} }
iconv_t taosAcquireConv(int32_t *idx) { iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
if (gConvMaxNum <= 0) { if (gConvMaxNum[type] <= 0) {
*idx = -1; *idx = -1;
return iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); if (type == M2C) {
return iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
} else {
return iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
}
} }
while (true) { while (true) {
int32_t used = atomic_add_fetch_32(&convUsed, 1); int32_t used = atomic_add_fetch_32(&convUsed[type], 1);
if (used > gConvMaxNum) { if (used > gConvMaxNum[type]) {
used = atomic_sub_fetch_32(&convUsed, 1); used = atomic_sub_fetch_32(&convUsed[type], 1);
sched_yield(); sched_yield();
continue; continue;
} }
@ -181,31 +188,31 @@ iconv_t taosAcquireConv(int32_t *idx) {
break; break;
} }
int32_t startId = taosGetSelfPthreadId() % gConvMaxNum; int32_t startId = taosGetSelfPthreadId() % gConvMaxNum[type];
while (true) { while (true) {
if (gConv[startId].inUse) { if (gConv[type][startId].inUse) {
startId = (startId + 1) % gConvMaxNum; startId = (startId + 1) % gConvMaxNum[type];
continue; continue;
} }
int8_t old = atomic_val_compare_exchange_8(&gConv[startId].inUse, 0, 1); int8_t old = atomic_val_compare_exchange_8(&gConv[type][startId].inUse, 0, 1);
if (0 == old) { if (0 == old) {
break; break;
} }
} }
*idx = startId; *idx = startId;
return gConv[startId].conv; return gConv[type][startId].conv;
} }
void taosReleaseConv(int32_t idx, iconv_t conv) { void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) {
if (idx < 0) { if (idx < 0) {
iconv_close(conv); iconv_close(conv);
return; return;
} }
atomic_store_8(&gConv[idx].inUse, 0); atomic_store_8(&gConv[type][idx].inUse, 0);
atomic_sub_fetch_32(&convUsed, 1); atomic_sub_fetch_32(&convUsed[type], 1);
} }
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
@ -216,15 +223,15 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
memset(ucs4, 0, ucs4_max_len); memset(ucs4, 0, ucs4_max_len);
int32_t idx = -1; int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx); iconv_t conv = taosAcquireConv(&idx, M2C);
size_t ucs4_input_len = mbsLength; size_t ucs4_input_len = mbsLength;
size_t outLeft = ucs4_max_len; size_t outLeft = ucs4_max_len;
if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) {
taosReleaseConv(idx, conv); taosReleaseConv(idx, conv, M2C);
return false; return false;
} }
taosReleaseConv(idx, conv); taosReleaseConv(idx, conv, M2C);
if (len != NULL) { if (len != NULL) {
*len = (int32_t)(ucs4_max_len - outLeft); *len = (int32_t)(ucs4_max_len - outLeft);
if (*len < 0) { if (*len < 0) {
@ -236,6 +243,24 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
#endif #endif
} }
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
return -1;
#else
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M);
size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len;
if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
taosReleaseConv(idx, conv, C2M);
return -1;
}
taosReleaseConv(idx, conv, C2M);
return (int32_t)(ucs4_max_len - outLen);
#endif
}
bool taosValidateEncodec(const char *encodec) { bool taosValidateEncodec(const char *encodec) {
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV #ifdef DISALLOW_NCHAR_WITHOUT_ICONV
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");