Merge branch '3.0' of https://github.com/taosdata/TDengine into 3.0
This commit is contained in:
plum-lihui
commit
2b6d4014dd
|
|
@ -2052,27 +2052,19 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
|
|||
buf = taosDecodeFixedI64(buf, &pSma->sliding);
|
||||
|
||||
if (pSma->exprLen > 0) {
|
||||
pSma->expr = (char*)calloc(pSma->exprLen, 1);
|
||||
if (pSma->expr != NULL) {
|
||||
buf = taosDecodeStringTo(buf, pSma->expr);
|
||||
} else {
|
||||
if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
|
||||
tdDestroyTSma(pSma);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
} else {
|
||||
pSma->expr = NULL;
|
||||
}
|
||||
|
||||
if (pSma->tagsFilterLen > 0) {
|
||||
pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1);
|
||||
if (pSma->tagsFilter != NULL) {
|
||||
buf = taosDecodeStringTo(buf, pSma->tagsFilter);
|
||||
} else {
|
||||
if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
|
||||
tdDestroyTSma(pSma);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
} else {
|
||||
pSma->tagsFilter = NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,8 +26,9 @@ typedef struct SDBFile SDBFile;
|
|||
typedef DB_ENV* TDBEnv;
|
||||
|
||||
struct SDBFile {
|
||||
DB* pDB;
|
||||
char* path;
|
||||
int32_t fid;
|
||||
DB* pDB;
|
||||
char* path;
|
||||
};
|
||||
|
||||
int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);
|
||||
|
|
|
|||
|
|
@ -884,7 +884,7 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) {
|
|||
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
|
||||
STSmaWrapper *pSW = NULL;
|
||||
|
||||
pSW = calloc(sizeof(*pSW), 1);
|
||||
pSW = calloc(1, sizeof(*pSW));
|
||||
if (pSW == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -156,10 +156,6 @@ static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) {
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
if (*pEnv) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (*pEnv == NULL) {
|
||||
if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) {
|
||||
return TSDB_CODE_FAILED;
|
||||
|
|
@ -260,10 +256,15 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
|
|||
int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
|
||||
if (pSmaStat) {
|
||||
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
|
||||
SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
|
||||
void *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
|
||||
while (item != NULL) {
|
||||
tfree(item->pSma);
|
||||
taosHashCleanup(item->expiredWindows);
|
||||
SSmaStatItem *pItem = *(SSmaStatItem **)item;
|
||||
if (pItem != NULL) {
|
||||
tdDestroyTSma(pItem->pSma);
|
||||
tfree(pItem->pSma);
|
||||
taosHashCleanup(pItem->expiredWindows);
|
||||
tfree(pItem);
|
||||
}
|
||||
item = taosHashIterate(pSmaStat->smaStatItems, item);
|
||||
}
|
||||
taosHashCleanup(pSmaStat->smaStatItems);
|
||||
|
|
@ -292,9 +293,10 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
|
|||
|
||||
// init sma env
|
||||
tsdbLockRepo(pTsdb);
|
||||
if (pTsdb->pTSmaEnv == NULL) {
|
||||
pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&pTsdb->pTSmaEnv) : atomic_load_ptr(&pTsdb->pRSmaEnv);
|
||||
if (pEnv == NULL) {
|
||||
char rname[TSDB_FILENAME_LEN] = {0};
|
||||
char aname[TSDB_FILENAME_LEN * 2 + 32] = {0}; // TODO: make TMPNAME_LEN public as TSDB_FILENAME_LEN?
|
||||
char aname[TSDB_FILENAME_LEN] = {0}; // use TSDB_FILENAME_LEN currently
|
||||
|
||||
SDiskID did = {0};
|
||||
tfsAllocDisk(pTsdb->pTfs, TFS_PRIMARY_LEVEL, &did);
|
||||
|
|
@ -315,11 +317,8 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
|
||||
atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv);
|
||||
} else {
|
||||
atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
|
||||
}
|
||||
(smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&pTsdb->pTSmaEnv, pEnv)
|
||||
: atomic_store_ptr(&pTsdb->pRSmaEnv, pEnv);
|
||||
}
|
||||
tsdbUnlockRepo(pTsdb);
|
||||
|
||||
|
|
@ -359,8 +358,10 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) {
|
|||
SSmaStat *pStat = SMA_ENV_STAT(pEnv);
|
||||
SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
|
||||
|
||||
TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL);
|
||||
|
||||
tsdbRefSmaStat(pTsdb, pStat);
|
||||
SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
|
||||
SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
|
||||
if (pItem == NULL) {
|
||||
pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state
|
||||
if (pItem == NULL) {
|
||||
|
|
@ -421,9 +422,9 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind
|
|||
tsdbRefSmaStat(pTsdb, pStat);
|
||||
|
||||
if (pStat && pStat->smaStatItems) {
|
||||
pItem = *(SSmaStatItem **)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
|
||||
pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
|
||||
}
|
||||
if (pItem != NULL) {
|
||||
if ((pItem != NULL) && ((pItem = *(SSmaStatItem **)pItem) != NULL)) {
|
||||
// pItem resides in hash buffer all the time unless drop sma index
|
||||
// TODO: multithread protect
|
||||
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
|
||||
|
|
@ -494,7 +495,7 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
|
|||
* @brief Insert TSma data blocks to DB File build by B+Tree
|
||||
*
|
||||
* @param pSmaH
|
||||
* @param smaKey
|
||||
* @param smaKey tableUid-colId-skeyOfWindow(8-2-8)
|
||||
* @param keyLen
|
||||
* @param pData
|
||||
* @param dataLen
|
||||
|
|
@ -502,12 +503,11 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
|
|||
*/
|
||||
static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
|
||||
SDBFile *pDBFile = &pSmaH->dFile;
|
||||
|
||||
// TODO: insert sma data blocks into B+Tree
|
||||
tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
|
||||
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
|
||||
*(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
|
||||
|
||||
// TODO: insert sma data blocks into B+Tree(TDB)
|
||||
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
|
@ -564,34 +564,34 @@ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit
|
|||
return interval / 1e3;
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
|
||||
return interval / 1e6;
|
||||
} else {
|
||||
} else { // ms
|
||||
return interval;
|
||||
}
|
||||
break;
|
||||
case TSDB_TIME_PRECISION_MICRO:
|
||||
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
|
||||
return interval;
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
|
||||
return interval / 1e3;
|
||||
} else {
|
||||
} else { // ms
|
||||
return interval * 1e3;
|
||||
}
|
||||
break;
|
||||
case TSDB_TIME_PRECISION_NANO:
|
||||
if (TIME_UNIT_MICROSECOND == intervalUnit) {
|
||||
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
|
||||
return interval * 1e3;
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
|
||||
return interval;
|
||||
} else {
|
||||
} else { // ms
|
||||
return interval * 1e6;
|
||||
}
|
||||
break;
|
||||
default: // ms
|
||||
if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
|
||||
return interval / 1e3;
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
|
||||
} else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
|
||||
return interval / 1e6;
|
||||
} else {
|
||||
} else { // ms
|
||||
return interval;
|
||||
}
|
||||
break;
|
||||
|
|
@ -663,9 +663,13 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
|
|||
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) {
|
||||
STsdb *pTsdb = pSmaH->pTsdb;
|
||||
ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL);
|
||||
|
||||
pSmaH->dFile.fid = fid;
|
||||
|
||||
char tSmaFile[TSDB_FILENAME_LEN] = {0};
|
||||
snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid);
|
||||
pSmaH->dFile.path = strdup(tSmaFile);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -705,7 +709,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
|
|||
STsdbCfg * pCfg = REPO_CFG(pTsdb);
|
||||
STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
|
||||
|
||||
if (!pTsdb->pTSmaEnv) {
|
||||
if (!atomic_load_ptr(&pTsdb->pTSmaEnv)) {
|
||||
terrno = TSDB_CODE_INVALID_PTR;
|
||||
tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
|
||||
return terrno;
|
||||
|
|
@ -883,15 +887,15 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
|
|||
static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
|
||||
int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey,
|
||||
int32_t nMaxResult) {
|
||||
if (!pTsdb->pTSmaEnv) {
|
||||
if (!atomic_load_ptr(&pTsdb->pTSmaEnv)) {
|
||||
terrno = TSDB_CODE_INVALID_PTR;
|
||||
tsdbWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
tsdbRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
|
||||
SSmaStatItem *pItem = *(SSmaStatItem **)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid));
|
||||
if (pItem == NULL) {
|
||||
SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid));
|
||||
if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) {
|
||||
// Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
|
||||
// it's NULL.
|
||||
tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
|
||||
|
|
|
|||
|
|
@ -144,6 +144,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
// record current timezone of server side
|
||||
tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN);
|
||||
|
||||
if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) {
|
||||
// TODO: handle error
|
||||
tdDestroyTSma(&vCreateSmaReq.tSma);
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
|
|||
STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma};
|
||||
uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper);
|
||||
|
||||
void *buf = calloc(bufLen, 1);
|
||||
void *buf = calloc(1, bufLen);
|
||||
ASSERT_NE(buf, nullptr);
|
||||
|
||||
STSmaWrapper *pSW = (STSmaWrapper *)buf;
|
||||
|
|
@ -84,6 +84,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
|
|||
}
|
||||
|
||||
// resource release
|
||||
tfree(pSW);
|
||||
tdDestroyTSma(&tSma);
|
||||
tdDestroyTSmaWrapper(&dstTSmaWrapper);
|
||||
}
|
||||
|
|
@ -113,7 +114,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
|
|||
tSma.tableUid = tbUid;
|
||||
|
||||
tSma.exprLen = strlen(expr);
|
||||
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1);
|
||||
tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
|
||||
ASSERT_NE(tSma.expr, nullptr);
|
||||
tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
|
||||
|
||||
|
|
@ -251,12 +252,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
|
|||
tSma.tableUid = tbUid;
|
||||
|
||||
tSma.exprLen = strlen(expr);
|
||||
tSma.expr = (char *)calloc(tSma.exprLen + 1, 1);
|
||||
tSma.expr = (char *)calloc(1, tSma.exprLen + 1);
|
||||
ASSERT_NE(tSma.expr, nullptr);
|
||||
tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
|
||||
|
||||
tSma.tagsFilterLen = strlen(tagsFilter);
|
||||
tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1);
|
||||
tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1);
|
||||
ASSERT_NE(tSma.tagsFilter, nullptr);
|
||||
tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
|
||||
|
||||
|
|
@ -273,20 +274,20 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
|
|||
|
||||
// step 2: insert data
|
||||
STSmaDataWrapper *pSmaData = NULL;
|
||||
STsdb tsdb = {0};
|
||||
STsdbCfg * pCfg = &tsdb.config;
|
||||
STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb));
|
||||
STsdbCfg * pCfg = &pTsdb->config;
|
||||
|
||||
tsdb.pMeta = pMeta;
|
||||
tsdb.vgId = 2;
|
||||
tsdb.config.daysPerFile = 10; // default days is 10
|
||||
tsdb.config.keep1 = 30;
|
||||
tsdb.config.keep2 = 90;
|
||||
tsdb.config.keep = 365;
|
||||
tsdb.config.precision = TSDB_TIME_PRECISION_MILLI;
|
||||
tsdb.config.update = TD_ROW_OVERWRITE_UPDATE;
|
||||
tsdb.config.compression = TWO_STAGE_COMP;
|
||||
pTsdb->pMeta = pMeta;
|
||||
pTsdb->vgId = 2;
|
||||
pTsdb->config.daysPerFile = 10; // default days is 10
|
||||
pTsdb->config.keep1 = 30;
|
||||
pTsdb->config.keep2 = 90;
|
||||
pTsdb->config.keep = 365;
|
||||
pTsdb->config.precision = TSDB_TIME_PRECISION_MILLI;
|
||||
pTsdb->config.update = TD_ROW_OVERWRITE_UPDATE;
|
||||
pTsdb->config.compression = TWO_STAGE_COMP;
|
||||
|
||||
switch (tsdb.config.precision) {
|
||||
switch (pTsdb->config.precision) {
|
||||
case TSDB_TIME_PRECISION_MILLI:
|
||||
skey1 *= 1e3;
|
||||
break;
|
||||
|
|
@ -304,12 +305,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
|
|||
SDiskCfg pDisks = {.level = 0, .primary = 1};
|
||||
strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN);
|
||||
int32_t numOfDisks = 1;
|
||||
tsdb.pTfs = tfsOpen(&pDisks, numOfDisks);
|
||||
ASSERT_NE(tsdb.pTfs, nullptr);
|
||||
pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
|
||||
ASSERT_NE(pTsdb->pTfs, nullptr);
|
||||
|
||||
char *msg = (char *)calloc(1, 100);
|
||||
ASSERT_NE(msg, nullptr);
|
||||
ASSERT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
|
||||
ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
|
||||
|
||||
// init
|
||||
int32_t allocCnt = 0;
|
||||
|
|
@ -367,13 +368,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
|
|||
ASSERT_GE(bufSize, pSmaData->dataLen);
|
||||
|
||||
// execute
|
||||
ASSERT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
|
||||
ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
|
||||
|
||||
// step 3: query
|
||||
uint32_t checkDataCnt = 0;
|
||||
for (int32_t t = 0; t < numOfTables; ++t) {
|
||||
for (col_id_t c = 0; c < numOfCols; ++c) {
|
||||
ASSERT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
|
||||
ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
|
||||
c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1),
|
||||
TSDB_CODE_SUCCESS);
|
||||
++checkDataCnt;
|
||||
|
|
@ -383,9 +384,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
|
|||
printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt);
|
||||
|
||||
// release data
|
||||
tfree(msg);
|
||||
taosTZfree(buf);
|
||||
// release meta
|
||||
tdDestroyTSma(&tSma);
|
||||
tfsClose(pTsdb->pTfs);
|
||||
tsdbClose(pTsdb);
|
||||
metaClose(pMeta);
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -204,7 +204,8 @@ void tfsDirname(const STfsFile *pFile, char *dest) {
|
|||
|
||||
void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) {
|
||||
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
|
||||
snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
|
||||
|
||||
snprintf(aname, TSDB_FILENAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
|
||||
}
|
||||
|
||||
int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); }
|
||||
|
|
|
|||
|
|
@ -15,10 +15,8 @@
|
|||
|
||||
#define ALLOW_FORBID_FUNC
|
||||
#define _DEFAULT_SOURCE
|
||||
#include <malloc.h>
|
||||
#include "os.h"
|
||||
// #include "tdef.h"
|
||||
// #include <wchar.h>
|
||||
// #include <wctype.h>
|
||||
|
||||
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
|
||||
#include "iconv.h"
|
||||
|
|
@ -32,14 +30,6 @@ int64_t taosStr2int64(const char *str) {
|
|||
return strtoll(str, &endptr, 10);
|
||||
}
|
||||
|
||||
bool taosCheckNcharValid(void) {
|
||||
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
|
||||
for (int32_t i = 0; i < bytes; i += sizeof(TdUcs4)) {
|
||||
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
|
||||
|
|
@ -77,11 +67,13 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
|
|||
|
||||
|
||||
TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
|
||||
memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
|
||||
assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4));
|
||||
return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4));
|
||||
}
|
||||
|
||||
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
|
||||
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
|
||||
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
|
||||
return -1;
|
||||
#else
|
||||
iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
|
||||
|
|
@ -99,6 +91,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
|
|||
|
||||
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
|
||||
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
|
||||
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
|
||||
return -1;
|
||||
#else
|
||||
memset(ucs4, 0, ucs4_max_len);
|
||||
|
|
@ -124,7 +117,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
|
|||
|
||||
bool taosValidateEncodec(const char *encodec) {
|
||||
#ifdef DISALLOW_NCHAR_WITHOUT_ICONV
|
||||
return false;
|
||||
printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n");
|
||||
return true;
|
||||
#else
|
||||
iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
|
||||
if (cd == (iconv_t)(-1)) {
|
||||
|
|
@ -164,341 +158,3 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size) { ret
|
|||
int32_t taosWcharToMb(char *pStr, TdWchar wchar) { return wctomb(pStr, wchar); }
|
||||
|
||||
int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size) { return wcstombs(pStrs, pWchars, size); }
|
||||
|
||||
// #ifdef USE_LIBICONV
|
||||
// #include "iconv.h"
|
||||
|
||||
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
|
||||
// iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
|
||||
// size_t ucs4_input_len = ucs4_max_len;
|
||||
// size_t outLen = ucs4_max_len;
|
||||
// if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
|
||||
// iconv_close(cd);
|
||||
// return -1;
|
||||
// }
|
||||
|
||||
// iconv_close(cd);
|
||||
// return (int32_t)(ucs4_max_len - outLen);
|
||||
// }
|
||||
|
||||
// bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
|
||||
// memset(ucs4, 0, ucs4_max_len);
|
||||
// iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
|
||||
// size_t ucs4_input_len = mbsLength;
|
||||
// size_t outLeft = ucs4_max_len;
|
||||
// if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
|
||||
// iconv_close(cd);
|
||||
// return false;
|
||||
// }
|
||||
|
||||
// iconv_close(cd);
|
||||
// if (len != NULL) {
|
||||
// *len = (int32_t)(ucs4_max_len - outLeft);
|
||||
// if (*len < 0) {
|
||||
// return false;
|
||||
// }
|
||||
// }
|
||||
|
||||
// return true;
|
||||
// }
|
||||
|
||||
// bool taosValidateEncodec(const char *encodec) {
|
||||
// iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
|
||||
// if (cd == (iconv_t)(-1)) {
|
||||
// return false;
|
||||
// }
|
||||
|
||||
// iconv_close(cd);
|
||||
// return true;
|
||||
// }
|
||||
|
||||
// #else
|
||||
|
||||
// int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
|
||||
// mbstate_t state = {0};
|
||||
// int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
|
||||
// if (len < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
|
||||
// memset(&state, 0, sizeof(state));
|
||||
// len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
|
||||
// if (len < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
|
||||
// return len;
|
||||
// }
|
||||
|
||||
// bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
|
||||
// memset(ucs4, 0, ucs4_max_len);
|
||||
// mbstate_t state = {0};
|
||||
// int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
|
||||
// *len = retlen;
|
||||
|
||||
// return retlen >= 0;
|
||||
// }
|
||||
|
||||
// bool taosValidateEncodec(const char *encodec) {
|
||||
// return true;
|
||||
// }
|
||||
|
||||
// #endif
|
||||
|
||||
// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
|
||||
|
||||
// /*
|
||||
// * windows implementation
|
||||
// */
|
||||
|
||||
// #ifdef HAVE_CONFIG_H
|
||||
// #include <config.h>
|
||||
// #endif
|
||||
|
||||
// #include <sys/types.h>
|
||||
// #include <stdio.h>
|
||||
// #include <assert.h>
|
||||
// #include <errno.h>
|
||||
// #include <stdint.h>
|
||||
|
||||
// #if STDC_HEADERS
|
||||
// #include <stdlib.h>
|
||||
// #else
|
||||
// char *malloc(), *realloc();
|
||||
// #endif
|
||||
|
||||
// /* Always add at least this many bytes when extending the buffer. */
|
||||
// #define MIN_CHUNK 64
|
||||
|
||||
// /* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR
|
||||
// + OFFSET (and null-terminate it). *LINEPTR is a pointer returned from
|
||||
// malloc (or NULL), pointing to *N characters of space. It is realloc'd
|
||||
// as necessary. Return the number of characters read (not including the
|
||||
// null terminator), or -1 on error or EOF. On a -1 return, the caller
|
||||
// should check feof(), if not then errno has been set to indicate
|
||||
// the error. */
|
||||
|
||||
// int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t offset) {
|
||||
// int32_t nchars_avail; /* Allocated but unused chars in *LINEPTR. */
|
||||
// char * read_pos; /* Where we're reading into *LINEPTR. */
|
||||
// int32_t ret;
|
||||
|
||||
// if (!lineptr || !n || !stream) {
|
||||
// errno = EINVAL;
|
||||
// return -1;
|
||||
// }
|
||||
|
||||
// if (!*lineptr) {
|
||||
// *n = MIN_CHUNK;
|
||||
// *lineptr = malloc(*n);
|
||||
// if (!*lineptr) {
|
||||
// errno = ENOMEM;
|
||||
// return -1;
|
||||
// }
|
||||
// }
|
||||
|
||||
// nchars_avail = (int32_t)(*n - offset);
|
||||
// read_pos = *lineptr + offset;
|
||||
|
||||
// for (;;) {
|
||||
// int32_t save_errno;
|
||||
// register int32_t c = getc(stream);
|
||||
|
||||
// save_errno = errno;
|
||||
|
||||
// /* We always want at least one char left in the buffer, since we
|
||||
// always (unless we get an error while reading the first char)
|
||||
// NUL-terminate the line buffer. */
|
||||
|
||||
// assert((*lineptr + *n) == (read_pos + nchars_avail));
|
||||
// if (nchars_avail < 2) {
|
||||
// if (*n > MIN_CHUNK)
|
||||
// *n *= 2;
|
||||
// else
|
||||
// *n += MIN_CHUNK;
|
||||
|
||||
// nchars_avail = (int32_t)(*n + *lineptr - read_pos);
|
||||
// char* lineptr1 = realloc(*lineptr, *n);
|
||||
// if (!lineptr1) {
|
||||
// errno = ENOMEM;
|
||||
// return -1;
|
||||
// }
|
||||
// *lineptr = lineptr1;
|
||||
|
||||
// read_pos = *n - nchars_avail + *lineptr;
|
||||
// assert((*lineptr + *n) == (read_pos + nchars_avail));
|
||||
// }
|
||||
|
||||
// if (ferror(stream)) {
|
||||
// /* Might like to return partial line, but there is no
|
||||
// place for us to store errno. And we don't want to just
|
||||
// lose errno. */
|
||||
// errno = save_errno;
|
||||
// return -1;
|
||||
// }
|
||||
|
||||
// if (c == EOF) {
|
||||
// /* Return partial line, if any. */
|
||||
// if (read_pos == *lineptr)
|
||||
// return -1;
|
||||
// else
|
||||
// break;
|
||||
// }
|
||||
|
||||
// *read_pos++ = c;
|
||||
// nchars_avail--;
|
||||
|
||||
// if (c == terminator) /* Return the line. */
|
||||
// break;
|
||||
// }
|
||||
|
||||
// /* Done - NUL terminate and return the number of chars read. */
|
||||
// *read_pos = '\0';
|
||||
|
||||
// ret = (int32_t)(read_pos - (*lineptr + offset));
|
||||
// return ret;
|
||||
// }
|
||||
|
||||
// int32_t tgetline(char **lineptr, size_t *n, FILE *stream) { return getstr(lineptr, n, stream, '\n', 0); }
|
||||
|
||||
|
||||
// /*
|
||||
// * Get next token from string *stringp, where tokens are possibly-empty
|
||||
// * strings separated by characters from delim.
|
||||
// *
|
||||
// * Writes NULs into the string at *stringp to end tokens.
|
||||
// * delim need not remain constant from call to call.
|
||||
// * On return, *stringp points past the last NUL written (if there might
|
||||
// * be further tokens), or is NULL (if there are definitely no moretokens).
|
||||
// *
|
||||
// * If *stringp is NULL, strsep returns NULL.
|
||||
// */
|
||||
// char *strsep(char **stringp, const char *delim) {
|
||||
// char * s;
|
||||
// const char *spanp;
|
||||
// int32_t c, sc;
|
||||
// char *tok;
|
||||
// if ((s = *stringp) == NULL)
|
||||
// return (NULL);
|
||||
// for (tok = s;;) {
|
||||
// c = *s++;
|
||||
// spanp = delim;
|
||||
// do {
|
||||
// if ((sc = *spanp++) == c) {
|
||||
// if (c == 0)
|
||||
// s = NULL;
|
||||
// else
|
||||
// s[-1] = 0;
|
||||
// *stringp = s;
|
||||
// return (tok);
|
||||
// }
|
||||
// } while (sc != 0);
|
||||
// }
|
||||
// /* NOTREACHED */
|
||||
// }
|
||||
|
||||
// char *getpass(const char *prefix) {
|
||||
// static char passwd[TSDB_PASSWORD_LEN] = {0};
|
||||
// memset(passwd, 0, TSDB_PASSWORD_LEN);
|
||||
// //printf("%s", prefix);
|
||||
|
||||
// int32_t index = 0;
|
||||
// char ch;
|
||||
// while (index < TSDB_PASSWORD_LEN) {
|
||||
// ch = getch();
|
||||
// if (ch == '\n' || ch == '\r') {
|
||||
// break;
|
||||
// } else {
|
||||
// passwd[index++] = ch;
|
||||
// }
|
||||
// }
|
||||
|
||||
// return passwd;
|
||||
// }
|
||||
|
||||
// int32_t twcslen(const wchar_t *wcs) {
|
||||
// int32_t *wstr = (int32_t *)wcs;
|
||||
// if (NULL == wstr) {
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
// int32_t n = 0;
|
||||
// while (1) {
|
||||
// if (0 == *wstr++) {
|
||||
// break;
|
||||
// }
|
||||
// n++;
|
||||
// }
|
||||
|
||||
// return n;
|
||||
// }
|
||||
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
|
||||
// for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) {
|
||||
// int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
|
||||
// int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
|
||||
|
||||
// if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
|
||||
// return f1 - f2;
|
||||
// } else if (f1 == 0 && f2 == 0) {
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
// if (f1 != f2) {
|
||||
// return f1 - f2;
|
||||
// }
|
||||
// }
|
||||
|
||||
// return 0;
|
||||
|
||||
// #if 0
|
||||
// int32_t ucs4_max_len = bytes + 4;
|
||||
// char *f1_mbs = calloc(bytes, 1);
|
||||
// char *f2_mbs = calloc(bytes, 1);
|
||||
// if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
// if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
// int32_t ret = strcmp(f1_mbs, f2_mbs);
|
||||
// free(f1_mbs);
|
||||
// free(f2_mbs);
|
||||
// return ret;
|
||||
// #endif
|
||||
// }
|
||||
|
||||
// /* Copy memory to memory until the specified number of bytes
|
||||
// has been copied, return pointer to following byte.
|
||||
// Overlap is NOT handled correctly. */
|
||||
// void *mempcpy(void *dest, const void *src, size_t len) {
|
||||
// return (char*)memcpy(dest, src, len) + len;
|
||||
// }
|
||||
|
||||
// /* Copy SRC to DEST, returning the address of the terminating '\0' in DEST. */
|
||||
// char *stpcpy (char *dest, const char *src) {
|
||||
// size_t len = strlen (src);
|
||||
// return (char*)memcpy(dest, src, len + 1) + len;
|
||||
// }
|
||||
|
||||
// /* Copy no more than N characters of SRC to DEST, returning the address of
|
||||
// the terminating '\0' in DEST, if any, or else DEST + N. */
|
||||
// char *stpncpy (char *dest, const char *src, size_t n) {
|
||||
// size_t size = strnlen (src, n);
|
||||
// memcpy (dest, src, size);
|
||||
// dest += size;
|
||||
// if (size == n)
|
||||
// return dest;
|
||||
// return memset (dest, '\0', n - size);
|
||||
// }
|
||||
|
||||
// #else
|
||||
|
||||
// /*
|
||||
// * linux and darwin implementation
|
||||
// */
|
||||
|
||||
// int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize) {
|
||||
// return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / ncharSize);
|
||||
// }
|
||||
|
||||
// #endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue