Merge branch '3.0' of https://github.com/taosdata/TDengine into refact/tsdb_new_format

This commit is contained in:
Hongze Cheng 2022-09-02 13:53:00 +08:00
commit d14a928307
14 changed files with 105 additions and 110 deletions

View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 212c34d GIT_TAG f169c0f
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

View File

@ -33,6 +33,7 @@ adapterName="taosadapter"
benchmarkName="taosBenchmark" benchmarkName="taosBenchmark"
dumpName="taosdump" dumpName="taosdump"
demoName="taosdemo" demoName="taosdemo"
xname="taosx"
data_dir=${dataDir} data_dir=${dataDir}
log_dir=${logDir} log_dir=${logDir}
@ -199,6 +200,7 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/${demoName} || : ${csudo}rm -f ${bin_link_dir}/${demoName} || :
${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || :
${csudo}rm -f ${bin_link_dir}/${dumpName} || : ${csudo}rm -f ${bin_link_dir}/${dumpName} || :
${csudo}rm -f ${bin_link_dir}/${xname} || :
${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
@ -212,6 +214,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
[ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
[ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${xname} || :
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

View File

@ -172,6 +172,7 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/taosx || :
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
${csudo}rm -f ${bin_link_dir}/perfMonitor || : ${csudo}rm -f ${bin_link_dir}/perfMonitor || :
@ -184,6 +185,7 @@ function install_bin() {
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || :
${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || :
@ -199,6 +201,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || :
[ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || :
[ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
@ -215,6 +218,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || : [ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || :
[ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
[ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/taosx ] || [ -x ${install_main_2_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || ln -s ${install_main_2_dir}/bin/taosx ${bin_link_dir}/taosx || :
fi fi
} }

View File

@ -80,10 +80,12 @@ else
${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/taosBenchmark \
${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/TDinsight.sh \
$tdinsight_caches" $tdinsight_caches"
[ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx"
bin_files="${build_dir}/bin/${serverName} \ bin_files="${build_dir}/bin/${serverName} \
${build_dir}/bin/${clientName} \ ${build_dir}/bin/${clientName} \
${taostools_bin_files} \ ${taostools_bin_files} \
${taosx_bin} \
${build_dir}/bin/taosadapter \ ${build_dir}/bin/taosadapter \
${build_dir}/bin/udfd \ ${build_dir}/bin/udfd \
${script_dir}/remove.sh \ ${script_dir}/remove.sh \

View File

@ -241,6 +241,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
continue; continue;
} }
} }
if (pHandle->fetchMeta) {
SSubmitBlk* pBlk = pReader->pBlock;
if (pBlk->schemaLen > 0) {
if (pRsp->createTableNum == 0) {
pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
}
void* createReq = taosMemoryCalloc(1, pBlk->schemaLen);
memcpy(createReq, pBlk->data, pBlk->schemaLen);
taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen);
taosArrayPush(pRsp->createTableReq, &createReq);
pRsp->createTableNum++;
}
}
tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
blockDataFreeRes(&block); blockDataFreeRes(&block);
tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
@ -261,34 +275,25 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
continue; continue;
} }
} }
if (pHandle->fetchMeta) {
SSubmitBlk* pBlk = pReader->pBlock;
if (pBlk->schemaLen > 0) {
if (pRsp->createTableNum == 0) {
pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
}
void* createReq = taosMemoryCalloc(1, pBlk->schemaLen);
memcpy(createReq, pBlk->data, pBlk->schemaLen);
taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen);
taosArrayPush(pRsp->createTableReq, &createReq);
pRsp->createTableNum++;
}
}
tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
blockDataFreeRes(&block); blockDataFreeRes(&block);
tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
pRsp->blockNum++; pRsp->blockNum++;
} }
#if 1
if (pHandle->fetchMeta && pRsp->blockNum) {
SSubmitMsgIter iter = {0};
tInitSubmitMsgIter(pReq, &iter);
STaosxRsp* pXrsp = (STaosxRsp*)pRsp;
while (1) {
SSubmitBlk* pBlk = NULL;
if (tGetSubmitMsgNext(&iter, &pBlk) < 0) break;
if (pBlk == NULL) break;
if (pBlk->schemaLen > 0) {
if (pXrsp->createTableNum == 0) {
pXrsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
pXrsp->createTableReq = taosArrayInit(0, sizeof(void*));
}
void* createReq = taosMemoryCalloc(1, pBlk->schemaLen);
memcpy(createReq, pBlk->data, pBlk->schemaLen);
taosArrayPush(pXrsp->createTableLen, &pBlk->schemaLen);
taosArrayPush(pXrsp->createTableReq, &createReq);
pXrsp->createTableNum++;
}
}
}
#endif
} }
if (pRsp->blockNum == 0) { if (pRsp->blockNum == 0) {

View File

@ -811,7 +811,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} }
} }
// TODO after dropping table, table may be not found // TODO after dropping table, table may not found
ASSERT(found); ASSERT(found);
if (pTableScanInfo->dataReader == NULL) { if (pTableScanInfo->dataReader == NULL) {

View File

@ -40,26 +40,31 @@ extern "C" {
#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0) #define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0)
// clang-format on // clang-format on
extern void* indexQhandle;
typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType; typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType;
typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef enum { kTypeValue, kTypeDeletion } STermValueType;
typedef enum { kRebuild, kFinished } SIdxStatus; typedef enum { kRebuild, kFinished } SIdxStatus;
typedef struct SIndexStat { typedef struct SIndexStat {
int32_t totalAdded; // int32_t total;
int32_t totalDeled; // int32_t add; //
int32_t totalUpdated; // int32_t del; //
int32_t totalTerms; // int32_t update; //
int32_t distinctCol; // distinct column int32_t terms; //
int32_t distCol; // distinct column
} SIndexStat; } SIndexStat;
struct SIndex { struct SIndex {
SIndexOpts opts;
int64_t refId; int64_t refId;
void* cache; void* cache;
void* tindex; void* tindex;
SHashObj* colObj; // < field name, field id> SHashObj* colObj; // < field name, field id>
int64_t suid; // current super table id, -1 is normal table int64_t suid; // current super table id, -1 is normal table
int32_t cVersion; // current version allocated to cache int32_t version; // current version allocated to cache
SLRUCache* lru; SLRUCache* lru;
char* path; char* path;
@ -68,7 +73,6 @@ struct SIndex {
TdThreadMutex mtx; TdThreadMutex mtx;
tsem_t sem; tsem_t sem;
bool quit; bool quit;
SIndexOpts opts;
}; };
struct SIndexMultiTermQuery { struct SIndexMultiTermQuery {
@ -111,14 +115,15 @@ typedef struct Iterate {
void iterateValueDestroy(IterateValue* iv, bool destroy); void iterateValueDestroy(IterateValue* iv, bool destroy);
extern void* indexQhandle;
typedef struct TFileCacheKey { typedef struct TFileCacheKey {
uint64_t suid; uint64_t suid;
uint8_t colType; uint8_t colType;
char* colName; char* colName;
int32_t nColName; int32_t nColName;
} ICacheKey; } ICacheKey;
int32_t idxSerialCacheKey(ICacheKey* key, char* buf);
int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit); int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit);
int64_t idxAddRef(void* p); int64_t idxAddRef(void* p);
@ -126,10 +131,6 @@ int32_t idxRemoveRef(int64_t ref);
void idxAcquireRef(int64_t ref); void idxAcquireRef(int64_t ref);
void idxReleaseRef(int64_t ref); void idxReleaseRef(int64_t ref);
int32_t idxSerialCacheKey(ICacheKey* key, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf);
// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
#define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) #define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
#define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F) #define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F)

View File

@ -25,10 +25,6 @@
#include "tref.h" #include "tref.h"
#include "tsched.h" #include "tsched.h"
#ifdef USE_LUCENE
#include "lucene++/Lucene_c.h"
#endif
#define INDEX_NUM_OF_THREADS 5 #define INDEX_NUM_OF_THREADS 5
#define INDEX_QUEUE_SIZE 200 #define INDEX_QUEUE_SIZE 200
@ -74,7 +70,7 @@ void indexCleanup() {
typedef struct SIdxColInfo { typedef struct SIdxColInfo {
int colId; // generated by index internal int colId; // generated by index internal
int cVersion; int version;
} SIdxColInfo; } SIdxColInfo;
static TdThreadOnce isInit = PTHREAD_ONCE_INIT; static TdThreadOnce isInit = PTHREAD_ONCE_INIT;
@ -123,7 +119,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
} }
idx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); idx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
idx->cVersion = 1; idx->version = 1;
idx->path = tstrdup(path); idx->path = tstrdup(path);
taosThreadMutexInit(&idx->mtx, NULL); taosThreadMutexInit(&idx->mtx, NULL);
tsem_init(&idx->sem, 0, 0); tsem_init(&idx->sem, 0, 0);

View File

@ -566,7 +566,6 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
taosThreadMutexUnlock(&pCache->mtx); taosThreadMutexUnlock(&pCache->mtx);
idxCacheUnRef(pCache); idxCacheUnRef(pCache);
return 0; return 0;
// encode end
} }
void idxCacheForceToMerge(void* cache) { void idxCacheForceToMerge(void* cache) {
IndexCache* pCache = cache; IndexCache* pCache = cache;
@ -602,10 +601,10 @@ static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr,
} }
} }
int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) {
int64_t st = taosGetTimestampUs();
if (cache == NULL) { if (cache == NULL) {
return 0; return 0;
} }
IndexCache* pCache = cache; IndexCache* pCache = cache;
MemTable *mem = NULL, *imm = NULL; MemTable *mem = NULL, *imm = NULL;
@ -616,6 +615,8 @@ int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STerm
idxMemRef(imm); idxMemRef(imm);
taosThreadMutexUnlock(&pCache->mtx); taosThreadMutexUnlock(&pCache->mtx);
int64_t st = taosGetTimestampUs();
int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0; int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0;
if (ret == 0 && *s != kTypeDeletion) { if (ret == 0 && *s != kTypeDeletion) {
// continue search in imm // continue search in imm

View File

@ -178,9 +178,9 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
// optime later // optime later
int32_t ret = func(a, b); int32_t ret = func(a, b);
switch (comparType) { switch (comparType) {
case QUERY_LESS_THAN: { case QUERY_LESS_THAN:
if (ret < 0) return MATCH; if (ret < 0) return MATCH;
} break; break;
case QUERY_LESS_EQUAL: { case QUERY_LESS_EQUAL: {
if (ret <= 0) return MATCH; if (ret <= 0) return MATCH;
break; break;

View File

@ -27,6 +27,44 @@
#define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
// clang-format on // clang-format on
typedef union {
uint8_t u8;
uint16_t u16;
uint32_t u32;
uint64_t u64;
int8_t i8;
int16_t i16;
int32_t i32;
int64_t i64;
double d;
float f;
} SDataTypeBuf;
#define SIF_DATA_CONVERT(type, val, dst) \
do { \
if (type == TSDB_DATA_TYPE_DOUBLE) \
dst = GET_DOUBLE_VAL(val); \
else if (type == TSDB_DATA_TYPE_BIGINT) \
dst = *(int64_t *)val; \
else if (type == TSDB_DATA_TYPE_INT) \
dst = *(int32_t *)val; \
else if (type == TSDB_DATA_TYPE_SMALLINT) \
dst = *(int16_t *)val; \
else if (type == TSDB_DATA_TYPE_TINYINT) \
dst = *(int8_t *)val; \
else if (type == TSDB_DATA_TYPE_UTINYINT) \
dst = *(uint8_t *)val; \
else if (type == TSDB_DATA_TYPE_USMALLINT) \
dst = *(uint16_t *)val; \
else if (type == TSDB_DATA_TYPE_UINT) \
dst = *(uint32_t *)val; \
else if (type == TSDB_DATA_TYPE_UBIGINT) \
dst = *(uint64_t *)val; \
} while (0);
typedef struct SIFParam { typedef struct SIFParam {
SHashObj *pFilter; SHashObj *pFilter;
@ -48,7 +86,6 @@ typedef struct SIFCtx {
SHashObj *pRes; /* element is SIFParam */ SHashObj *pRes; /* element is SIFParam */
bool noExec; // true: just iterate condition tree, and add hint to executor plan bool noExec; // true: just iterate condition tree, and add hint to executor plan
SIndexMetaArg arg; SIndexMetaArg arg;
// SIdxFltStatus st;
} SIFCtx; } SIFCtx;
static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
@ -75,11 +112,6 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output);
static sif_func_t sifNullFunc = NULL; static sif_func_t sifNullFunc = NULL;
// typedef struct SIFWalkParm
// construct tag filter operator later
// static void destroyTagFilterOperatorInfo(void *param) {
// STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param;
//}
static void sifFreeParam(SIFParam *param) { static void sifFreeParam(SIFParam *param) {
if (param == NULL) return; if (param == NULL) return;
@ -365,42 +397,6 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
} }
return NULL; return NULL;
} }
typedef union {
uint8_t u8;
uint16_t u16;
uint32_t u32;
uint64_t u64;
int8_t i8;
int16_t i16;
int32_t i32;
int64_t i64;
double d;
float f;
} SDataTypeBuf;
#define SIF_DATA_CONVERT(type, val, dst) \
do { \
if (type == TSDB_DATA_TYPE_DOUBLE) \
dst = GET_DOUBLE_VAL(val); \
else if (type == TSDB_DATA_TYPE_BIGINT) \
dst = *(int64_t *)val; \
else if (type == TSDB_DATA_TYPE_INT) \
dst = *(int32_t *)val; \
else if (type == TSDB_DATA_TYPE_SMALLINT) \
dst = *(int16_t *)val; \
else if (type == TSDB_DATA_TYPE_TINYINT) \
dst = *(int8_t *)val; \
else if (type == TSDB_DATA_TYPE_UTINYINT) \
dst = *(uint8_t *)val; \
else if (type == TSDB_DATA_TYPE_USMALLINT) \
dst = *(uint16_t *)val; \
else if (type == TSDB_DATA_TYPE_UINT) \
dst = *(uint32_t *)val; \
else if (type == TSDB_DATA_TYPE_UBIGINT) \
dst = *(uint64_t *)val; \
} while (0);
static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) {
int8_t ltype = left->colValType, rtype = right->colValType; int8_t ltype = left->colValType, rtype = right->colValType;
@ -693,11 +689,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou
for (int32_t m = 0; m < node->pParameterList->length; m++) { for (int32_t m = 0; m < node->pParameterList->length; m++) {
if (node->condType == LOGIC_COND_TYPE_AND) { if (node->condType == LOGIC_COND_TYPE_AND) {
taosArrayAddAll(output->result, params[m].result); taosArrayAddAll(output->result, params[m].result);
// taosArrayDestroy(params[m].result);
// params[m].result = NULL;
} else if (node->condType == LOGIC_COND_TYPE_OR) { } else if (node->condType == LOGIC_COND_TYPE_OR) {
taosArrayAddAll(output->result, params[m].result); taosArrayAddAll(output->result, params[m].result);
// params[m].result = NULL;
} else if (node->condType == LOGIC_COND_TYPE_NOT) { } else if (node->condType == LOGIC_COND_TYPE_NOT) {
// taosArrayAddAll(output->result, params[m].result); // taosArrayAddAll(output->result, params[m].result);
} }

View File

@ -211,9 +211,7 @@ IdxFstFile* idxFileCreate(void* wrt) {
return cw; return cw;
} }
void idxFileDestroy(IdxFstFile* cw) { void idxFileDestroy(IdxFstFile* cw) {
// free wrt object: close fd or free mem
idxFileFlush(cw); idxFileFlush(cw);
// idxFileCtxDestroy((IFileCtx *)(cw->wrt));
taosMemoryFree(cw); taosMemoryFree(cw);
} }
@ -222,10 +220,8 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) {
return 0; return 0;
} }
// update checksum // update checksum
// write data to file/socket or mem
IFileCtx* ctx = write->wrt; IFileCtx* ctx = write->wrt;
int nWrite = ctx->write(ctx, buf, len);
int nWrite = ctx->write(ctx, buf, len);
assert(nWrite == len); assert(nWrite == len);
write->count += len; write->count += len;

View File

@ -183,13 +183,14 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) {
return NULL; return NULL;
} }
reader->ctx = ctx; reader->ctx = ctx;
reader->remove = false;
if (0 != tfileReaderVerify(reader)) { if (0 != tfileReaderVerify(reader)) {
indexError("invalid tfile, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); indexError("invalid tfile, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName);
tfileReaderDestroy(reader); tfileReaderDestroy(reader);
return NULL; return NULL;
} }
// T_REF_INC(reader);
if (0 != tfileReaderLoadHeader(reader)) { if (0 != tfileReaderLoadHeader(reader)) {
indexError("failed to load index header, suid:%" PRIu64 ", colName:%s", reader->header.suid, indexError("failed to load index header, suid:%" PRIu64 ", colName:%s", reader->header.suid,
reader->header.colName); reader->header.colName);
@ -203,7 +204,6 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) {
tfileReaderDestroy(reader); tfileReaderDestroy(reader);
return NULL; return NULL;
} }
reader->remove = false;
return reader; return reader;
} }
@ -211,7 +211,6 @@ void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { if (reader == NULL) {
return; return;
} }
// T_REF_INC(reader);
fstDestroy(reader->fst); fstDestroy(reader->fst);
if (reader->remove) { if (reader->remove) {
indexInfo("%s is removed", reader->ctx->file.buf); indexInfo("%s is removed", reader->ctx->file.buf);
@ -222,6 +221,7 @@ void tfileReaderDestroy(TFileReader* reader) {
taosMemoryFree(reader); taosMemoryFree(reader);
} }
static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
int ret = 0; int ret = 0;
char* p = tem->colVal; char* p = tem->colVal;
@ -494,7 +494,6 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr
TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* colName, uint8_t colType) { TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* colName, uint8_t colType) {
char fullname[256] = {0}; char fullname[256] = {0};
tfileGenFileFullName(fullname, path, suid, colName, version); tfileGenFileFullName(fullname, path, suid, colName, version);
// indexInfo("open write file name %s", fullname);
IFileCtx* wcx = idxFileCtxCreate(TFILE, fullname, false, 1024 * 1024 * 64); IFileCtx* wcx = idxFileCtxCreate(TFILE, fullname, false, 1024 * 1024 * 64);
if (wcx == NULL) { if (wcx == NULL) {
return NULL; return NULL;
@ -503,8 +502,8 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c
TFileHeader tfh = {0}; TFileHeader tfh = {0};
tfh.suid = suid; tfh.suid = suid;
tfh.version = version; tfh.version = version;
memcpy(tfh.colName, colName, strlen(colName));
tfh.colType = colType; tfh.colType = colType;
memcpy(tfh.colName, colName, strlen(colName));
return tfileWriterCreate(wcx, &tfh); return tfileWriterCreate(wcx, &tfh);
} }
@ -706,7 +705,6 @@ static bool tfileIteratorNext(Iterate* iiter) {
iv->type = ADD_VALUE; // value in tfile always ADD_VALUE iv->type = ADD_VALUE; // value in tfile always ADD_VALUE
iv->colVal = colVal; iv->colVal = colVal;
return true; return true;
// std::string key(ch, sz);
} }
static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }

View File

@ -1304,11 +1304,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
pTransInst->cfp(pTransInst->parent, pResp, NULL); pTransInst->cfp(pTransInst->parent, pResp, NULL);
return 0; return 0;
} }
/*
* no retry
* 1. query conn
* 2. rpc thread already receive quit msg
*/
STransConnCtx* pCtx = pMsg->ctx; STransConnCtx* pCtx = pMsg->ctx;
int32_t code = pResp->code; int32_t code = pResp->code;