xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update index TFile manage and format code

This commit is contained in:
yihaoDeng 2021-12-21 16:36:31 +08:00
parent 6fe118c516
commit 5d4d7b47a0
22 changed files with 2205 additions and 2023 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/libs/index/index.h
View File

@ -76,11 +76,10 @@ void indexOptsDestroy(SIndexOpts *opts);
* @param: * @param:
*/ */
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName,
const char *colName, int32_t nColName, const char *colVal, int32_t nColVal); int32_t nColName, const char *colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm *p); void indexTermDestroy(SIndexTerm *p);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

51
source/libs/index/inc/indexInt.h
View File

@ -18,15 +18,14 @@
#include "index.h" #include "index.h"
#include "index_fst.h" #include "index_fst.h"
#include "tlog.h"
#include "thash.h"
#include "taos.h" #include "taos.h"
#include "thash.h"
#include "tlog.h"
#ifdef USE_LUCENE #ifdef USE_LUCENE
#include <lucene++/Lucene_c.h> #include <lucene++/Lucene_c.h>
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -66,7 +65,6 @@ struct SIndexOpts {
int32_t cacheSize; // MB int32_t cacheSize; // MB
// add cache module later // add cache module later
#endif #endif
}; };
struct SIndexMultiTermQuery { struct SIndexMultiTermQuery {
@ -90,15 +88,42 @@ typedef struct SIndexTermQuery {
EIndexQueryType qType; EIndexQueryType qType;
} SIndexTermQuery; } SIndexTermQuery;
#define indexFatal(...) \
do { \
#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) if (sDebugFlag & DEBUG_FATAL) { \
#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) taosPrintLog("index FATAL ", 255, __VA_ARGS__); \
#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); }} while(0) } \
#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); }} while(0) } while (0)
#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) #define indexError(...) \
#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) do { \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("index ERROR ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("index WARN ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("index ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }

12
source/libs/index/inc/index_cache.h
View File

@ -22,8 +22,10 @@
// ----------------- key structure in skiplist --------------------- // ----------------- key structure in skiplist ---------------------
/* A data row, the format is like below: /* A data row, the format is like below:
* content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->| * content: |<--totalLen-->|<-- fieldid-->|<--field type-->|<-- value len--->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| * |<-- value -->|<--uid -->|<--version--->|<-- itermType -->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|
* <--valuelen->|<--uint64_t->| * <-- int32_t-->|<-- int8_t --->|
*/ */
#ifdef __cplusplus #ifdef __cplusplus
@ -35,7 +37,6 @@ typedef struct IndexCache {
SSkipList *skiplist; SSkipList *skiplist;
} IndexCache; } IndexCache;
// //
IndexCache *indexCacheCreate(); IndexCache *indexCacheCreate();
@ -44,12 +45,11 @@ void indexCacheDestroy(void *cache);
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid);
// int indexCacheGet(void *cache, uint64_t *rst); // int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); int indexCacheSearch(
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif #endif

60
source/libs/index/inc/index_fst.h
View File

@ -20,11 +20,11 @@
extern "C" { extern "C" {
#endif #endif
#include "tarray.h"
#include "index_fst_util.h"
#include "index_fst_registry.h"
#include "index_fst_counting_writer.h"
#include "index_fst_automation.h" #include "index_fst_automation.h"
#include "index_fst_counting_writer.h"
#include "index_fst_registry.h"
#include "index_fst_util.h"
#include "tarray.h"
#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a)
@ -51,18 +51,15 @@ typedef struct FstRange {
uint64_t end; uint64_t end;
} FstRange; } FstRange;
typedef enum { GE, GT, LE, LT } RangeType; typedef enum { GE, GT, LE, LT } RangeType;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data); FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data);
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice);
bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); bool fstBoundWithDataIsEmpty(FstBoundWithData *bound);
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); bool fstBoundWithDataIsIncluded(FstBoundWithData *bound);
typedef struct FstOutput { typedef struct FstOutput {
bool null; bool null;
Output out; Output out;
@ -82,15 +79,15 @@ typedef struct FstUnFinishedNodes {
FstUnFinishedNodes *fstUnFinishedNodesCreate(); FstUnFinishedNodes *fstUnFinishedNodesCreate();
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node);
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal);
FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes);
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out);
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr);
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out);
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs);
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes);
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out);
typedef struct FstBuilder { typedef struct FstBuilder {
FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`. FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`.
@ -101,21 +98,16 @@ typedef struct FstBuilder {
uint64_t len; // num of keys added uint64_t len; // num of keys added
} FstBuilder; } FstBuilder;
FstBuilder *fstBuilderCreate(void *w, FstType ty); FstBuilder *fstBuilderCreate(void *w, FstType ty);
void fstBuilderDestroy(FstBuilder *b); void fstBuilderDestroy(FstBuilder *b);
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in);
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup);
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate);
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn);
void * fstBuilerIntoInner(FstBuilder *b); void * fstBuilerIntoInner(FstBuilder *b);
void fstBuilderFinish(FstBuilder *b); void fstBuilderFinish(FstBuilder *b);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup);
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn);
typedef struct FstTransitions { typedef struct FstTransitions {
FstNode *node; FstNode *node;
@ -147,11 +139,11 @@ uint8_t fstStateCommInput(FstState* state, bool *null);
uint64_t fstStateInputLen(FstState *state); uint64_t fstStateInputLen(FstState *state);
// end_addr // end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data); uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data);
uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes);
uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); uint64_t fstStateEndAddrForAnyTrans(
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
// input // input
uint8_t fstStateInput(FstState *state, FstNode *node); uint8_t fstStateInput(FstState *state, FstNode *node);
uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i);
@ -180,14 +172,11 @@ uint64_t fstStateNtrans(FstState *state, FstSlice *slice);
Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null);
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) #define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans)
#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) #define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal)
typedef struct FstLastTransition { typedef struct FstLastTransition {
uint8_t inp; uint8_t inp;
Output out; Output out;
@ -202,9 +191,8 @@ typedef struct FstBuilderNodeUnfinished {
FstLastTransition *last; FstLastTransition *last;
} FstBuilderNodeUnfinished; } FstBuilderNodeUnfinished;
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr);
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out);
/* /*
@ -222,18 +210,20 @@ typedef struct FstNode {
Output finalOutput; Output finalOutput;
} FstNode; } FstNode;
// If this node is final and has a terminal output value, then it is, returned. Otherwise, a zero output is returned // If this node is final and has a terminal output value, then it is, returned.
// Otherwise, a zero output is returned
#define FST_NODE_FINAL_OUTPUT(node) node->finalOutput #define FST_NODE_FINAL_OUTPUT(node) node->finalOutput
// Returns true if and only if this node corresponds to a final or "match", state in the finite state transducer. // Returns true if and only if this node corresponds to a final or "match",
// state in the finite state transducer.
#define FST_NODE_IS_FINAL(node) node->isFinal #define FST_NODE_IS_FINAL(node) node->isFinal
// Returns the number of transitions in this node, The maximum number of transitions is 256. // Returns the number of transitions in this node, The maximum number of
// transitions is 256.
#define FST_NODE_LEN(node) node->nTrans #define FST_NODE_LEN(node) node->nTrans
// Returns true if and only if this node has zero transitions. // Returns true if and only if this node has zero transitions.
#define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0) #define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0)
// Return the address of this node. // Return the address of this node.
#define FST_NODE_ADDR(node) node->start #define FST_NODE_ADDR(node) node->start
FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data);
void fstNodeDestroy(FstNode *fstNode); void fstNodeDestroy(FstNode *fstNode);
@ -242,7 +232,9 @@ FstTransitions* fstNodeTransitions(FstNode *node);
bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res);
bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res);
bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res);
bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode);
FstSlice fstNodeAsSlice(FstNode *node); FstSlice fstNodeAsSlice(FstNode *node);
// ops // ops
@ -255,7 +247,6 @@ typedef struct FstIndexedValue {
FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out);
void fstLastTransitionDestroy(FstLastTransition *trn); void fstLastTransitionDestroy(FstLastTransition *trn);
typedef struct FstMeta { typedef struct FstMeta {
uint64_t version; uint64_t version;
CompiledAddr rootAddr; CompiledAddr rootAddr;
@ -282,14 +273,13 @@ FstType fstGetType(Fst *fst);
CompiledAddr fstGetRootAddr(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst);
Output fstEmptyFinalOutput(Fst *fst, bool *null); Output fstEmptyFinalOutput(Fst *fst, bool *null);
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx);
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
// into stream to expand later // into stream to expand later
StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb); StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb);
bool fstVerify(Fst *fst); bool fstVerify(Fst *fst);
// refactor this function // refactor this function
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
@ -321,8 +311,11 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
void swsResultDestroy(StreamWithStateResult *result); void swsResultDestroy(StreamWithStateResult *result);
typedef void *(*StreamCallback)(void *); typedef void *(*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ; StreamWithState *streamWithStateCreate(
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max);
void streamWithStateDestroy(StreamWithState *sws); void streamWithStateDestroy(StreamWithState *sws);
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);
StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
@ -333,7 +326,6 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut);
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

8
source/libs/index/inc/index_fst_automation.h
View File

@ -21,13 +21,9 @@ extern "C" {
#include "index_fst_util.h" #include "index_fst_util.h"
typedef struct AutomationCtx AutomationCtx; typedef struct AutomationCtx AutomationCtx;
typedef enum AutomationType { typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
AUTOMATION_PREFIX,
AUTMMATION_MATCH
} AutomationType;
typedef struct StartWith { typedef struct StartWith {
AutomationCtx *autoSelf; AutomationCtx *autoSelf;
@ -44,7 +40,6 @@ typedef struct AutomationCtx {
char * data; char * data;
} AutomationCtx; } AutomationCtx;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
@ -63,7 +58,6 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv);
void startWithStateValueDestroy(void *sv); void startWithStateValueDestroy(void *sv);
typedef struct AutomationFunc { typedef struct AutomationFunc {
void *(*start)(AutomationCtx *ctx); void *(*start)(AutomationCtx *ctx);
bool (*isMatch)(AutomationCtx *ctx, void *); bool (*isMatch)(AutomationCtx *ctx, void *);

4
source/libs/index/inc/index_fst_common.h
View File

@ -1,9 +1,9 @@
#ifndef __INDEX_FST_COMM_H__ #ifndef __INDEX_FST_COMM_H__
#define __INDEX_FST_COMM_H__ #define __INDEX_FST_COMM_H__
#include "tutil.h"
extern const uint8_t COMMON_INPUTS[]; extern const uint8_t COMMON_INPUTS[];
extern char const COMMON_INPUTS_INV[]; extern const char COMMON_INPUTS_INV[];
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {

7
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -22,7 +22,6 @@ extern "C" {
#include "tfile.h" #include "tfile.h"
#define DefaultMem 1024 * 1024 #define DefaultMem 1024 * 1024
static char tmpFile[] = "./index"; static char tmpFile[] = "./index";
@ -56,7 +55,6 @@ void writerCtxDestroy(WriterCtx *w);
typedef uint32_t CheckSummer; typedef uint32_t CheckSummer;
typedef struct FstCountingWriter { typedef struct FstCountingWriter {
void * wrt; // wrap any writer that counts and checksum bytes written void * wrt; // wrap any writer that counts and checksum bytes written
uint64_t count; uint64_t count;
@ -69,17 +67,14 @@ int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len);
int fstCountingWriterFlush(FstCountingWriter *write); int fstCountingWriterFlush(FstCountingWriter *write);
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write);
FstCountingWriter *fstCountingWriterCreate(void *wtr); FstCountingWriter *fstCountingWriterCreate(void *wtr);
void fstCountingWriterDestroy(FstCountingWriter *w); void fstCountingWriterDestroy(FstCountingWriter *w);
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes);
uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer) #define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
@ -89,5 +84,3 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#endif #endif
#endif #endif

5
source/libs/index/inc/index_fst_node.h
View File

@ -20,8 +20,8 @@
extern "C" { extern "C" {
#endif #endif
#include "index_fst_util.h"
#include "index_fst_counting_writer.h" #include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) #define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) #define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
@ -45,7 +45,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src);
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src);
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); // bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
// CompiledAddr lastAddr, CompiledAddr startAddr);
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2);
void fstBuilderNodeDestroy(FstBuilderNode *node); void fstBuilderNodeDestroy(FstBuilderNode *node);

11
source/libs/index/inc/index_fst_registry.h
View File

@ -19,9 +19,9 @@
extern "C" { extern "C" {
#endif #endif
#include "index_fst_node.h"
#include "index_fst_util.h" #include "index_fst_util.h"
#include "tarray.h" #include "tarray.h"
#include "index_fst_node.h"
typedef struct FstRegistryCell { typedef struct FstRegistryCell {
CompiledAddr addr; CompiledAddr addr;
@ -29,8 +29,10 @@ typedef struct FstRegistryCell {
} FstRegistryCell; } FstRegistryCell;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0) #define FST_REGISTRY_CELL_INSERT(cell, tAddr) \
do { \
cell->addr = tAddr; \
} while (0)
// typedef struct FstRegistryCache { // typedef struct FstRegistryCache {
// SArray *cells; // SArray *cells;
@ -46,8 +48,6 @@ typedef struct FstRegistryEntry {
FstRegistryCell * cell; FstRegistryCell * cell;
} FstRegistryEntry; } FstRegistryEntry;
// Registry relation function // Registry relation function
typedef struct FstRegistry { typedef struct FstRegistry {
SArray * table; //<FstRegistryCell> SArray * table; //<FstRegistryCell>
@ -59,7 +59,6 @@ typedef struct FstRegistry {
FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize);
void fstRegistryDestroy(FstRegistry *registry); void fstRegistryDestroy(FstRegistry *registry);
FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode);
void fstRegistryEntryDestroy(FstRegistryEntry *entry); void fstRegistryEntryDestroy(FstRegistryEntry *entry);

22
source/libs/index/inc/index_fst_util.h
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef __INDEX_FST_UTIL_H__ #ifndef __INDEX_FST_UTIL_H__
#define __INDEX_FST_UTIL_H__ #define __INDEX_FST_UTIL_H__
@ -21,15 +20,14 @@
extern "C" { extern "C" {
#endif #endif
#include "tarray.h"
#include "index_fst_common.h" #include "index_fst_common.h"
#include "tarray.h"
typedef uint64_t FstType; typedef uint64_t FstType;
typedef uint64_t CompiledAddr; typedef uint64_t CompiledAddr;
typedef uint64_t Output; typedef uint64_t Output;
typedef uint8_t PackSizes; typedef uint8_t PackSizes;
// A sentinel value used to indicate an empty final state // A sentinel value used to indicate an empty final state
extern const CompiledAddr EMPTY_ADDRESS; extern const CompiledAddr EMPTY_ADDRESS;
/// A sentinel value used to indicate an invalid state. /// A sentinel value used to indicate an invalid state.
@ -48,20 +46,25 @@ extern const uint64_t TRANS_INDEX_THRESHOLD;
// //
// `0` is a legal value which means there are no transitions/outputs // `0` is a legal value which means there are no transitions/outputs
#define FST_SET_TRANSITION_PACK_SIZE(v, sz) \
#define FST_SET_TRANSITION_PACK_SIZE(v, sz) do {v = (v & 0b00001111) | (sz << 4); } while(0) do { \
v = (v & 0b00001111) | (sz << 4); \
} while (0)
#define FST_GET_TRANSITION_PACK_SIZE(v) (((v)&0b11110000) >> 4) #define FST_GET_TRANSITION_PACK_SIZE(v) (((v)&0b11110000) >> 4)
#define FST_SET_OUTPUT_PACK_SIZE(v, sz) do { v = (v & 0b11110000) | sz; } while(0) #define FST_SET_OUTPUT_PACK_SIZE(v, sz) \
do { \
v = (v & 0b11110000) | sz; \
} while (0)
#define FST_GET_OUTPUT_PACK_SIZE(v) ((v)&0b00001111) #define FST_GET_OUTPUT_PACK_SIZE(v) ((v)&0b00001111)
#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx)-1] #define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx)-1]
#define COMMON_INDEX(v, max, val) do { \ #define COMMON_INDEX(v, max, val) \
do { \
val = ((uint16_t)COMMON_INPUTS[v] + 1) % 256; \ val = ((uint16_t)COMMON_INPUTS[v] + 1) % 256; \
val = val > max ? 0 : val; \ val = val > max ? 0 : val; \
} while (0) } while (0)
// uint8_t commonInput(uint8_t idx); // uint8_t commonInput(uint8_t idx);
// uint8_t commonIdx(uint8_t v, uint8_t max); // uint8_t commonIdx(uint8_t v, uint8_t max);
@ -70,7 +73,6 @@ uint64_t unpackUint64(uint8_t *ch, uint8_t sz);
uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr);
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr);
typedef struct FstString { typedef struct FstString {
uint8_t *data; uint8_t *data;
uint32_t len; uint32_t len;
@ -113,8 +115,6 @@ uint8_t *fstSliceData(FstSlice *s, int32_t *sz);
// void fstStackDestory(FstStack *); // void fstStackDestory(FstStack *);
// //
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

15
source/libs/index/inc/index_tfile.h
View File

@ -17,10 +17,10 @@
#include "index.h" #include "index.h"
#include "indexInt.h" #include "indexInt.h"
#include "tlockfree.h"
#include "index_tfile.h"
#include "index_fst_counting_writer.h"
#include "index_fst.h" #include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_tfile.h"
#include "tlockfree.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -48,7 +48,6 @@ typedef struct TFileCacheKey {
int32_t nColName; int32_t nColName;
} TFileCacheKey; } TFileCacheKey;
// table cache // table cache
// refactor to LRU cache later // refactor to LRU cache later
typedef struct TFileCache { typedef struct TFileCache {
@ -57,7 +56,6 @@ typedef struct TFileCache {
// add more param // add more param
} TFileCache; } TFileCache;
typedef struct TFileWriter { typedef struct TFileWriter {
FstBuilder *fb; FstBuilder *fb;
WriterCtx * ctx; WriterCtx * ctx;
@ -67,6 +65,7 @@ typedef struct TFileReader {
T_REF_DECLARE() T_REF_DECLARE()
Fst * fst; Fst * fst;
WriterCtx * ctx; WriterCtx * ctx;
TFileReadHeader header;
} TFileReader; } TFileReader;
typedef struct IndexTFile { typedef struct IndexTFile {
@ -87,7 +86,6 @@ typedef struct TFileReaderOpt {
uint64_t suid; uint64_t suid;
char * colName; char * colName;
int32_t nColName; int32_t nColName;
} TFileReaderOpt; } TFileReaderOpt;
// tfile cache, manage tindex reader // tfile cache, manage tindex reader
@ -99,7 +97,6 @@ void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader);
TFileReader *tfileReaderCreate(); TFileReader *tfileReaderCreate();
void TFileReaderDestroy(TFileReader *reader); void TFileReaderDestroy(TFileReader *reader);
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); TFileWriter *tfileWriterCreate(const char *suid, const char *colName);
void tfileWriterDestroy(TFileWriter *tw); void tfileWriterDestroy(TFileWriter *tw);
@ -108,13 +105,9 @@ IndexTFile *indexTFileCreate(const char *path);
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid);
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif #endif

1
source/libs/index/inc/index_util.h
View File

@ -45,7 +45,6 @@ extern "C" {
buf += len; \ buf += len; \
} while (0) } while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

40
source/libs/index/src/index.c
View File

@ -26,8 +26,11 @@
static int uidCompare(const void *a, const void *b) { static int uidCompare(const void *a, const void *b) {
uint64_t u1 = *(uint64_t *)a; uint64_t u1 = *(uint64_t *)a;
uint64_t u2 = *(uint64_t *)b; uint64_t u2 = *(uint64_t *)b;
if (u1 == u2) { return 0; } if (u1 == u2) {
else { return u1 < u2 ? -1 : 1; } return 0;
} else {
return u1 < u2 ? -1 : 1;
}
} }
typedef struct SIdxColInfo { typedef struct SIdxColInfo {
int colId; // generated by index internal int colId; // generated by index internal
@ -37,7 +40,6 @@ typedef struct SIdxColInfo {
static pthread_once_t isInit = PTHREAD_ONCE_INIT; static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit(); static void indexInit();
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result);
static int indexMergeCacheIntoTindex(SIndex *sIdx); static int indexMergeCacheIntoTindex(SIndex *sIdx);
@ -47,7 +49,9 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType
int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
pthread_once(&isInit, indexInit); pthread_once(&isInit, indexInit);
SIndex *sIdx = calloc(1, sizeof(SIndex)); SIndex *sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; } if (sIdx == NULL) {
return -1;
}
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t *index = index_open(path); index_t *index = index_open(path);
@ -81,7 +85,6 @@ void indexClose(SIndex *sIdx) {
} }
int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_document_t *doc = index_document_create(); index_document_t *doc = index_document_create();
@ -186,8 +189,6 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
return 1; return 1;
} }
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
#endif #endif
@ -200,7 +201,6 @@ int indexRebuild(SIndex *index, SIndexOpts *opts) {
} }
SIndexOpts *indexOptsCreate() { SIndexOpts *indexOptsCreate() {
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
@ -209,15 +209,16 @@ return NULL;
void indexOptsDestroy(SIndexOpts *opts){ void indexOptsDestroy(SIndexOpts *opts){
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
} } /*
/*
* @param: oper * @param: oper
* *
*/ */
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; } if (p == NULL) {
return NULL;
}
p->opera = opera; p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p; return p;
@ -236,10 +237,12 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EInde
return 0; return 0;
} }
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName,
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) { int32_t nColName, const char *colVal, int32_t nColVal) {
SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { return NULL; } if (t == NULL) {
return NULL;
}
t->suid = suid; t->suid = suid;
t->operType = oper; t->operType = oper;
@ -260,9 +263,7 @@ void indexTermDestroy(SIndexTerm *p) {
free(p); free(p);
} }
SIndexMultiTerm *indexMultiTermCreate() { SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); }
return taosArrayInit(4, sizeof(SIndexTerm *));
}
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) {
taosArrayPush(terms, &term); taosArrayPush(terms, &term);
@ -319,7 +320,9 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
return 0; return 0;
} }
static void indexInterResultsDestroy(SArray *results) { static void indexInterResultsDestroy(SArray *results) {
if (results == NULL) { return; } if (results == NULL) {
return;
}
size_t sz = taosArrayGetSize(results); size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
@ -327,7 +330,6 @@ static void indexInterResultsDestroy(SArray *results) {
taosArrayDestroy(p); taosArrayDestroy(p);
} }
taosArrayDestroy(results); taosArrayDestroy(results);
} }
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) {
// refactor, merge interResults into fResults by oType // refactor, merge interResults into fResults by oType

50
source/libs/index/src/index_cache.c
View File

@ -14,17 +14,17 @@
*/ */
#include "index_cache.h" #include "index_cache.h"
#include "tcompare.h"
#include "index_util.h" #include "index_util.h"
#include "tcompare.h"
#define MAX_INDEX_KEY_LEN 256 // test only, change later #define MAX_INDEX_KEY_LEN 256 // test only, change later
// ref index_cache.h:22 // ref index_cache.h:22
#define CACHE_KEY_LEN(p) (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) #define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \
sizeof(p->operType))
static char* getIndexKey(const void *pData) { static char * getIndexKey(const void *pData) { return NULL; }
return NULL;
}
static int32_t compareKey(const void *l, const void *r) { static int32_t compareKey(const void *l, const void *r) {
char *lp = (char *)l; char *lp = (char *)l;
char *rp = (char *)r; char *rp = (char *)r;
@ -64,11 +64,17 @@ static int32_t compareKey(const void *l, const void *r) {
// compare value // compare value
int32_t i, j; int32_t i, j;
for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) {
if (lp[i] == rp[j]) { continue; } if (lp[i] == rp[j]) {
else { return lp[i] < rp[j] ? -1 : 1;} continue;
} else {
return lp[i] < rp[j] ? -1 : 1;
}
}
if (i < lfl) {
return 1;
} else if (j < rfl) {
return -1;
} }
if (i < lfl) { return 1;}
else if (j < rfl) { return -1; }
lp += lfl; lp += lfl;
rp += rfl; rp += rfl;
@ -91,24 +97,27 @@ static int32_t compareKey(const void *l, const void *r) {
// not care item type // not care item type
return 0; return 0;
} }
IndexCache *indexCacheCreate() { IndexCache *indexCacheCreate() {
IndexCache *cache = calloc(1, sizeof(IndexCache)); IndexCache *cache = calloc(1, sizeof(IndexCache));
cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); cache->skiplist = tSkipListCreate(
MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
return cache; return cache;
} }
void indexCacheDestroy(void *cache) { void indexCacheDestroy(void *cache) {
IndexCache *pCache = cache; IndexCache *pCache = cache;
if (pCache == NULL) { return; } if (pCache == NULL) {
return;
}
tSkipListDestroy(pCache->skiplist); tSkipListDestroy(pCache->skiplist);
free(pCache); free(pCache);
} }
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) {
if (cache == NULL) { return -1;} if (cache == NULL) {
return -1;
}
IndexCache *pCache = cache; IndexCache *pCache = cache;
// encode data // encode data
@ -131,14 +140,16 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version,
tSkipListPut(pCache->skiplist, (void *)buf); tSkipListPut(pCache->skiplist, (void *)buf);
return 0; return 0;
// encode end // encode end
} }
int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
IndexCache *pCache = cache; IndexCache *pCache = cache;
return 0; return 0;
} }
int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { int indexCacheSearch(
if (cache == NULL) { return -1; } void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) {
if (cache == NULL) {
return -1;
}
IndexCache * pCache = cache; IndexCache * pCache = cache;
SIndexTerm * term = query->term; SIndexTerm * term = query->term;
EIndexQueryType qtype = query->qType; EIndexQueryType qtype = query->qType;
@ -147,15 +158,10 @@ int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t
char *buf = calloc(1, keyLen); char *buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) { if (qtype == QUERY_TERM) {
} else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_PREFIX) {
} else if (qtype == QUERY_SUFFIX) { } else if (qtype == QUERY_SUFFIX) {
} else if (qtype == QUERY_REGEX) { } else if (qtype == QUERY_REGEX) {
} }
return 0; return 0;
} }

294
source/libs/index/src/index_fst.c
View File

@ -14,11 +14,10 @@
*/ */
#include "index_fst.h" #include "index_fst.h"
#include "tcoding.h"
#include "tchecksum.h"
#include "indexInt.h" #include "indexInt.h"
#include "index_fst_automation.h" #include "index_fst_automation.h"
#include "tchecksum.h"
#include "tcoding.h"
static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr;
@ -32,7 +31,9 @@ static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, Compi
FstUnFinishedNodes *fstUnFinishedNodesCreate() { FstUnFinishedNodes *fstUnFinishedNodesCreate() {
FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes)); FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes));
if (nodes == NULL) { return NULL; } if (nodes == NULL) {
return NULL;
}
nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished));
fstUnFinishedNodesPushEmpty(nodes, false); fstUnFinishedNodesPushEmpty(nodes, false);
@ -45,7 +46,9 @@ void unFinishedNodeDestroyElem(void* elem) {
b->last = NULL; b->last = NULL;
} }
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) { void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) {
if (nodes == NULL) { return; } if (nodes == NULL) {
return;
}
taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem);
free(nodes); free(nodes);
@ -59,7 +62,6 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) {
FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; FstBuilderNodeUnfinished un = {.node = node, .last = NULL};
taosArrayPush(nodes->stack, &un); taosArrayPush(nodes->stack, &un);
} }
FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) {
assert(taosArrayGetSize(nodes->stack) == 1); assert(taosArrayGetSize(nodes->stack) == 1);
@ -81,7 +83,6 @@ FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes) {
FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack);
assert(un->last == NULL); assert(un->last == NULL);
return un->node; return un->node;
} }
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) { void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) {
FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0); FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0);
@ -127,7 +128,6 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
fstUnFinishedNodesPushEmpty(nodes, true); fstUnFinishedNodesPushEmpty(nodes, true);
} }
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) {
FstSlice *s = &bs; FstSlice *s = &bs;
@ -177,7 +177,6 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
return i; return i;
} }
FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) {
FstState fs = {.state = EmptyFinal, .val = 0}; FstState fs = {.state = EmptyFinal, .val = 0};
if (addr == EMPTY_ADDRESS) { if (addr == EMPTY_ADDRESS) {
@ -198,12 +197,8 @@ FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
return fs; return fs;
} }
static FstState fstStateDict[] = { static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, {.state = OneTrans, .val = 0b10000000},
{.state = OneTransNext, .val = 0b11000000}, {.state = AnyTrans, .val = 0b00000000}, {.state = EmptyFinal, .val = 0b00000000}};
{.state = OneTrans, .val = 0b10000000},
{.state = AnyTrans, .val = 0b00000000},
{.state = EmptyFinal, .val = 0b00000000}
};
// debug // debug
static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"};
@ -246,7 +241,6 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran
} }
fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val)); fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val));
return; return;
} }
void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) { void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) {
size_t sz = taosArrayGetSize(node->trans); size_t sz = taosArrayGetSize(node->trans);
@ -265,8 +259,11 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
} }
PackSizes packSizes = 0; PackSizes packSizes = 0;
if (anyOuts) { FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); } if (anyOuts) {
else { FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); } FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize);
} else {
FST_SET_OUTPUT_PACK_SIZE(packSizes, 0);
}
FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize);
@ -318,8 +315,11 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// the # of transitions can never be 1 here, since 1 is always // the # of transitions can never be 1 here, since 1 is always
// encoded in the state byte. // encoded in the state byte.
uint8_t v = 1; uint8_t v = 1;
if (sz == 256) { fstCountingWriterWrite(w, (char *)&v, 1); } if (sz == 256) {
else { fstCountingWriterWrite(w, (char *)&sz, 1); } fstCountingWriterWrite(w, (char *)&v, 1);
} else {
fstCountingWriterWrite(w, (char *)&sz, 1);
}
} }
fstCountingWriterWrite(w, (char *)(&(st.val)), 1); fstCountingWriterWrite(w, (char *)(&(st.val)), 1);
return; return;
@ -362,22 +362,15 @@ uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice *data) {
} }
uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) { uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) {
assert(s->state == OneTrans); assert(s->state == OneTrans);
return FST_SLICE_LEN(data) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size
- 1 - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes);
- fstStateInputLen(s)
- 1 // pack size
- FST_GET_TRANSITION_PACK_SIZE(sizes)
- FST_GET_OUTPUT_PACK_SIZE(sizes);
} }
uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) { uint64_t fstStateEndAddrForAnyTrans(
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes;
return FST_SLICE_LEN(date) return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size
- 1 - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values
- fstStateNtransLen(state)
- 1 //pack size
- fstStateTotalTransSize(state, version, sizes, nTrans)
- nTrans * oSizes // output values
- finalOsize; // final output - finalOsize; // final output
} }
// input // input
@ -393,12 +386,8 @@ uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
FstSlice *slice = &node->data; FstSlice *slice = &node->data;
uint64_t at = node->start uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateNtransLen(s) - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size
- 1 // pack size
- fstStateTransIndexSize(s, node->version, node->nTrans)
- i
- 1; // the output size
uint8_t *data = fstSliceData(slice, NULL); uint8_t *data = fstSliceData(slice, NULL);
return data[at]; return data[at];
@ -413,9 +402,7 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) {
} else { } else {
PackSizes sizes = node->sizes; PackSizes sizes = node->sizes;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes);
uint64_t i = node->start uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes
- fstStateInputLen(s)
- 1 // PackSizes
- tSizes; - tSizes;
// refactor error logic // refactor error logic
@ -428,13 +415,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i
FstSlice *slice = &node->data; FstSlice *slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
uint64_t at = node->start uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) -
- fstStateNtransLen(s) node->nTrans - (i * tSizes) - tSizes;
- 1
- fstStateTransIndexSize(s, node->version, node->nTrans)
- node->nTrans
- (i * tSizes)
- tSizes;
uint8_t *data = fstSliceData(slice, NULL); uint8_t *data = fstSliceData(slice, NULL);
return unpackDelta(data + at, tSizes, node->end); return unpackDelta(data + at, tSizes, node->end);
} }
@ -463,14 +445,9 @@ Output fstStateOutput(FstState *s, FstNode *node) {
FstSlice *slice = &node->data; FstSlice *slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
uint64_t i = node->start uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes;
- fstStateInputLen(s)
- 1
- tSizes
- oSizes;
uint8_t *data = fstSliceData(slice, NULL); uint8_t *data = fstSliceData(slice, NULL);
return unpackUint64(data + i, oSizes); return unpackUint64(data + i, oSizes);
} }
Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
@ -481,12 +458,8 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
} }
FstSlice *slice = &node->data; FstSlice *slice = &node->data;
uint8_t * data = fstSliceData(slice, NULL); uint8_t * data = fstSliceData(slice, NULL);
uint64_t at = node->start uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateNtransLen(s) - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes;
- 1 // pack size
- fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans)
- (i * oSizes)
- oSizes;
return unpackUint64(data + at, oSizes); return unpackUint64(data + at, oSizes);
} }
@ -495,7 +468,9 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
void fstStateSetFinalState(FstState *s, bool yes) { void fstStateSetFinalState(FstState *s, bool yes) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (yes) { s->val |= 0b01000000; } if (yes) {
s->val |= 0b01000000;
}
return; return;
} }
bool fstStateIsFinalState(FstState *s) { bool fstStateIsFinalState(FstState *s) {
@ -546,7 +521,8 @@ uint64_t fstStateNtrans(FstState *s, FstSlice *slice) {
uint8_t *data = fstSliceData(slice, &len); uint8_t *data = fstSliceData(slice, &len);
n = data[len - 2]; n = data[len - 2];
// n = data[slice->end - 1]; // data[data.len() - 2] // n = data[slice->end - 1]; // data[data.len() - 2]
return n == 1 ? 256: n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, then it is encoded in the state byte return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition,
// then it is encoded in the state byte
} }
Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
@ -554,24 +530,16 @@ Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, Pack
return 0; return 0;
} }
uint64_t at = FST_SLICE_LEN(slice) uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size
- 1 - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes;
- fstStateNtransLen(s)
- 1 // pack size
- fstStateTotalTransSize(s, version, sizes, nTrans)
- (nTrans * oSizes)
- oSizes;
uint8_t *data = fstSliceData(slice, NULL); uint8_t *data = fstSliceData(slice, NULL);
return unpackUint64(data + at, (uint8_t)oSizes); return unpackUint64(data + at, (uint8_t)oSizes);
} }
uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
FstSlice *slice = &node->data; FstSlice *slice = &node->data;
if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) {
uint64_t at = node->start uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateNtransLen(s)
- 1 // pack size
- fstStateTransIndexSize(s, node->version, node->nTrans); - fstStateTransIndexSize(s, node->version, node->nTrans);
int32_t dlen = 0; int32_t dlen = 0;
uint8_t *data = fstSliceData(slice, &dlen); uint8_t *data = fstSliceData(slice, &dlen);
@ -582,9 +550,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
} }
return i; return i;
} else { } else {
uint64_t start = node->start uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size
- fstStateNtransLen(s)
- 1 // pack size
- node->nTrans; - node->nTrans;
uint64_t end = start + node->nTrans; uint64_t end = start + node->nTrans;
FstSlice t = fstSliceCopy(slice, start, end - 1); FstSlice t = fstSliceCopy(slice, start, end - 1);
@ -598,17 +564,20 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
return node->nTrans - i - 1; // bug return node->nTrans - i - 1; // bug
} }
} }
if (i == len) { *null = true; } if (i == len) {
*null = true;
}
fstSliceDestroy(&t); fstSliceDestroy(&t);
} }
} }
// fst node function // fst node function
FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) {
FstNode *n = (FstNode *)malloc(sizeof(FstNode)); FstNode *n = (FstNode *)malloc(sizeof(FstNode));
if (n == NULL) { return NULL; } if (n == NULL) {
return NULL;
}
FstState st = fstStateCreateFrom(slice, addr); FstState st = fstStateCreateFrom(slice, addr);
@ -656,7 +625,8 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) {
n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->isFinal = fstStateIsFinalState(&st); // s.is_final_state();
n->nTrans = nTrans; n->nTrans = nTrans;
n->sizes = sz; n->sizes = sz;
n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); n->finalOutput =
fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans);
} }
return n; return n;
} }
@ -667,7 +637,6 @@ static const char *fstNodeState(FstNode *node) {
return fstStateStr[st->state]; return fstStateStr[st->state];
} }
void fstNodeDestroy(FstNode *node) { void fstNodeDestroy(FstNode *node) {
fstSliceDestroy(&node->data); fstSliceDestroy(&node->data);
free(node); free(node);
@ -731,16 +700,25 @@ bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) {
bool s = true; bool s = true;
FstState *st = &node->state; FstState *st = &node->state;
if (st->state == OneTransNext) { if (st->state == OneTransNext) {
if (fstStateInput(st,node) == b) { *res = 0; } if (fstStateInput(st, node) == b) {
else { s = false; } } *res = 0;
else if (st->state == OneTrans) { } else {
if (fstStateInput(st, node) == b) { *res = 0 ;} s = false;
else { s = false; } }
} else if (st->state == OneTrans) {
if (fstStateInput(st, node) == b) {
*res = 0;
} else {
s = false;
}
} else if (st->state == AnyTrans) { } else if (st->state == AnyTrans) {
bool null = false; bool null = false;
uint64_t out = fstStateFindInput(st, node, b, &null); uint64_t out = fstStateFindInput(st, node, b, &null);
if (null == false) { *res = out; } if (null == false) {
else { s = false;} *res = out;
} else {
s = false;
}
} }
return s; return s;
} }
@ -772,12 +750,11 @@ bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, Compiled
return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b);
} }
FstBuilder *fstBuilderCreate(void *w, FstType ty) { FstBuilder *fstBuilderCreate(void *w, FstType ty) {
FstBuilder *b = malloc(sizeof(FstBuilder)); FstBuilder *b = malloc(sizeof(FstBuilder));
if (NULL == b) { return b; } if (NULL == b) {
return b;
}
b->wrt = fstCountingWriterCreate(w); b->wrt = fstCountingWriterCreate(w);
b->unfinished = fstUnFinishedNodesCreate(); b->unfinished = fstUnFinishedNodesCreate();
@ -799,7 +776,9 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
return b; return b;
} }
void fstBuilderDestroy(FstBuilder *b) { void fstBuilderDestroy(FstBuilder *b) {
if (b == NULL) { return; } if (b == NULL) {
return;
}
fstCountingWriterDestroy(b->wrt); fstCountingWriterDestroy(b->wrt);
fstUnFinishedNodesDestroy(b->unfinished); fstUnFinishedNodesDestroy(b->unfinished);
@ -808,7 +787,6 @@ void fstBuilderDestroy(FstBuilder *b) {
free(b); free(b);
} }
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) {
OrderType t = fstBuilderCheckLastKey(b, bs, true); OrderType t = fstBuilderCheckLastKey(b, bs, true);
if (t == Ordered) { if (t == Ordered) {
@ -888,9 +866,7 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) {
return; return;
} }
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) {
if (FST_BUILDER_NODE_IS_FINAL(bn) if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
&& FST_BUILDER_NODE_TRANS_ISEMPTY(bn)
&& FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
return EMPTY_ADDRESS; return EMPTY_ADDRESS;
} }
FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn);
@ -938,11 +914,7 @@ void* fstBuilderInsertInner(FstBuilder *b) {
// b->wrt = NULL; // b->wrt = NULL;
return b->wrt; return b->wrt;
} }
void fstBuilderFinish(FstBuilder *b) { void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); }
fstBuilderInsertInner(b);
}
FstSlice fstNodeAsSlice(FstNode *node) { FstSlice fstNodeAsSlice(FstNode *node) {
FstSlice *slice = &node->data; FstSlice *slice = &node->data;
@ -952,19 +924,21 @@ FstSlice fstNodeAsSlice(FstNode *node) {
FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) {
FstLastTransition *trn = malloc(sizeof(FstLastTransition)); FstLastTransition *trn = malloc(sizeof(FstLastTransition));
if (trn == NULL) { return NULL; } if (trn == NULL) {
return NULL;
}
trn->inp = inp; trn->inp = inp;
trn->out = out; trn->out = out;
return trn; return trn;
} }
void fstLastTransitionDestroy(FstLastTransition *trn) { void fstLastTransitionDestroy(FstLastTransition *trn) { free(trn); }
free(trn);
}
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) { void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) {
FstLastTransition *trn = unNode->last; FstLastTransition *trn = unNode->last;
if (trn == NULL) { return; } if (trn == NULL) {
return;
}
FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
taosArrayPush(unNode->node->trans, &t); taosArrayPush(unNode->node->trans, &t);
fstLastTransitionDestroy(trn); fstLastTransitionDestroy(trn);
@ -1020,7 +994,9 @@ Fst* fstCreate(FstSlice *slice) {
taosDecodeFixedU64(buf + len, &fstLen); taosDecodeFixedU64(buf + len, &fstLen);
// TODO(validate root addr) // TODO(validate root addr)
Fst *fst = (Fst *)calloc(1, sizeof(Fst)); Fst *fst = (Fst *)calloc(1, sizeof(Fst));
if (fst == NULL) { return NULL; } if (fst == NULL) {
return NULL;
}
fst->meta = (FstMeta *)malloc(sizeof(FstMeta)); fst->meta = (FstMeta *)malloc(sizeof(FstMeta));
if (NULL == fst->meta) { if (NULL == fst->meta) {
@ -1042,7 +1018,6 @@ Fst* fstCreate(FstSlice *slice) {
FST_CREAT_FAILED: FST_CREAT_FAILED:
free(fst->meta); free(fst->meta);
free(fst); free(fst);
} }
void fstDestroy(Fst *fst) { void fstDestroy(Fst *fst) {
if (fst) { if (fst) {
@ -1091,16 +1066,14 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
return true; return true;
} }
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); }
return fstStreamBuilderCreate(fst, ctx);
}
StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) { StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) {
if (sb == NULL) { return NULL; } if (sb == NULL) {
return NULL;
}
return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
} }
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); }
return fstStreamBuilderCreate(fst, ctx);
}
FstNode *fstGetRoot(Fst *fst) { FstNode *fstGetRoot(Fst *fst) {
if (fst->root != NULL) { if (fst->root != NULL) {
@ -1110,16 +1083,9 @@ FstNode *fstGetRoot(Fst *fst) {
fst->root = fstGetNode(fst, rAddr); fst->root = fstGetNode(fst, rAddr);
return fst->root; return fst->root;
} }
FstNode* fstGetNode(Fst *fst, CompiledAddr addr) { FstNode * fstGetNode(Fst *fst, CompiledAddr addr) { return fstNodeCreate(fst->meta->version, addr, fst->data); }
return fstNodeCreate(fst->meta->version, addr, fst->data); FstType fstGetType(Fst *fst) { return fst->meta->ty; }
CompiledAddr fstGetRootAddr(Fst *fst) { return fst->meta->rootAddr; }
}
FstType fstGetType(Fst *fst) {
return fst->meta->ty;
}
CompiledAddr fstGetRootAddr(Fst *fst) {
return fst->meta->rootAddr;
}
Output fstEmptyFinalOutput(Fst *fst, bool *null) { Output fstEmptyFinalOutput(Fst *fst, bool *null) {
Output res = 0; Output res = 0;
@ -1147,7 +1113,9 @@ bool fstVerify(Fst *fst) {
// data bound function // data bound function
FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) {
FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData));
if (b == NULL) { return NULL; } if (b == NULL) {
return NULL;
}
if (data != NULL) { if (data != NULL) {
b->data = fstSliceCopy(data, data->start, data->end); b->data = fstSliceCopy(data, data->start, data->end);
@ -1159,7 +1127,6 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) {
return b; return b;
} }
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) {
int comp = fstSliceCompare(slice, &bound->data); int comp = fstSliceCompare(slice, &bound->data);
if (bound->type == Included) { if (bound->type == Included) {
@ -1178,18 +1145,16 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
} }
} }
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { return bound->type == Excluded ? false : true; }
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { void fstBoundDestroy(FstBoundWithData *bound) { free(bound); }
return bound->type == Excluded? false : true;
}
void fstBoundDestroy(FstBoundWithData *bound) { StreamWithState *streamWithStateCreate(
free(bound); Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) {
}
StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) {
StreamWithState *sws = calloc(1, sizeof(StreamWithState)); StreamWithState *sws = calloc(1, sizeof(StreamWithState));
if (sws == NULL) { return NULL; } if (sws == NULL) {
return NULL;
}
sws->fst = fst; sws->fst = fst;
sws->aut = automation; sws->aut = automation;
@ -1205,7 +1170,9 @@ StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstB
return sws; return sws;
} }
void streamWithStateDestroy(StreamWithState *sws) { void streamWithStateDestroy(StreamWithState *sws) {
if (sws == NULL) { return; } if (sws == NULL) {
return;
}
taosArrayDestroy(sws->inp); taosArrayDestroy(sws->inp);
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
@ -1214,7 +1181,6 @@ void streamWithStateDestroy(StreamWithState *sws) {
} }
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
AutomationCtx *aut = sws->aut; AutomationCtx *aut = sws->aut;
if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsEmpty(min)) {
if (fstBoundWithDataIsIncluded(min)) { if (fstBoundWithDataIsIncluded(min)) {
@ -1228,7 +1194,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
return true; return true;
} }
FstSlice *key = NULL; FstSlice *key = NULL;
bool inclusize = false;; bool inclusize = false;
;
if (min->type == Included) { if (min->type == Included) {
key = &min->data; key = &min->data;
@ -1257,16 +1224,12 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
// autState = sws->aut->accept(preState, b); // autState = sws->aut->accept(preState, b);
autState = automFuncs[aut->type].accept(aut, preState, b); autState = automFuncs[aut->type].accept(aut, preState, b);
taosArrayPush(sws->inp, &b); taosArrayPush(sws->inp, &b);
StreamState s = {.node = node, StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState};
.trans = res + 1,
.out = {.null = false, .out = out},
.autState = preState};
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
out += trn.out; out += trn.out;
node = fstGetNode(sws->fst, trn.addr); node = fstGetNode(sws->fst, trn.addr);
fstNodeDestroy(node); fstNodeDestroy(node);
} else { } else {
// This is a little tricky. We're in this case if the // This is a little tricky. We're in this case if the
// given bound is not a prefix of any key in the FST. // given bound is not a prefix of any key in the FST.
// Since this is a minimum bound, we need to find the // Since this is a minimum bound, we need to find the
@ -1281,10 +1244,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
} }
} }
StreamState s = {.node = node, StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
.trans = i,
.out = {.null = false, .out = out},
.autState = autState};
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
return true; return true;
} }
@ -1300,10 +1260,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
uint64_t trans = s->trans; uint64_t trans = s->trans;
FstTransition trn; FstTransition trn;
fstNodeGetTransitionAt(n, trans - 1, &trn); fstNodeGetTransitionAt(n, trans - 1, &trn);
StreamState s = {.node = fstGetNode(sws->fst, trn.addr), StreamState s = {
.trans= 0, .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState};
.out = {.null = false, .out = out},
.autState = autState};
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
return true; return true;
} }
@ -1360,7 +1318,6 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState};
taosArrayPush(sws->stack, &s2); taosArrayPush(sws->stack, &s2);
size_t isz = taosArrayGetSize(sws->inp); size_t isz = taosArrayGetSize(sws->inp);
uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t));
for (uint32_t i = 0; i < isz; i++) { for (uint32_t i = 0; i < isz; i++) {
@ -1390,12 +1347,13 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
} }
taosArrayDestroy(nodes); taosArrayDestroy(nodes);
return NULL; return NULL;
} }
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) { StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) {
StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult));
if (result == NULL) { return NULL; } if (result == NULL) {
return NULL;
}
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut; result->out = fOut;
@ -1404,7 +1362,9 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
return result; return result;
} }
void swsResultDestroy(StreamWithStateResult *result) { void swsResultDestroy(StreamWithStateResult *result) {
if (NULL == result) { return; } if (NULL == result) {
return;
}
fstSliceDestroy(&result->data); fstSliceDestroy(&result->data);
startWithStateValueDestroy(result->state); startWithStateValueDestroy(result->state);
@ -1412,7 +1372,9 @@ void swsResultDestroy(StreamWithStateResult *result) {
} }
void streamStateDestroy(void *s) { void streamStateDestroy(void *s) {
if (NULL == s) { return; } if (NULL == s) {
return;
}
StreamState *ss = (StreamState *)s; StreamState *ss = (StreamState *)s;
fstNodeDestroy(ss->node); fstNodeDestroy(ss->node);
@ -1421,7 +1383,9 @@ void streamStateDestroy(void *s) {
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) {
FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { return NULL; } if (NULL == b) {
return NULL;
}
b->fst = fst; b->fst = fst;
b->aut = aut; b->aut = aut;
@ -1437,7 +1401,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder *b) {
free(b); free(b);
} }
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) {
if (b == NULL) { return NULL; } if (b == NULL) {
return NULL;
}
if (type == GE) { if (type == GE) {
b->min->type = Included; b->min->type = Included;
@ -1458,9 +1424,3 @@ FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, Rang
} }
return b; return b;
} }

74
source/libs/index/src/index_fst_automation.c
View File

@ -15,10 +15,11 @@
#include "index_fst_automation.h" #include "index_fst_automation.h"
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; } if (nsv == NULL) {
return NULL;
}
nsv->kind = kind; nsv->kind = kind;
nsv->type = ty; nsv->type = ty;
@ -36,7 +37,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp
} }
void startWithStateValueDestroy(void *val) { void startWithStateValueDestroy(void *val) {
StartWithStateValue *sv = (StartWithStateValue *)val; StartWithStateValue *sv = (StartWithStateValue *)val;
if (sv == NULL) { return; } if (sv == NULL) {
return;
}
if (sv->type == FST_INT) { if (sv->type == FST_INT) {
// //
@ -49,7 +52,9 @@ void startWithStateValueDestroy(void *val) {
} }
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; } if (nsv == NULL) {
return NULL;
}
nsv->kind = sv->kind; nsv->kind = sv->kind;
nsv->type = sv->type; nsv->type = sv->type;
@ -64,7 +69,6 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
return nsv; return nsv;
} }
// prefix query, impl later // prefix query, impl later
static void *prefixStart(AutomationCtx *ctx) { static void *prefixStart(AutomationCtx *ctx) {
@ -79,12 +83,12 @@ static bool prefixCanMatch(AutomationCtx *ctx, void *sv) {
StartWithStateValue *ssv = (StartWithStateValue *)sv; StartWithStateValue *ssv = (StartWithStateValue *)sv;
return ssv->val >= 0; return ssv->val >= 0;
} }
static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; }
return true;
}
static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
StartWithStateValue *ssv = (StartWithStateValue *)state; StartWithStateValue *ssv = (StartWithStateValue *)state;
if (ssv == NULL || ctx == NULL) {return NULL;} if (ssv == NULL || ctx == NULL) {
return NULL;
}
char *data = ctx->data; char *data = ctx->data;
if (ssv->kind == Done) { if (ssv->kind == Done) {
@ -102,55 +106,30 @@ static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
} }
return NULL; return NULL;
} }
static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; }
return NULL;
}
// pattern query, impl later // pattern query, impl later
static void* patternStart(AutomationCtx *ctx) { static void *patternStart(AutomationCtx *ctx) { return NULL; }
return NULL; static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; }
} static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; }
static bool patternIsMatch(AutomationCtx *ctx, void *data) { static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; }
return true;
}
static bool patternCanMatch(AutomationCtx *ctx, void *data) {
return true;
}
static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) {
return true;
}
static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; }
return NULL;
}
static void* patternAcceptEof(AutomationCtx *ctx, void *state) { static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; }
return NULL;
}
AutomationFunc automFuncs[] = {{ AutomationFunc automFuncs[] = {
prefixStart, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof},
prefixIsMatch, {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof}
prefixCanMatch,
prefixWillAlwaysMatch,
prefixAccept,
prefixAcceptEof
},
{
patternStart,
patternIsMatch,
patternCanMatch,
patternWillAlwaysMatch,
patternAccept,
patternAcceptEof
}
// add more search type // add more search type
}; };
AutomationCtx *automCtxCreate(void *data, AutomationType atype) { AutomationCtx *automCtxCreate(void *data, AutomationType atype) {
AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) {
return NULL;
}
StartWithStateValue *sv = NULL; StartWithStateValue *sv = NULL;
if (atype == AUTOMATION_PREFIX) { if (atype == AUTOMATION_PREFIX) {
@ -158,7 +137,6 @@ AutomationCtx* automCtxCreate(void *data,AutomationType atype) {
sv = startWithStateValueCreate(Running, FST_INT, &val); sv = startWithStateValueCreate(Running, FST_INT, &val);
ctx->stdata = (void *)sv; ctx->stdata = (void *)sv;
} else if (atype == AUTMMATION_MATCH) { } else if (atype == AUTMMATION_MATCH) {
} else { } else {
// add more search type // add more search type
} }

288
source/libs/index/src/index_fst_common.c
View File

@ -12,8 +12,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "index_fst_common.h"
#include "tutil.h"
const uint8_t COMMON_INPUTS[] = { const uint8_t COMMON_INPUTS[] = {
84, // '\x00' 84, // '\x00'
@ -275,33 +274,260 @@ const uint8_t COMMON_INPUTS[] = {
}; };
const char COMMON_INPUTS_INV[] = { const char COMMON_INPUTS_INV[] = {
't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', 't',
'.', 'h', 'l', 'm', '-', 'd', 'u', '0', '1', '2', 'g', '=', 'e',
':', 'b', 'f', '3', 'y', '5', '&', '_', '4', 'v', '9', '6', '/',
'7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', 'o',
'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'a',
'q', 'H', 'G', 'W', 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 's',
'Q', ';', ')', '(', '~', '[', ']', '$', '!', '\'', '*', '@', 'r',
'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', 'i',
'\x08', '\t', '\n', '\x0b', '\x0c', '\r', '\x0e', '\x0f', '\x10', 'p',
'\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', 'c',
'\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', 'n',
'#', '<', '>', '\\', '^', '`', '{', '|', '}','\x7f','\x80', 'w',
'\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '.',
'\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', '\x90', 'h',
'\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', 'l',
'\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', '\xa0', 'm',
'\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '-',
'\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', '\xb0', 'd',
'\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', 'u',
'\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', '\xc0', '0',
'\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '1',
'\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', '\xd0', '2',
'\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', 'g',
'\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', '\xe0', '=',
'\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', ':',
'\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', '\xf0', 'b',
'\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', 'f',
'\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', '3',
'y',
'5',
'&',
'_',
'4',
'v',
'9',
'6',
'7',
'8',
'k',
'%',
'?',
'x',
'C',
'D',
'A',
'S',
'F',
'I',
'B',
'E',
'j',
'P',
'T',
'z',
'R',
'N',
'M',
'+',
'L',
'O',
'q',
'H',
'G',
'W',
'U',
'V',
',',
'Y',
'K',
'J',
'Z',
'X',
'Q',
';',
')',
'(',
'~',
'[',
']',
'$',
'!',
'\'',
'*',
'@',
'\x00',
'\x01',
'\x02',
'\x03',
'\x04',
'\x05',
'\x06',
'\x07',
'\x08',
'\t',
'\n',
'\x0b',
'\x0c',
'\r',
'\x0e',
'\x0f',
'\x10',
'\x11',
'\x12',
'\x13',
'\x14',
'\x15',
'\x16',
'\x17',
'\x18',
'\x19',
'\x1a',
'\x1b',
'\x1c',
'\x1d',
'\x1e',
'\x1f',
' ',
'"',
'#',
'<',
'>',
'\\',
'^',
'`',
'{',
'|',
'}',
'\x7f',
'\x80',
'\x81',
'\x82',
'\x83',
'\x84',
'\x85',
'\x86',
'\x87',
'\x88',
'\x89',
'\x8a',
'\x8b',
'\x8c',
'\x8d',
'\x8e',
'\x8f',
'\x90',
'\x91',
'\x92',
'\x93',
'\x94',
'\x95',
'\x96',
'\x97',
'\x98',
'\x99',
'\x9a',
'\x9b',
'\x9c',
'\x9d',
'\x9e',
'\x9f',
'\xa0',
'\xa1',
'\xa2',
'\xa3',
'\xa4',
'\xa5',
'\xa6',
'\xa7',
'\xa8',
'\xa9',
'\xaa',
'\xab',
'\xac',
'\xad',
'\xae',
'\xaf',
'\xb0',
'\xb1',
'\xb2',
'\xb3',
'\xb4',
'\xb5',
'\xb6',
'\xb7',
'\xb8',
'\xb9',
'\xba',
'\xbb',
'\xbc',
'\xbd',
'\xbe',
'\xbf',
'\xc0',
'\xc1',
'\xc2',
'\xc3',
'\xc4',
'\xc5',
'\xc6',
'\xc7',
'\xc8',
'\xc9',
'\xca',
'\xcb',
'\xcc',
'\xcd',
'\xce',
'\xcf',
'\xd0',
'\xd1',
'\xd2',
'\xd3',
'\xd4',
'\xd5',
'\xd6',
'\xd7',
'\xd8',
'\xd9',
'\xda',
'\xdb',
'\xdc',
'\xdd',
'\xde',
'\xdf',
'\xe0',
'\xe1',
'\xe2',
'\xe3',
'\xe4',
'\xe5',
'\xe6',
'\xe7',
'\xe8',
'\xe9',
'\xea',
'\xeb',
'\xec',
'\xed',
'\xee',
'\xef',
'\xf0',
'\xf1',
'\xf2',
'\xf3',
'\xf4',
'\xf5',
'\xf6',
'\xf7',
'\xf8',
'\xf9',
'\xfa',
'\xfb',
'\xfc',
'\xfd',
'\xfe',
'\xff',
}; };

32
source/libs/index/src/index_fst_counting_writer.c
View File

@ -12,10 +12,10 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tutil.h" #include "index_fst_counting_writer.h"
#include "indexInt.h" #include "indexInt.h"
#include "index_fst_util.h" #include "index_fst_util.h"
#include "index_fst_counting_writer.h" #include "tutil.h"
static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
if (ctx->offset + len > ctx->limit) { if (ctx->offset + len > ctx->limit) {
@ -53,7 +53,9 @@ static int writeCtxDoFlush(WriterCtx *ctx) {
WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) {
WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); WriterCtx *ctx = calloc(1, sizeof(WriterCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) {
return NULL;
}
ctx->type = type; ctx->type = type;
if (ctx->type == TFile) { if (ctx->type == TFile) {
@ -81,7 +83,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int
return ctx; return ctx;
END: END:
if (ctx->type == TMemory) { free(ctx->mem.buf); } if (ctx->type == TMemory) {
free(ctx->mem.buf);
}
free(ctx); free(ctx);
} }
void writerCtxDestroy(WriterCtx *ctx) { void writerCtxDestroy(WriterCtx *ctx) {
@ -93,10 +97,11 @@ void writerCtxDestroy(WriterCtx *ctx) {
free(ctx); free(ctx);
} }
FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *fstCountingWriterCreate(void *wrt) {
FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter));
if (cw == NULL) { return NULL; } if (cw == NULL) {
return NULL;
}
cw->wrt = wrt; cw->wrt = wrt;
//(void *)(writerCtxCreate(TFile, readOnly)); //(void *)(writerCtxCreate(TFile, readOnly));
@ -110,7 +115,9 @@ void fstCountingWriterDestroy(FstCountingWriter *cw) {
} }
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) {
if (write == NULL) { return 0; } if (write == NULL) {
return 0;
}
// update checksum // update checksum
// write data to file/socket or mem // write data to file/socket or mem
WriterCtx *ctx = write->wrt; WriterCtx *ctx = write->wrt;
@ -121,17 +128,16 @@ int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len)
return len; return len;
} }
int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) { int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) {
if (write == NULL) { return 0; } if (write == NULL) {
return 0;
}
WriterCtx *ctx = write->wrt; WriterCtx *ctx = write->wrt;
int nRead = ctx->read(ctx, buf, len); int nRead = ctx->read(ctx, buf, len);
// assert(nRead == len); // assert(nRead == len);
return nRead; return nRead;
} }
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; }
return 0;
}
int fstCountingWriterFlush(FstCountingWriter *write) { int fstCountingWriterFlush(FstCountingWriter *write) {
WriterCtx *ctx = write->wrt; WriterCtx *ctx = write->wrt;
ctx->flush(ctx); ctx->flush(ctx);
@ -156,5 +162,3 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) {
fstCountingWriterPackUintIn(writer, n, nBytes); fstCountingWriterPackUintIn(writer, n, nBytes);
return nBytes; return nBytes;
} }

23
source/libs/index/src/index_fst_node.c
View File

@ -22,14 +22,18 @@ FstBuilderNode *fstBuilderNodeDefault() {
return bn; return bn;
} }
void fstBuilderNodeDestroy(FstBuilderNode *node) { void fstBuilderNodeDestroy(FstBuilderNode *node) {
if (node == NULL) { return; } if (node == NULL) {
return;
}
taosArrayDestroy(node->trans); taosArrayDestroy(node->trans);
free(node); free(node);
} }
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) {
if (n1 == n2) { return true; } if (n1 == n2) {
return true;
}
if (n1 == NULL || n2 == NULL) { if (n1 == NULL || n2 == NULL) {
return false; return false;
} }
@ -54,7 +58,9 @@ bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) {
} }
FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); FstBuilderNode *node = malloc(sizeof(FstBuilderNode));
if (node == NULL) { return NULL; } if (node == NULL) {
return NULL;
}
// //
size_t sz = taosArrayGetSize(src->trans); size_t sz = taosArrayGetSize(src->trans);
@ -69,11 +75,12 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
node->isFinal = src->isFinal; node->isFinal = src->isFinal;
node->finalOutput = src->finalOutput; node->finalOutput = src->finalOutput;
return node; return node;
} }
// not destroy src, User's bussiness // not destroy src, User's bussiness
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
if (dst == NULL || src == NULL) { return; } if (dst == NULL || src == NULL) {
return;
}
dst->isFinal = src->isFinal; dst->isFinal = src->isFinal;
dst->finalOutput = src->finalOutput; dst->finalOutput = src->finalOutput;
@ -88,8 +95,8 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
} }
} }
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { // startAddr) {
// size_t sz = taosArrayGetSize(b->trans); // size_t sz = taosArrayGetSize(b->trans);
// assert(sz < 256); // assert(sz < 256);
@ -111,5 +118,3 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
//} //}
// return true; // return true;
//} //}

24
source/libs/index/src/index_fst_registry.c
View File

@ -15,7 +15,6 @@
#include "index_fst_registry.h" #include "index_fst_registry.h"
uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
// TODO(yihaoDeng): refactor later // TODO(yihaoDeng): refactor later
const uint64_t FNV_PRIME = 1099511628211; const uint64_t FNV_PRIME = 1099511628211;
@ -32,11 +31,12 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME;
} }
return h % (registry->tableSize); return h % (registry->tableSize);
} }
static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (a >= sz || b >= sz) { return; } if (a >= sz || b >= sz) {
return;
}
FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a);
FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b);
@ -53,7 +53,9 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (start >= sz && end >= sz) {return; } if (start >= sz && end >= sz) {
return;
}
assert(start >= end); assert(start >= end);
@ -67,7 +69,9 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
FstRegistry *registry = malloc(sizeof(FstRegistry)); FstRegistry *registry = malloc(sizeof(FstRegistry));
if (registry == NULL) { return NULL ;} if (registry == NULL) {
return NULL;
}
uint64_t nCells = tableSize * mruSize; uint64_t nCells = tableSize * mruSize;
SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell));
@ -88,7 +92,9 @@ FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
} }
void fstRegistryDestroy(FstRegistry *registry) { void fstRegistryDestroy(FstRegistry *registry) {
if (registry == NULL) { return; } if (registry == NULL) {
return;
}
SArray *tb = registry->table; SArray *tb = registry->table;
size_t sz = taosArrayGetSize(tb); size_t sz = taosArrayGetSize(tb);
@ -168,8 +174,4 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
} }
return entry; return entry;
} }
void fstRegistryEntryDestroy(FstRegistryEntry *entry) { void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); }
free(entry);
}

33
source/libs/index/src/index_fst_util.c
View File

@ -15,8 +15,6 @@
#include "index_fst_util.h" #include "index_fst_util.h"
#include "index_fst_common.h" #include "index_fst_common.h"
// A sentinel value used to indicate an empty final state // A sentinel value used to indicate an empty final state
const CompiledAddr EMPTY_ADDRESS = 0; const CompiledAddr EMPTY_ADDRESS = 0;
/// A sentinel value used to indicate an invalid state. /// A sentinel value used to indicate an invalid state.
@ -31,7 +29,6 @@ const uint64_t VERSION = 3;
const uint64_t TRANS_INDEX_THRESHOLD = 32; const uint64_t TRANS_INDEX_THRESHOLD = 32;
// uint8_t commonInput(uint8_t idx) { // uint8_t commonInput(uint8_t idx) {
// if (idx == 0) { return -1; } // if (idx == 0) { return -1; }
// else { // else {
@ -44,8 +41,6 @@ const uint64_t TRANS_INDEX_THRESHOLD = 32;
// return v > max ? 0: v; // return v > max ? 0: v;
//} //}
uint8_t packSize(uint64_t n) { uint8_t packSize(uint64_t n) {
if (n < (1u << 8)) { if (n < (1u << 8)) {
return 1; return 1;
@ -115,7 +110,6 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) {
return t; return t;
} }
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
int32_t tlen = end - start + 1; int32_t tlen = end - start + 1;
int32_t slen; int32_t slen;
uint8_t *data = fstSliceData(s, &slen); uint8_t *data = fstSliceData(s, &slen);
@ -135,9 +129,7 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
ans.end = tlen - 1; ans.end = tlen - 1;
return ans; return ans;
} }
bool fstSliceIsEmpty(FstSlice *s) { bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; }
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
}
uint8_t *fstSliceData(FstSlice *s, int32_t *size) { uint8_t *fstSliceData(FstSlice *s, int32_t *size) {
FstString *str = s->str; FstString *str = s->str;
@ -165,13 +157,21 @@ int fstSliceCompare(FstSlice *a, FstSlice *b) {
for (i = 0, j = 0; i < alen && j < blen; i++, j++) { for (i = 0, j = 0; i < alen && j < blen; i++, j++) {
uint8_t x = aBuf[i]; uint8_t x = aBuf[i];
uint8_t y = bBuf[j]; uint8_t y = bBuf[j];
if (x == y) { continue;} if (x == y) {
else if (x < y) { return -1; } continue;
else { return 1; }; } else if (x < y) {
return -1;
} else {
return 1;
};
}
if (i < alen) {
return 1;
} else if (j < blen) {
return -1;
} else {
return 0;
} }
if (i < alen) { return 1; }
else if (j < blen) { return -1; }
else { return 0; }
} }
// FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { // FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) {
@ -186,6 +186,3 @@ int fstSliceCompare(FstSlice *a, FstSlice *b) {
// size_t fstStackLen(FstStack *s); // size_t fstStackLen(FstStack *s);
// void *fstStackGetAt(FstStack *s, size_t i); // void *fstStackGetAt(FstStack *s, size_t i);
// void fstStackDestory(FstStack *); // void fstStackDestory(FstStack *);

91
source/libs/index/src/index_tfile.c
View File

@ -16,18 +16,18 @@
//#include <sys/types.h> //#include <sys/types.h>
//#include <dirent.h> //#include <dirent.h>
#include "index_tfile.h" #include "index_tfile.h"
#include "index.h"
#include "index_fst.h" #include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_util.h" #include "index_util.h"
#include "taosdef.h" #include "taosdef.h"
#include "index.h"
#include "index_fst_counting_writer.h"
static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) {
static FORCE_INLINE int tfileLoadHeader(WriterCtx *ctx, TFileReadHeader *header) {
// TODO simple tfile header later // TODO simple tfile header later
char buf[TFILE_HADER_PRE_SIZE]; char buf[TFILE_HADER_PRE_SIZE];
char * p = buf; char * p = buf;
int64_t nread = ctx->read(ctx, buf, TFILE_HADER_PRE_SIZE); TFileReadHeader *header = &reader->header;
int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE);
assert(nread == TFILE_HADER_PRE_SIZE); assert(nread == TFILE_HADER_PRE_SIZE);
memcpy(&header->suid, p, sizeof(header->suid)); memcpy(&header->suid, p, sizeof(header->suid));
@ -39,15 +39,17 @@ static FORCE_INLINE int tfileLoadHeader(WriterCtx *ctx, TFileReadHeader *header)
int32_t colLen = 0; int32_t colLen = 0;
memcpy(&colLen, p, sizeof(colLen)); memcpy(&colLen, p, sizeof(colLen));
assert(colLen < sizeof(header->colName)); assert(colLen < sizeof(header->colName));
nread = ctx->read(ctx, header->colName, colLen); nread = reader->ctx->read(reader->ctx, header->colName, colLen);
assert(nread == colLen); assert(nread == colLen);
nread = ctx->read(ctx, &header->colType, sizeof(header->colType)); nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType));
return 0; return 0;
}; };
static int tfileGetFileList(const char *path, SArray *result) { static int tfileGetFileList(const char *path, SArray *result) {
DIR *dir = opendir(path); DIR *dir = opendir(path);
if (NULL == dir) { return -1; } if (NULL == dir) {
return -1;
}
struct dirent *entry; struct dirent *entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
@ -90,7 +92,9 @@ static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) {
TFileCache *tfileCacheCreate(const char *path) { TFileCache *tfileCacheCreate(const char *path) {
TFileCache *tcache = calloc(1, sizeof(TFileCache)); TFileCache *tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; } if (tcache == NULL) {
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64; tcache->capacity = 64;
@ -107,29 +111,41 @@ TFileCache *tfileCacheCreate(const char *path) {
continue; continue;
} }
TFileReader *reader = calloc(1, sizeof(TFileReader)); WriterCtx *wc = writerCtxCreate(TFile, file, true, 1024 * 64);
reader->ctx = writerCtxCreate(TFile, file, true, 1024 * 64); if (wc == NULL) {
if (reader->ctx == NULL) {
TFileReaderDestroy(reader);
indexError("failed to open index: %s", file); indexError("failed to open index: %s", file);
goto End; goto End;
} }
TFileReadHeader header = {0}; TFileReader *reader = tfileReaderCreate(wc);
if (0 != tfileLoadHeader(reader->ctx, &header)) { if (0 != tfileReadLoadHeader(reader)) {
TFileReaderDestroy(reader); TFileReaderDestroy(reader);
indexError("failed to load index header, index Id: %s", file); indexError("failed to load index header, index Id: %s", file);
goto End;
} }
} }
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return tcache; return tcache;
End: End:
tfileCacheDestroy(tcache);
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache *tcache) { void tfileCacheDestroy(TFileCache *tcache) {
if (tcache == NULL) {
return;
}
// free table cache
TFileReader **reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) {
TFileReader *p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName,
p->header.colType);
TFileReaderDestroy(p);
reader = taosHashIterate(tcache->tableCache, reader);
}
taosHashCleanup(tcache->tableCache);
free(tcache); free(tcache);
} }
TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) {
@ -145,56 +161,47 @@ void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader)
return; return;
} }
TFileReader *tfileReaderCreate(WriterCtx *ctx) {
TFileReader* tfileReaderCreate() { TFileReader *reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) {
return NULL;
}
reader->ctx = ctx;
// T_REF_INC(reader);
return reader;
} }
void TFileReaderDestroy(TFileReader *reader) { void TFileReaderDestroy(TFileReader *reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
}
// T_REF_INC(reader);
writerCtxDestroy(reader->ctx); writerCtxDestroy(reader->ctx);
free(reader); free(reader);
} }
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); TFileWriter *tfileWriterCreate(const char *suid, const char *colName);
void tfileWriterDestroy(TFileWriter *tw); void tfileWriterDestroy(TFileWriter *tw);
IndexTFile *indexTFileCreate(const char *path) { IndexTFile *indexTFileCreate(const char *path) {
IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); IndexTFile *tfile = calloc(1, sizeof(IndexTFile));
tfile->cache = tfileCacheCreate(path); tfile->cache = tfileCacheCreate(path);
return tfile; return tfile;
} }
void IndexTFileDestroy(IndexTFile *tfile) { void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); }
free(tfile);
}
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) {
IndexTFile *pTfile = (IndexTFile *)tfile; IndexTFile *pTfile = (IndexTFile *)tfile;
SIndexTerm * term = query->term; SIndexTerm * term = query->term;
TFileCacheKey key = {.suid = term->suid, TFileCacheKey key = {
.colType = term->colType, .suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName};
.version = 0,
.colName = term->colName,
.nColName= term->nColName};
TFileReader *reader = tfileCacheGet(pTfile->cache, &key); TFileReader *reader = tfileCacheGet(pTfile->cache, &key);
return 0; return 0;
} }
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) {
TFileWriterOpt wOpt = {.suid = term->suid, TFileWriterOpt wOpt = {
.colType = term->colType, .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1};
.colName = term->colName,
.nColName= term->nColName,
.version = 1};
return 0; return 0;
} }