xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update index TFile manage and format code

This commit is contained in:
yihaoDeng 2021-12-21 16:36:31 +08:00
parent 6fe118c516
commit 5d4d7b47a0
22 changed files with 2205 additions and 2023 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
include/libs/index/index.h
View File

@ -23,64 +23,63 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SIndex SIndex; typedef struct SIndex SIndex;
typedef struct SIndexTerm SIndexTerm; typedef struct SIndexTerm SIndexTerm;
typedef struct SIndexOpts SIndexOpts; typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm; typedef struct SArray SIndexMultiTerm;
typedef enum { typedef enum {
ADD_VALUE, // add index colume value ADD_VALUE, // add index colume value
DEL_VALUE, // delete index column value DEL_VALUE, // delete index column value
UPDATE_VALUE, // update index column value UPDATE_VALUE, // update index column value
ADD_INDEX, // add index on specify column ADD_INDEX, // add index on specify column
DROP_INDEX, // drop existed index DROP_INDEX, // drop existed index
DROP_SATBLE // drop stable DROP_SATBLE // drop stable
} SIndexOperOnColumn; } SIndexOperOnColumn;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType;
/* /*
* @param: oper * @param: oper
* *
*/ */
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type);
/* /*
* @param: * @param:
* @param: * @param:
*/ */
int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); int indexOpen(SIndexOpts *opt, const char *path, SIndex **index);
void indexClose(SIndex *index); void indexClose(SIndex *index);
int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid);
int indexDelete(SIndex *index, SIndexMultiTermQuery *query); int indexDelete(SIndex *index, SIndexMultiTermQuery *query);
int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result);
int indexRebuild(SIndex *index, SIndexOpts *opt); int indexRebuild(SIndex *index, SIndexOpts *opt);
/* /*
* @param * @param
* @param * @param
*/ */
SIndexMultiTerm *indexMultiTermCreate(); SIndexMultiTerm *indexMultiTermCreate();
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term);
void indexMultiTermDestroy(SIndexMultiTerm *terms); void indexMultiTermDestroy(SIndexMultiTerm *terms);
/* /*
* @param: * @param:
* @param: * @param:
*/ */
SIndexOpts *indexOptsCreate(); SIndexOpts *indexOptsCreate();
void indexOptsDestroy(SIndexOpts *opts); void indexOptsDestroy(SIndexOpts *opts);
/* /*
* @param: * @param:
* @param: * @param:
*/ */
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName,
const char *colName, int32_t nColName, const char *colVal, int32_t nColVal); int32_t nColName, const char *colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm *p); void indexTermDestroy(SIndexTerm *p);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

123
source/libs/index/inc/indexInt.h
View File

@ -18,87 +18,112 @@
#include "index.h" #include "index.h"
#include "index_fst.h" #include "index_fst.h"
#include "tlog.h"
#include "thash.h"
#include "taos.h" #include "taos.h"
#include "thash.h"
#include "tlog.h"
#ifdef USE_LUCENE #ifdef USE_LUCENE
#include <lucene++/Lucene_c.h> #include <lucene++/Lucene_c.h>
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef enum {kTypeValue, kTypeDeletion} STermValueType ; typedef enum { kTypeValue, kTypeDeletion } STermValueType;
typedef struct SIndexStat { typedef struct SIndexStat {
int32_t totalAdded; // int32_t totalAdded; //
int32_t totalDeled; // int32_t totalDeled; //
int32_t totalUpdated; // int32_t totalUpdated; //
int32_t totalTerms; // int32_t totalTerms; //
int32_t distinctCol; // distinct column int32_t distinctCol; // distinct column
} SIndexStat; } SIndexStat;
struct SIndex { struct SIndex {
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t *index; index_t *index;
#endif #endif
void *cache; void * cache;
void *tindex; void * tindex;
SHashObj *colObj;// < field name, field id> SHashObj *colObj; // < field name, field id>
int64_t suid; // current super table id, -1 is normal table
int colId; // field id allocated to cache
int32_t cVersion; // current version allocated to cache
SIndexStat stat; int64_t suid; // current super table id, -1 is normal table
pthread_mutex_t mtx; int colId; // field id allocated to cache
}; int32_t cVersion; // current version allocated to cache
SIndexStat stat;
pthread_mutex_t mtx;
};
struct SIndexOpts { struct SIndexOpts {
#ifdef USE_LUCENE #ifdef USE_LUCENE
void *opts; void *opts;
#endif
#ifdef USE_INVERTED_INDEX
int32_t cacheSize; // MB
// add cache module later
#endif #endif
#ifdef USE_INVERTED_INDEX
int32_t cacheSize; // MB
// add cache module later
#endif
}; };
struct SIndexMultiTermQuery { struct SIndexMultiTermQuery {
EIndexOperatorType opera; EIndexOperatorType opera;
SArray *query; SArray * query;
}; };
// field and key; // field and key;
typedef struct SIndexTerm { typedef struct SIndexTerm {
int64_t suid; int64_t suid;
SIndexOperOnColumn operType; // oper type, add/del/update SIndexOperOnColumn operType; // oper type, add/del/update
uint8_t colType; // term data type, str/interger/json uint8_t colType; // term data type, str/interger/json
char *colName; char * colName;
int32_t nColName; int32_t nColName;
char *colVal; char * colVal;
int32_t nColVal; int32_t nColVal;
} SIndexTerm; } SIndexTerm;
typedef struct SIndexTermQuery { typedef struct SIndexTermQuery {
SIndexTerm* term; SIndexTerm * term;
EIndexQueryType qType; EIndexQueryType qType;
} SIndexTermQuery; } SIndexTermQuery;
#define indexFatal(...) \
do { \
#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) if (sDebugFlag & DEBUG_FATAL) { \
#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) taosPrintLog("index FATAL ", 255, __VA_ARGS__); \
#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); }} while(0) } \
#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); }} while(0) } while (0)
#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) #define indexError(...) \
#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) do { \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("index ERROR ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("index WARN ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("index ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }

20
source/libs/index/inc/index_cache.h
View File

@ -22,8 +22,10 @@
// ----------------- key structure in skiplist --------------------- // ----------------- key structure in skiplist ---------------------
/* A data row, the format is like below: /* A data row, the format is like below:
* content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->| * content: |<--totalLen-->|<-- fieldid-->|<--field type-->|<-- value len--->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| * |<-- value -->|<--uid -->|<--version--->|<-- itermType -->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|
* <--valuelen->|<--uint64_t->| * <-- int32_t-->|<-- int8_t --->|
*/ */
#ifdef __cplusplus #ifdef __cplusplus
@ -31,25 +33,23 @@ extern "C" {
#endif #endif
typedef struct IndexCache { typedef struct IndexCache {
T_REF_DECLARE() T_REF_DECLARE()
SSkipList *skiplist; SSkipList *skiplist;
} IndexCache; } IndexCache;
//
//
IndexCache *indexCacheCreate(); IndexCache *indexCacheCreate();
void indexCacheDestroy(void *cache); void indexCacheDestroy(void *cache);
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid);
//int indexCacheGet(void *cache, uint64_t *rst); // int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); int indexCacheSearch(
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif #endif

284
source/libs/index/inc/index_fst.h
View File

@ -20,29 +20,29 @@
extern "C" { extern "C" {
#endif #endif
#include "tarray.h"
#include "index_fst_util.h"
#include "index_fst_registry.h"
#include "index_fst_counting_writer.h"
#include "index_fst_automation.h" #include "index_fst_automation.h"
#include "index_fst_counting_writer.h"
#include "index_fst_registry.h"
#include "index_fst_util.h"
#include "tarray.h"
#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a)
typedef struct Fst Fst; typedef struct Fst Fst;
typedef struct FstNode FstNode; typedef struct FstNode FstNode;
typedef struct StreamWithState StreamWithState; typedef struct StreamWithState StreamWithState;
typedef enum { Included, Excluded, Unbounded} FstBound; typedef enum { Included, Excluded, Unbounded } FstBound;
typedef struct FstBoundWithData { typedef struct FstBoundWithData {
FstSlice data; FstSlice data;
FstBound type; FstBound type;
} FstBoundWithData; } FstBoundWithData;
typedef struct FstStreamBuilder { typedef struct FstStreamBuilder {
Fst *fst; Fst * fst;
AutomationCtx *aut; AutomationCtx * aut;
FstBoundWithData *min; FstBoundWithData *min;
FstBoundWithData *max; FstBoundWithData *max;
} FstStreamBuilder, FstStreamWithStateBuilder; } FstStreamBuilder, FstStreamWithStateBuilder;
@ -51,17 +51,14 @@ typedef struct FstRange {
uint64_t end; uint64_t end;
} FstRange; } FstRange;
typedef enum { GE, GT, LE, LT } RangeType;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
typedef enum {GE, GT, LE, LT} RangeType; FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data);
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice);
typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; bool fstBoundWithDataIsEmpty(FstBoundWithData *bound);
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound);
FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data);
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice);
bool fstBoundWithDataIsEmpty(FstBoundWithData *bound);
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound);
typedef struct FstOutput { typedef struct FstOutput {
bool null; bool null;
@ -69,110 +66,105 @@ typedef struct FstOutput {
} FstOutput; } FstOutput;
/* /*
* *
* UnFinished node and helper function * UnFinished node and helper function
* TODO: simple function name * TODO: simple function name
*/ */
typedef struct FstUnFinishedNodes { typedef struct FstUnFinishedNodes {
SArray *stack; // <FstBuilderNodeUnfinished> } FstUnFinishedNodes; SArray *stack; // <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
} FstUnFinishedNodes; } FstUnFinishedNodes;
#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) #define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack)
FstUnFinishedNodes *fstUnFinishedNodesCreate();
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node);
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal);
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out);
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr);
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out);
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs);
FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes);
FstUnFinishedNodes *fstUnFinishedNodesCreate();
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node);
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal);
FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes);
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out);
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr);
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out);
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs);
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out);
typedef struct FstBuilder { typedef struct FstBuilder {
FstCountingWriter *wrt; // The FST raw data is written directly to `wtr`. FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`.
FstUnFinishedNodes *unfinished; // The stack of unfinished nodes FstUnFinishedNodes *unfinished; // The stack of unfinished nodes
FstRegistry* registry; // A map of finished nodes. FstRegistry * registry; // A map of finished nodes.
FstSlice last; // The last word added FstSlice last; // The last word added
CompiledAddr lastAddr; // The address of the last compiled node CompiledAddr lastAddr; // The address of the last compiled node
uint64_t len; // num of keys added uint64_t len; // num of keys added
} FstBuilder; } FstBuilder;
FstBuilder *fstBuilderCreate(void *w, FstType ty); FstBuilder *fstBuilderCreate(void *w, FstType ty);
void fstBuilderDestroy(FstBuilder *b);
void fstBuilderDestroy(FstBuilder *b); void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in);
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in);
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); void * fstBuilerIntoInner(FstBuilder *b);
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); void fstBuilderFinish(FstBuilder *b);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup);
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn);
void* fstBuilerIntoInner(FstBuilder *b);
void fstBuilderFinish(FstBuilder *b);
typedef struct FstTransitions { typedef struct FstTransitions {
FstNode *node; FstNode *node;
FstRange range; FstRange range;
} FstTransitions; } FstTransitions;
//FstState and relation function // FstState and relation function
typedef struct FstState { typedef struct FstState {
State state; State state;
uint8_t val; uint8_t val;
} FstState; } FstState;
FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr); FstState fstStateCreateFrom(FstSlice *data, CompiledAddr addr);
FstState fstStateCreate(State state); FstState fstStateCreate(State state);
//compile // compile
void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp); void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp);
void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn); void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn);
void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node); void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node);
// set_comm_input // set_comm_input
void fstStateSetCommInput(FstState* state, uint8_t inp); void fstStateSetCommInput(FstState *state, uint8_t inp);
// comm_input // comm_input
uint8_t fstStateCommInput(FstState* state, bool *null); uint8_t fstStateCommInput(FstState *state, bool *null);
// input_len // input_len
uint64_t fstStateInputLen(FstState* state); uint64_t fstStateInputLen(FstState *state);
// end_addr
// end_addr uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data);
uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice *data);
uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes);
uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); uint64_t fstStateEndAddrForAnyTrans(
// input FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
uint8_t fstStateInput(FstState *state, FstNode *node); // input
uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); uint8_t fstStateInput(FstState *state, FstNode *node);
uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i);
// trans_addr // trans_addr
CompiledAddr fstStateTransAddr(FstState *state, FstNode *node); CompiledAddr fstStateTransAddr(FstState *state, FstNode *node);
CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i); CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i);
// sizes // sizes
PackSizes fstStateSizes(FstState *state, FstSlice *data); PackSizes fstStateSizes(FstState *state, FstSlice *data);
// Output // Output
Output fstStateOutput(FstState *state, FstNode *node); Output fstStateOutput(FstState *state, FstNode *node);
Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i); Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i);
// anyTrans specify function // anyTrans specify function
void fstStateSetFinalState(FstState *state, bool yes); void fstStateSetFinalState(FstState *state, bool yes);
bool fstStateIsFinalState(FstState *state); bool fstStateIsFinalState(FstState *state);
void fstStateSetStateNtrans(FstState *state, uint8_t n); void fstStateSetStateNtrans(FstState *state, uint8_t n);
// state_ntrans // state_ntrans
uint8_t fstStateStateNtrans(FstState *state, bool *null); uint8_t fstStateStateNtrans(FstState *state, bool *null);
uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans); uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans);
uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans); uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans);
uint64_t fstStateNtransLen(FstState *state); uint64_t fstStateNtransLen(FstState *state);
@ -180,72 +172,72 @@ uint64_t fstStateNtrans(FstState *state, FstSlice *slice);
Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null);
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) #define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans)
#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) #define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal)
typedef struct FstLastTransition { typedef struct FstLastTransition {
uint8_t inp; uint8_t inp;
Output out; Output out;
} FstLastTransition; } FstLastTransition;
/* /*
* FstBuilderNodeUnfinished and helper function * FstBuilderNodeUnfinished and helper function
* TODO: simple function name * TODO: simple function name
*/ */
typedef struct FstBuilderNodeUnfinished { typedef struct FstBuilderNodeUnfinished {
FstBuilderNode *node; FstBuilderNode * node;
FstLastTransition* last; FstLastTransition *last;
} FstBuilderNodeUnfinished; } FstBuilderNodeUnfinished;
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr);
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out);
/* /*
* FstNode and helper function * FstNode and helper function
*/ */
typedef struct FstNode { typedef struct FstNode {
FstSlice data; FstSlice data;
uint64_t version; uint64_t version;
FstState state; FstState state;
CompiledAddr start; CompiledAddr start;
CompiledAddr end; CompiledAddr end;
bool isFinal; bool isFinal;
uint64_t nTrans; uint64_t nTrans;
PackSizes sizes; PackSizes sizes;
Output finalOutput; Output finalOutput;
} FstNode; } FstNode;
// If this node is final and has a terminal output value, then it is, returned. Otherwise, a zero output is returned // If this node is final and has a terminal output value, then it is, returned.
// Otherwise, a zero output is returned
#define FST_NODE_FINAL_OUTPUT(node) node->finalOutput #define FST_NODE_FINAL_OUTPUT(node) node->finalOutput
// Returns true if and only if this node corresponds to a final or "match", state in the finite state transducer. // Returns true if and only if this node corresponds to a final or "match",
// state in the finite state transducer.
#define FST_NODE_IS_FINAL(node) node->isFinal #define FST_NODE_IS_FINAL(node) node->isFinal
// Returns the number of transitions in this node, The maximum number of transitions is 256. // Returns the number of transitions in this node, The maximum number of
// transitions is 256.
#define FST_NODE_LEN(node) node->nTrans #define FST_NODE_LEN(node) node->nTrans
// Returns true if and only if this node has zero transitions. // Returns true if and only if this node has zero transitions.
#define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0) #define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0)
// Return the address of this node. // Return the address of this node.
#define FST_NODE_ADDR(node) node->start #define FST_NODE_ADDR(node) node->start
FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data);
void fstNodeDestroy(FstNode *fstNode); void fstNodeDestroy(FstNode *fstNode);
FstTransitions fstNodeTransitionIter(FstNode *node); FstTransitions fstNodeTransitionIter(FstNode *node);
FstTransitions* fstNodeTransitions(FstNode *node); FstTransitions *fstNodeTransitions(FstNode *node);
bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res);
bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res);
bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res);
bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode);
FstSlice fstNodeAsSlice(FstNode *node);
// ops bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode);
FstSlice fstNodeAsSlice(FstNode *node);
// ops
typedef struct FstIndexedValue { typedef struct FstIndexedValue {
uint64_t index; uint64_t index;
@ -253,87 +245,87 @@ typedef struct FstIndexedValue {
} FstIndexedValue; } FstIndexedValue;
FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out);
void fstLastTransitionDestroy(FstLastTransition *trn); void fstLastTransitionDestroy(FstLastTransition *trn);
typedef struct FstMeta { typedef struct FstMeta {
uint64_t version; uint64_t version;
CompiledAddr rootAddr; CompiledAddr rootAddr;
FstType ty; FstType ty;
uint64_t len; uint64_t len;
uint32_t checkSum; uint32_t checkSum;
} FstMeta; } FstMeta;
typedef struct Fst { typedef struct Fst {
FstMeta *meta; FstMeta * meta;
FstSlice *data; // FstSlice *data; //
FstNode *root; // FstNode * root; //
} Fst; } Fst;
// refactor simple function // refactor simple function
Fst* fstCreate(FstSlice *data); Fst *fstCreate(FstSlice *data);
void fstDestroy(Fst *fst); void fstDestroy(Fst *fst);
bool fstGet(Fst *fst, FstSlice *b, Output *out); bool fstGet(Fst *fst, FstSlice *b, Output *out);
FstNode* fstGetNode(Fst *fst, CompiledAddr); FstNode * fstGetNode(Fst *fst, CompiledAddr);
FstNode* fstGetRoot(Fst *fst); FstNode * fstGetRoot(Fst *fst);
FstType fstGetType(Fst *fst); FstType fstGetType(Fst *fst);
CompiledAddr fstGetRootAddr(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst);
Output fstEmptyFinalOutput(Fst *fst, bool *null); Output fstEmptyFinalOutput(Fst *fst, bool *null);
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx);
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
// into stream to expand later
StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb);
// into stream to expand later bool fstVerify(Fst *fst);
StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb);
bool fstVerify(Fst *fst); // refactor this function
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
//refactor this function
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
typedef struct StreamState { typedef struct StreamState {
FstNode *node; FstNode * node;
uint64_t trans; uint64_t trans;
FstOutput out; FstOutput out;
void *autState; void * autState;
} StreamState; } StreamState;
void streamStateDestroy(void *s); void streamStateDestroy(void *s);
typedef struct StreamWithState { typedef struct StreamWithState {
Fst *fst; Fst * fst;
AutomationCtx *aut; AutomationCtx * aut;
SArray *inp; SArray * inp;
FstOutput emptyOutput; FstOutput emptyOutput;
SArray *stack; // <StreamState> SArray * stack; // <StreamState>
FstBoundWithData *endAt; FstBoundWithData *endAt;
} StreamWithState; } StreamWithState;
typedef struct StreamWithStateResult { typedef struct StreamWithStateResult {
FstSlice data; FstSlice data;
FstOutput out; FstOutput out;
void *state; void * state;
} StreamWithStateResult; } StreamWithStateResult;
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state);
void swsResultDestroy(StreamWithStateResult *result); void swsResultDestroy(StreamWithStateResult *result);
typedef void *(*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max);
typedef void* (*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ;
void streamWithStateDestroy(StreamWithState *sws); void streamWithStateDestroy(StreamWithState *sws);
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);
StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut);
// set up bound range // set up bound range
// refator, simple code by marco // refator, simple code by marco
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

48
source/libs/index/inc/index_fst_automation.h
View File

@ -21,62 +21,56 @@ extern "C" {
#include "index_fst_util.h" #include "index_fst_util.h"
typedef struct AutomationCtx AutomationCtx; typedef struct AutomationCtx AutomationCtx;
typedef enum AutomationType { typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
AUTOMATION_PREFIX,
AUTMMATION_MATCH
} AutomationType;
typedef struct StartWith { typedef struct StartWith {
AutomationCtx *autoSelf; AutomationCtx *autoSelf;
} StartWith; } StartWith;
typedef struct Complement { typedef struct Complement {
AutomationCtx *autoSelf; AutomationCtx *autoSelf;
} Complement; } Complement;
// automation // automation
typedef struct AutomationCtx { typedef struct AutomationCtx {
AutomationType type; AutomationType type;
void *stdata; void * stdata;
char *data; char * data;
} AutomationCtx; } AutomationCtx;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType; typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
typedef struct StartWithStateValue { typedef struct StartWithStateValue {
StartWithStateKind kind; StartWithStateKind kind;
ValueType type; ValueType type;
union { union {
int val; int val;
char *ptr; char * ptr;
SArray *arr; SArray *arr;
// add more type // add more type
} ; };
} StartWithStateValue; } StartWithStateValue;
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val);
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv);
void startWithStateValueDestroy(void *sv); void startWithStateValueDestroy(void *sv);
typedef struct AutomationFunc { typedef struct AutomationFunc {
void* (*start)(AutomationCtx *ctx) ; void *(*start)(AutomationCtx *ctx);
bool (*isMatch)(AutomationCtx *ctx, void *); bool (*isMatch)(AutomationCtx *ctx, void *);
bool (*canMatch)(AutomationCtx *ctx, void *data); bool (*canMatch)(AutomationCtx *ctx, void *data);
bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state);
void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte); void *(*accept)(AutomationCtx *ctx, void *state, uint8_t byte);
void* (*acceptEof)(AutomationCtx *ct, void *state); void *(*acceptEof)(AutomationCtx *ct, void *state);
} AutomationFunc; } AutomationFunc;
AutomationCtx *automCtxCreate(void *data, AutomationType atype); AutomationCtx *automCtxCreate(void *data, AutomationType atype);
void automCtxDestroy(AutomationCtx *ctx); void automCtxDestroy(AutomationCtx *ctx);
extern AutomationFunc automFuncs[]; extern AutomationFunc automFuncs[];
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
source/libs/index/inc/index_fst_common.h
View File

@ -1,9 +1,9 @@
#ifndef __INDEX_FST_COMM_H__ #ifndef __INDEX_FST_COMM_H__
#define __INDEX_FST_COMM_H__ #define __INDEX_FST_COMM_H__
#include "tutil.h"
extern const uint8_t COMMON_INPUTS[]; extern const uint8_t COMMON_INPUTS[];
extern char const COMMON_INPUTS_INV[]; extern const char COMMON_INPUTS_INV[];
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {

35
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -22,25 +22,24 @@ extern "C" {
#include "tfile.h" #include "tfile.h"
#define DefaultMem 1024 * 1024
#define DefaultMem 1024*1024
static char tmpFile[] = "./index"; static char tmpFile[] = "./index";
typedef enum WriterType {TMemory, TFile} WriterType; typedef enum WriterType { TMemory, TFile } WriterType;
typedef struct WriterCtx { typedef struct WriterCtx {
int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len);
int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len);
int (*flush)(struct WriterCtx *ctx); int (*flush)(struct WriterCtx *ctx);
WriterType type; WriterType type;
union { union {
struct { struct {
int fd; int fd;
bool readOnly; bool readOnly;
} file; } file;
struct { struct {
int32_t capa; int32_t capa;
char *buf; char * buf;
} mem; } mem;
}; };
int32_t offset; int32_t offset;
@ -51,35 +50,31 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoFlush(WriterCtx *ctx); static int writeCtxDoFlush(WriterCtx *ctx);
WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity);
void writerCtxDestroy(WriterCtx *w); void writerCtxDestroy(WriterCtx *w);
typedef uint32_t CheckSummer; typedef uint32_t CheckSummer;
typedef struct FstCountingWriter { typedef struct FstCountingWriter {
void* wrt; // wrap any writer that counts and checksum bytes written void * wrt; // wrap any writer that counts and checksum bytes written
uint64_t count; uint64_t count;
CheckSummer summer; CheckSummer summer;
} FstCountingWriter; } FstCountingWriter;
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len);
int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len); int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len);
int fstCountingWriterFlush(FstCountingWriter *write); int fstCountingWriterFlush(FstCountingWriter *write);
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write);
FstCountingWriter *fstCountingWriterCreate(void *wtr); FstCountingWriter *fstCountingWriterCreate(void *wtr);
void fstCountingWriterDestroy(FstCountingWriter *w); void fstCountingWriterDestroy(FstCountingWriter *w);
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes);
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes);
uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer) #define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
@ -89,5 +84,3 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#endif #endif
#endif #endif

23
source/libs/index/inc/index_fst_node.h
View File

@ -20,24 +20,24 @@
extern "C" { extern "C" {
#endif #endif
#include "index_fst_util.h"
#include "index_fst_counting_writer.h" #include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) #define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) #define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
#define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0) #define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0)
typedef struct FstTransition { typedef struct FstTransition {
uint8_t inp; //The byte input associated with this transition. uint8_t inp; // The byte input associated with this transition.
Output out; //The output associated with this transition Output out; // The output associated with this transition
CompiledAddr addr; //The address of the node that this transition points to CompiledAddr addr; // The address of the node that this transition points to
} FstTransition; } FstTransition;
typedef struct FstBuilderNode { typedef struct FstBuilderNode {
bool isFinal; bool isFinal;
Output finalOutput; Output finalOutput;
SArray *trans; // <FstTransition> SArray *trans; // <FstTransition>
} FstBuilderNode; } FstBuilderNode;
FstBuilderNode *fstBuilderNodeDefault(); FstBuilderNode *fstBuilderNodeDefault();
@ -45,8 +45,9 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src);
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src);
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); // bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); // CompiledAddr lastAddr, CompiledAddr startAddr);
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2);
void fstBuilderNodeDestroy(FstBuilderNode *node); void fstBuilderNodeDestroy(FstBuilderNode *node);

47
source/libs/index/inc/index_fst_registry.h
View File

@ -19,49 +19,48 @@
extern "C" { extern "C" {
#endif #endif
#include "index_fst_node.h"
#include "index_fst_util.h" #include "index_fst_util.h"
#include "tarray.h" #include "tarray.h"
#include "index_fst_node.h"
typedef struct FstRegistryCell { typedef struct FstRegistryCell {
CompiledAddr addr; CompiledAddr addr;
FstBuilderNode *node; FstBuilderNode *node;
} FstRegistryCell; } FstRegistryCell;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0) #define FST_REGISTRY_CELL_INSERT(cell, tAddr) \
do { \
cell->addr = tAddr; \
} while (0)
// typedef struct FstRegistryCache {
//typedef struct FstRegistryCache { // SArray *cells;
// SArray *cells;
// uint32_t start; // uint32_t start;
// uint32_t end; // uint32_t end;
//} FstRegistryCache; //} FstRegistryCache;
typedef enum {FOUND, NOTFOUND, REJECTED} FstRegistryEntryState; typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState;
typedef struct FstRegistryEntry { typedef struct FstRegistryEntry {
FstRegistryEntryState state; FstRegistryEntryState state;
CompiledAddr addr; CompiledAddr addr;
FstRegistryCell *cell; FstRegistryCell * cell;
} FstRegistryEntry; } FstRegistryEntry;
// Registry relation function
// Registry relation function
typedef struct FstRegistry { typedef struct FstRegistry {
SArray *table; //<FstRegistryCell> SArray * table; //<FstRegistryCell>
uint64_t tableSize; // num of rows uint64_t tableSize; // num of rows
uint64_t mruSize; // num of columns uint64_t mruSize; // num of columns
} FstRegistry; } FstRegistry;
// //
FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize);
void fstRegistryDestroy(FstRegistry *registry); void fstRegistryDestroy(FstRegistry *registry);
FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode);
FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); void fstRegistryEntryDestroy(FstRegistryEntry *entry);
void fstRegistryEntryDestroy(FstRegistryEntry *entry);
#ifdef __cplusplus #ifdef __cplusplus
} }

94
source/libs/index/inc/index_fst_util.h
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef __INDEX_FST_UTIL_H__ #ifndef __INDEX_FST_UTIL_H__
#define __INDEX_FST_UTIL_H__ #define __INDEX_FST_UTIL_H__
@ -21,16 +20,15 @@
extern "C" { extern "C" {
#endif #endif
#include "tarray.h"
#include "index_fst_common.h" #include "index_fst_common.h"
#include "tarray.h"
typedef uint64_t FstType; typedef uint64_t FstType;
typedef uint64_t CompiledAddr; typedef uint64_t CompiledAddr;
typedef uint64_t Output; typedef uint64_t Output;
typedef uint8_t PackSizes; typedef uint8_t PackSizes;
// A sentinel value used to indicate an empty final state
//A sentinel value used to indicate an empty final state
extern const CompiledAddr EMPTY_ADDRESS; extern const CompiledAddr EMPTY_ADDRESS;
/// A sentinel value used to indicate an invalid state. /// A sentinel value used to indicate an invalid state.
extern const CompiledAddr NONE_ADDRESS; extern const CompiledAddr NONE_ADDRESS;
@ -38,9 +36,9 @@ extern const CompiledAddr NONE_ADDRESS;
// This version number is written to every finite state transducer created by // This version number is written to every finite state transducer created by
// this version When a finite state transducer is read, its version number is // this version When a finite state transducer is read, its version number is
// checked against this value. // checked against this value.
extern const uint64_t VERSION; extern const uint64_t VERSION;
// The threshold (in number of transitions) at which an index is created for // The threshold (in number of transitions) at which an index is created for
// a node's transitions. This speeds up lookup time at the expense of FST size // a node's transitions. This speeds up lookup time at the expense of FST size
extern const uint64_t TRANS_INDEX_THRESHOLD; extern const uint64_t TRANS_INDEX_THRESHOLD;
// high 4 bits is transition address packed size. // high 4 bits is transition address packed size.
@ -48,73 +46,75 @@ extern const uint64_t TRANS_INDEX_THRESHOLD;
// //
// `0` is a legal value which means there are no transitions/outputs // `0` is a legal value which means there are no transitions/outputs
#define FST_SET_TRANSITION_PACK_SIZE(v, sz) \
do { \
v = (v & 0b00001111) | (sz << 4); \
} while (0)
#define FST_GET_TRANSITION_PACK_SIZE(v) (((v)&0b11110000) >> 4)
#define FST_SET_OUTPUT_PACK_SIZE(v, sz) \
do { \
v = (v & 0b11110000) | sz; \
} while (0)
#define FST_GET_OUTPUT_PACK_SIZE(v) ((v)&0b00001111)
#define FST_SET_TRANSITION_PACK_SIZE(v, sz) do {v = (v & 0b00001111) | (sz << 4); } while(0) #define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx)-1]
#define FST_GET_TRANSITION_PACK_SIZE(v) (((v) & 0b11110000) >> 4)
#define FST_SET_OUTPUT_PACK_SIZE(v, sz) do { v = (v & 0b11110000) | sz; } while(0)
#define FST_GET_OUTPUT_PACK_SIZE(v) ((v) & 0b00001111)
#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx) - 1] #define COMMON_INDEX(v, max, val) \
do { \
val = ((uint16_t)COMMON_INPUTS[v] + 1) % 256; \
val = val > max ? 0 : val; \
} while (0)
#define COMMON_INDEX(v, max, val) do { \ // uint8_t commonInput(uint8_t idx);
val = ((uint16_t)COMMON_INPUTS[v] + 1)%256; \ // uint8_t commonIdx(uint8_t v, uint8_t max);
val = val > max ? 0: val; \
} while(0)
uint8_t packSize(uint64_t n);
//uint8_t commonInput(uint8_t idx);
//uint8_t commonIdx(uint8_t v, uint8_t max);
uint8_t packSize(uint64_t n);
uint64_t unpackUint64(uint8_t *ch, uint8_t sz); uint64_t unpackUint64(uint8_t *ch, uint8_t sz);
uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr);
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr);
typedef struct FstString { typedef struct FstString {
uint8_t *data; uint8_t *data;
uint32_t len; uint32_t len;
int32_t ref; int32_t ref;
} FstString; } FstString;
typedef struct FstSlice { typedef struct FstSlice {
FstString *str; FstString *str;
int32_t start; int32_t start;
int32_t end; int32_t end;
} FstSlice; } FstSlice;
FstSlice fstSliceCreate(uint8_t *data, uint64_t len); FstSlice fstSliceCreate(uint8_t *data, uint64_t len);
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end);
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end);
bool fstSliceIsEmpty(FstSlice *s); bool fstSliceIsEmpty(FstSlice *s);
int fstSliceCompare(FstSlice *s1, FstSlice *s2); int fstSliceCompare(FstSlice *s1, FstSlice *s2);
void fstSliceDestroy(FstSlice *s); void fstSliceDestroy(FstSlice *s);
uint8_t *fstSliceData(FstSlice *s, int32_t *sz); uint8_t *fstSliceData(FstSlice *s, int32_t *sz);
#define FST_SLICE_LEN(s) (s->end - s->start + 1) #define FST_SLICE_LEN(s) (s->end - s->start + 1)
//// stack //// stack
// //
//typedef (*StackFreeElemFn)(void *elem); // typedef (*StackFreeElemFn)(void *elem);
// //
//typedef struct FstStack { // typedef struct FstStack {
// void *first; // void *first;
// void *end; // void *end;
// size_t elemSize; // size_t elemSize;
// size_t nElem; // size_t nElem;
// StackFreeElemFn fn; // StackFreeElemFn fn;
//} FstStack; //} FstStack;
// //
// //
//FstStack* fstStackCreate(size_t elemSize, stackFreeElem); // FstStack* fstStackCreate(size_t elemSize, stackFreeElem);
//void *fstStackPush(FstStack *s, void *elem); // void *fstStackPush(FstStack *s, void *elem);
//void *fstStackTop(FstStack *s); // void *fstStackTop(FstStack *s);
//size_t fstStackLen(FstStack *s); // size_t fstStackLen(FstStack *s);
//void fstStackDestory(FstStack *); // void fstStackDestory(FstStack *);
// //
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

89
source/libs/index/inc/index_tfile.h
View File

@ -17,10 +17,10 @@
#include "index.h" #include "index.h"
#include "indexInt.h" #include "indexInt.h"
#include "tlockfree.h"
#include "index_tfile.h"
#include "index_fst_counting_writer.h"
#include "index_fst.h" #include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_tfile.h"
#include "tlockfree.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -29,92 +29,85 @@ extern "C" {
// tfile header // tfile header
// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| // |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->|
// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| // |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->|
typedef struct TFileReadHeader { typedef struct TFileReadHeader {
uint64_t suid; uint64_t suid;
int32_t version; int32_t version;
char colName[128]; // char colName[128]; //
uint8_t colType; uint8_t colType;
} TFileReadHeader; } TFileReadHeader;
#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t)); #define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t));
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) #define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileCacheKey { typedef struct TFileCacheKey {
uint64_t suid; uint64_t suid;
uint8_t colType; uint8_t colType;
int32_t version; int32_t version;
const char *colName; const char *colName;
int32_t nColName; int32_t nColName;
} TFileCacheKey; } TFileCacheKey;
// table cache // table cache
// refactor to LRU cache later // refactor to LRU cache later
typedef struct TFileCache { typedef struct TFileCache {
SHashObj *tableCache; SHashObj *tableCache;
int16_t capacity; int16_t capacity;
// add more param // add more param
} TFileCache; } TFileCache;
typedef struct TFileWriter { typedef struct TFileWriter {
FstBuilder *fb; FstBuilder *fb;
WriterCtx *ctx; WriterCtx * ctx;
} TFileWriter; } TFileWriter;
typedef struct TFileReader { typedef struct TFileReader {
T_REF_DECLARE() T_REF_DECLARE()
Fst *fst; Fst * fst;
WriterCtx *ctx; WriterCtx * ctx;
} TFileReader; TFileReadHeader header;
} TFileReader;
typedef struct IndexTFile { typedef struct IndexTFile {
char *path; char * path;
TFileCache *cache; TFileCache * cache;
TFileWriter *tw; TFileWriter *tw;
} IndexTFile; } IndexTFile;
typedef struct TFileWriterOpt { typedef struct TFileWriterOpt {
uint64_t suid; uint64_t suid;
int8_t colType; int8_t colType;
char *colName; char * colName;
int32_t nColName; int32_t nColName;
int32_t version; int32_t version;
} TFileWriterOpt; } TFileWriterOpt;
typedef struct TFileReaderOpt { typedef struct TFileReaderOpt {
uint64_t suid; uint64_t suid;
char *colName; char * colName;
int32_t nColName; int32_t nColName;
} TFileReaderOpt; } TFileReaderOpt;
// tfile cache, manage tindex reader // tfile cache, manage tindex reader
TFileCache *tfileCacheCreate(const char *path); TFileCache * tfileCacheCreate(const char *path);
void tfileCacheDestroy(TFileCache *tcache); void tfileCacheDestroy(TFileCache *tcache);
TFileReader* tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key);
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader);
TFileReader* tfileReaderCreate();
void TFileReaderDestroy(TFileReader *reader);
TFileReader *tfileReaderCreate();
void TFileReaderDestroy(TFileReader *reader);
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); TFileWriter *tfileWriterCreate(const char *suid, const char *colName);
void tfileWriterDestroy(TFileWriter *tw); void tfileWriterDestroy(TFileWriter *tw);
// //
IndexTFile *indexTFileCreate(const char *path); IndexTFile *indexTFileCreate(const char *path);
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid);
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif #endif

33
source/libs/index/inc/index_util.h
View File

@ -19,33 +19,32 @@
extern "C" { extern "C" {
#endif #endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ #define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \ do { \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \ buf += sizeof(key->mem); \
} while (0) } while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \ do { \
memcpy((void *)buf, (void *)key->mem, len); \ memcpy((void *)buf, (void *)key->mem, len); \
buf += len; \ buf += len; \
} while (0) } while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ #define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \ do { \
type c = var; \ type c = var; \
assert(sizeof(var) == sizeof(type));\ assert(sizeof(var) == sizeof(type)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \ memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \ buf += sizeof(c); \
} while (0) } while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \ do { \
memcpy((void *)buf, (void *)var, len); \ memcpy((void *)buf, (void *)var, len); \
buf += len;\ buf += len; \
} while (0) } while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

280
source/libs/index/src/index.c
View File

@ -26,105 +26,108 @@
static int uidCompare(const void *a, const void *b) { static int uidCompare(const void *a, const void *b) {
uint64_t u1 = *(uint64_t *)a; uint64_t u1 = *(uint64_t *)a;
uint64_t u2 = *(uint64_t *)b; uint64_t u2 = *(uint64_t *)b;
if (u1 == u2) { return 0; } if (u1 == u2) {
else { return u1 < u2 ? -1 : 1; } return 0;
} else {
return u1 < u2 ? -1 : 1;
}
} }
typedef struct SIdxColInfo { typedef struct SIdxColInfo {
int colId; // generated by index internal int colId; // generated by index internal
int cVersion; int cVersion;
} SIdxColInfo; } SIdxColInfo;
static pthread_once_t isInit = PTHREAD_ONCE_INIT; static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit(); static void indexInit();
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result);
static int indexMergeCacheIntoTindex(SIndex *sIdx); static int indexMergeCacheIntoTindex(SIndex *sIdx);
static void indexInterResultsDestroy(SArray *results); static void indexInterResultsDestroy(SArray *results);
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult);
int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
pthread_once(&isInit, indexInit); pthread_once(&isInit, indexInit);
SIndex *sIdx = calloc(1, sizeof(SIndex)); SIndex *sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; } if (sIdx == NULL) {
return -1;
}
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t *index = index_open(path); index_t *index = index_open(path);
sIdx->index = index; sIdx->index = index;
#endif #endif
sIdx->cache = (void*)indexCacheCreate(); sIdx->cache = (void *)indexCacheCreate();
sIdx->tindex = NULL; sIdx->tindex = NULL;
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->colId = 1; sIdx->colId = 1;
sIdx->cVersion = 1; sIdx->cVersion = 1;
pthread_mutex_init(&sIdx->mtx, NULL); pthread_mutex_init(&sIdx->mtx, NULL);
*index = sIdx; *index = sIdx;
return 0; return 0;
} }
void indexClose(SIndex *sIdx) { void indexClose(SIndex *sIdx) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_close(sIdex->index); index_close(sIdex->index);
sIdx->index = NULL; sIdx->index = NULL;
#endif #endif
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
indexCacheDestroy(sIdx->cache); indexCacheDestroy(sIdx->cache);
taosHashCleanup(sIdx->colObj); taosHashCleanup(sIdx->colObj);
pthread_mutex_destroy(&sIdx->mtx); pthread_mutex_destroy(&sIdx->mtx);
#endif #endif
free(sIdx); free(sIdx);
return; return;
} }
int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) { int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
#ifdef USE_LUCENE
#ifdef USE_LUCENE index_document_t *doc = index_document_create();
index_document_t *doc = index_document_create();
char buf[16] = {0}; char buf[16] = {0};
sprintf(buf, "%d", uid); sprintf(buf, "%d", uid);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1);
}
index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
index_put(index->index, doc); for (int i = 0; i < taosArrayGetSize(fVals); i++) {
index_document_destroy(doc); SIndexTerm *p = taosArrayGetP(fVals, i);
index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1);
}
index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
index_put(index->index, doc);
index_document_destroy(doc);
#endif #endif
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
//TODO(yihao): reduce the lock range // TODO(yihao): reduce the lock range
pthread_mutex_lock(&index->mtx); pthread_mutex_lock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i); SIndexTerm * p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
if (fi == NULL) { if (fi == NULL) {
SIdxColInfo tfi = {.colId = index->colId}; SIdxColInfo tfi = {.colId = index->colId};
index->cVersion++; index->cVersion++;
index->colId++; index->colId++;
taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi));
} else { } else {
//TODO, del // TODO, del
} }
} }
pthread_mutex_unlock(&index->mtx); pthread_mutex_unlock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) { for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i); SIndexTerm * p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
assert(fi != NULL); assert(fi != NULL);
int32_t colId = fi->colId; int32_t colId = fi->colId;
int32_t version = index->cVersion; int32_t version = index->cVersion;
int ret = indexCachePut(index->cache, p, colId, version, uid); int ret = indexCachePut(index->cache, p, colId, version, uid);
if (ret != 0) { if (ret != 0) {
return ret; return ret;
} }
} }
#endif #endif
@ -132,29 +135,29 @@ int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) {
return 0; return 0;
} }
int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) {
#ifdef USE_LUCENE #ifdef USE_LUCENE
EIndexOperatorType opera = multiQuerys->opera; EIndexOperatorType opera = multiQuerys->opera;
int nQuery = taosArrayGetSize(multiQuerys->query); int nQuery = taosArrayGetSize(multiQuerys->query);
char **fields = malloc(sizeof(char *) * nQuery); char **fields = malloc(sizeof(char *) * nQuery);
char **keys = malloc(sizeof(char *) * nQuery); char **keys = malloc(sizeof(char *) * nQuery);
int *types = malloc(sizeof(int) * nQuery); int * types = malloc(sizeof(int) * nQuery);
for (int i = 0; i < nQuery; i++) { for (int i = 0; i < nQuery; i++) {
SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i);
SIndexTerm *term = p->field_value; SIndexTerm * term = p->field_value;
fields[i] = calloc(1, term->nKey + 1); fields[i] = calloc(1, term->nKey + 1);
keys[i] = calloc(1, term->nVal + 1); keys[i] = calloc(1, term->nVal + 1);
memcpy(fields[i], term->key, term->nKey); memcpy(fields[i], term->key, term->nKey);
memcpy(keys[i], term->val, term->nVal); memcpy(keys[i], term->val, term->nVal);
types[i] = (int)(p->type); types[i] = (int)(p->type);
} }
int *tResult = NULL; int *tResult = NULL;
int tsz= 0; int tsz = 0;
index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz); index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz);
for (int i = 0; i < tsz; i++) { for (int i = 0; i < tsz; i++) {
taosArrayPush(result, &tResult[i]); taosArrayPush(result, &tResult[i]);
} }
@ -169,57 +172,55 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
#endif #endif
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
EIndexOperatorType opera = multiQuerys->opera; // relation of querys EIndexOperatorType opera = multiQuerys->opera; // relation of querys
SArray *interResults = taosArrayInit(4, POINTER_BYTES); SArray *interResults = taosArrayInit(4, POINTER_BYTES);
int nQuery = taosArrayGetSize(multiQuerys->query); int nQuery = taosArrayGetSize(multiQuerys->query);
for (size_t i = 0; i < nQuery; i++) { for (size_t i = 0; i < nQuery; i++) {
SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i);
SArray *tResult = NULL; SArray * tResult = NULL;
indexTermSearch(index, qTerm, &tResult); indexTermSearch(index, qTerm, &tResult);
taosArrayPush(interResults, (void *)&tResult); taosArrayPush(interResults, (void *)&tResult);
} }
indexMergeFinalResults(interResults, opera, result); indexMergeFinalResults(interResults, opera, result);
indexInterResultsDestroy(interResults); indexInterResultsDestroy(interResults);
#endif #endif
return 1; return 1;
} }
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
#endif #endif
return 1; return 1;
} }
int indexRebuild(SIndex *index, SIndexOpts *opts) { int indexRebuild(SIndex *index, SIndexOpts *opts){
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
#endif #endif
} }
SIndexOpts *indexOptsCreate() { SIndexOpts *indexOptsCreate() {
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
return NULL; return NULL;
} }
void indexOptsDestroy(SIndexOpts *opts) { void indexOptsDestroy(SIndexOpts *opts){
#ifdef USE_LUCENE #ifdef USE_LUCENE
#endif #endif
} } /*
/* * @param: oper
* @param: oper *
* */
*/
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; } if (p == NULL) {
p->opera = opera; return NULL;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); }
p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p; return p;
} }
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) {
@ -227,25 +228,27 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) {
SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i);
indexTermDestroy(p->term); indexTermDestroy(p->term);
} }
taosArrayDestroy(pQuery->query); taosArrayDestroy(pQuery->query);
free(pQuery); free(pQuery);
}; };
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){ int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType) {
SIndexTermQuery q = {.qType = qType, .term = term}; SIndexTermQuery q = {.qType = qType, .term = term};
taosArrayPush(pQuery->query, &q); taosArrayPush(pQuery->query, &q);
return 0; return 0;
} }
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName,
int32_t nColName, const char *colVal, int32_t nColVal) {
SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) {
return NULL;
}
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) { t->suid = suid;
SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); t->operType = oper;
if (t == NULL) { return NULL; }
t->suid = suid;
t->operType= oper;
t->colType = colType; t->colType = colType;
t->colName = (char *)calloc(1, nColName + 1); t->colName = (char *)calloc(1, nColName + 1);
memcpy(t->colName, colName, nColName); memcpy(t->colName, colName, nColName);
t->nColName = nColName; t->nColName = nColName;
@ -258,15 +261,13 @@ void indexTermDestroy(SIndexTerm *p) {
free(p->colName); free(p->colName);
free(p->colVal); free(p->colVal);
free(p); free(p);
}
SIndexMultiTerm *indexMultiTermCreate() {
return taosArrayInit(4, sizeof(SIndexTerm *));
} }
SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); }
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) {
taosArrayPush(terms, &term); taosArrayPush(terms, &term);
return 0; return 0;
} }
void indexMultiTermDestroy(SIndexMultiTerm *terms) { void indexMultiTermDestroy(SIndexMultiTerm *terms) {
for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) {
@ -277,40 +278,40 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) {
} }
void indexInit() { void indexInit() {
//do nothing // do nothing
} }
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) {
int32_t version = -1; int32_t version = -1;
int16_t colId = -1; int16_t colId = -1;
SIdxColInfo *colInfo = NULL; SIdxColInfo *colInfo = NULL;
SIndexTerm *term = query->term; SIndexTerm *term = query->term;
const char *colName = term->colName; const char *colName = term->colName;
int32_t nColName = term->nColName; int32_t nColName = term->nColName;
pthread_mutex_lock(&sIdx->mtx); pthread_mutex_lock(&sIdx->mtx);
colInfo = taosHashGet(sIdx->colObj, colName, nColName); colInfo = taosHashGet(sIdx->colObj, colName, nColName);
if (colInfo == NULL) { if (colInfo == NULL) {
pthread_mutex_unlock(&sIdx->mtx); pthread_mutex_unlock(&sIdx->mtx);
return -1; return -1;
} }
colId = colInfo->colId; colId = colInfo->colId;
version = colInfo->cVersion; version = colInfo->cVersion;
pthread_mutex_unlock(&sIdx->mtx); pthread_mutex_unlock(&sIdx->mtx);
*result = taosArrayInit(4, sizeof(uint64_t)); *result = taosArrayInit(4, sizeof(uint64_t));
//TODO: iterator mem and tidex // TODO: iterator mem and tidex
STermValueType s; STermValueType s;
if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) { if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) {
if (s == kTypeDeletion) { if (s == kTypeDeletion) {
indexInfo("col: %s already drop by other opera", term->colName); indexInfo("col: %s already drop by other opera", term->colName);
// coloum already drop by other oper, no need to query tindex // coloum already drop by other oper, no need to query tindex
return 0; return 0;
} else { } else {
if (0 != indexTFileSearch(sIdx->tindex, query, *result)) { if (0 != indexTFileSearch(sIdx->tindex, query, *result)) {
indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal);
return -1; return -1;
} }
} }
} else { } else {
indexError("corrupt at index(cache) col:%s val: %s", term->colName, term->colVal); indexError("corrupt at index(cache) col:%s val: %s", term->colName, term->colVal);
@ -319,39 +320,40 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
return 0; return 0;
} }
static void indexInterResultsDestroy(SArray *results) { static void indexInterResultsDestroy(SArray *results) {
if (results == NULL) { return; } if (results == NULL) {
return;
}
size_t sz = taosArrayGetSize(results); size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
SArray *p = taosArrayGetP(results, i); SArray *p = taosArrayGetP(results, i);
taosArrayDestroy(p); taosArrayDestroy(p);
} }
taosArrayDestroy(results); taosArrayDestroy(results);
} }
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) {
//refactor, merge interResults into fResults by oType // refactor, merge interResults into fResults by oType
SArray *first = taosArrayGetP(interResults, 0); SArray *first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare); taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL); taosArrayRemoveDuplicate(first, uidCompare, NULL);
if (oType == MUST) { if (oType == MUST) {
// just one column index, enhance later // just one column index, enhance later
taosArrayAddAll(fResults, first); taosArrayAddAll(fResults, first);
} else if (oType == SHOULD) { } else if (oType == SHOULD) {
// just one column index, enhance later // just one column index, enhance later
taosArrayAddAll(fResults, first); taosArrayAddAll(fResults, first);
// tag1 condistion || tag2 condition // tag1 condistion || tag2 condition
} else if (oType == NOT) { } else if (oType == NOT) {
// just one column index, enhance later // just one column index, enhance later
taosArrayAddAll(fResults, first); taosArrayAddAll(fResults, first);
// not use currently // not use currently
} }
return 0; return 0;
} }
static int indexMergeCacheIntoTindex(SIndex *sIdx) { static int indexMergeCacheIntoTindex(SIndex *sIdx) {
if (sIdx == NULL) { if (sIdx == NULL) {
return -1; return -1;
} }
indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
return 0; return 0;
} }

126
source/libs/index/src/index_cache.c
View File

@ -14,148 +14,154 @@
*/ */
#include "index_cache.h" #include "index_cache.h"
#include "tcompare.h"
#include "index_util.h" #include "index_util.h"
#include "tcompare.h"
#define MAX_INDEX_KEY_LEN 256// test only, change later #define MAX_INDEX_KEY_LEN 256 // test only, change later
// ref index_cache.h:22 // ref index_cache.h:22
#define CACHE_KEY_LEN(p) (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) #define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \
sizeof(p->operType))
static char* getIndexKey(const void *pData) { static char * getIndexKey(const void *pData) { return NULL; }
return NULL;
}
static int32_t compareKey(const void *l, const void *r) { static int32_t compareKey(const void *l, const void *r) {
char *lp = (char *)l; char *lp = (char *)l;
char *rp = (char *)r; char *rp = (char *)r;
// skip total len, not compare // skip total len, not compare
int32_t ll, rl; // len int32_t ll, rl; // len
memcpy(&ll, lp, sizeof(int32_t)); memcpy(&ll, lp, sizeof(int32_t));
memcpy(&rl, rp, sizeof(int32_t)); memcpy(&rl, rp, sizeof(int32_t));
lp += sizeof(int32_t); lp += sizeof(int32_t);
rp += sizeof(int32_t); rp += sizeof(int32_t);
// compare field id // compare field id
int16_t lf, rf; // field id int16_t lf, rf; // field id
memcpy(&lf, lp, sizeof(lf)); memcpy(&lf, lp, sizeof(lf));
memcpy(&rf, rp, sizeof(rf)); memcpy(&rf, rp, sizeof(rf));
if (lf != rf) { if (lf != rf) {
return lf < rf ? -1: 1; return lf < rf ? -1 : 1;
} }
lp += sizeof(lf); lp += sizeof(lf);
rp += sizeof(rf); rp += sizeof(rf);
// compare field type // compare field type
int8_t lft, rft; int8_t lft, rft;
memcpy(&lft, lp, sizeof(lft)); memcpy(&lft, lp, sizeof(lft));
memcpy(&rft, rp, sizeof(rft)); memcpy(&rft, rp, sizeof(rft));
lp += sizeof(lft); lp += sizeof(lft);
rp += sizeof(rft); rp += sizeof(rft);
assert(rft == rft); assert(rft == rft);
// skip value len // skip value len
int32_t lfl, rfl; int32_t lfl, rfl;
memcpy(&lfl, lp, sizeof(lfl)); memcpy(&lfl, lp, sizeof(lfl));
memcpy(&rfl, rp, sizeof(rfl)); memcpy(&rfl, rp, sizeof(rfl));
lp += sizeof(lfl); lp += sizeof(lfl);
rp += sizeof(rfl); rp += sizeof(rfl);
// compare value // compare value
int32_t i, j; int32_t i, j;
for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) {
if (lp[i] == rp[j]) { continue; } if (lp[i] == rp[j]) {
else { return lp[i] < rp[j] ? -1 : 1;} continue;
} else {
return lp[i] < rp[j] ? -1 : 1;
}
}
if (i < lfl) {
return 1;
} else if (j < rfl) {
return -1;
} }
if (i < lfl) { return 1;}
else if (j < rfl) { return -1; }
lp += lfl; lp += lfl;
rp += rfl; rp += rfl;
// skip uid // skip uid
uint64_t lu, ru; uint64_t lu, ru;
memcpy(&lu, lp, sizeof(lu)); memcpy(&lu, lp, sizeof(lu));
memcpy(&ru, rp, sizeof(ru)); memcpy(&ru, rp, sizeof(ru));
lp += sizeof(lu); lp += sizeof(lu);
rp += sizeof(ru); rp += sizeof(ru);
// compare version, desc order // compare version, desc order
int32_t lv, rv; int32_t lv, rv;
memcpy(&lv, lp, sizeof(lv)); memcpy(&lv, lp, sizeof(lv));
memcpy(&rv, rp, sizeof(rv)); memcpy(&rv, rp, sizeof(rv));
if (lv != rv) { if (lv != rv) {
return lv > rv ? -1 : 1; return lv > rv ? -1 : 1;
} }
lp += sizeof(lv); lp += sizeof(lv);
rp += sizeof(rv); rp += sizeof(rv);
// not care item type // not care item type
return 0; return 0;
}
}
IndexCache *indexCacheCreate() { IndexCache *indexCacheCreate() {
IndexCache *cache = calloc(1, sizeof(IndexCache)); IndexCache *cache = calloc(1, sizeof(IndexCache));
cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); cache->skiplist = tSkipListCreate(
MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
return cache; return cache;
} }
void indexCacheDestroy(void *cache) { void indexCacheDestroy(void *cache) {
IndexCache *pCache = cache; IndexCache *pCache = cache;
if (pCache == NULL) { return; } if (pCache == NULL) {
return;
}
tSkipListDestroy(pCache->skiplist); tSkipListDestroy(pCache->skiplist);
free(pCache); free(pCache);
} }
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) {
if (cache == NULL) { return -1;} if (cache == NULL) {
return -1;
}
IndexCache *pCache = cache; IndexCache *pCache = cache;
// encode data // encode data
int32_t total = CACHE_KEY_LEN(term); int32_t total = CACHE_KEY_LEN(term);
char *buf = calloc(1, total); char * buf = calloc(1, total);
char *p = buf; char * p = buf;
SERIALIZE_VAR_TO_BUF(p, total,int32_t); SERIALIZE_VAR_TO_BUF(p, total, int32_t);
SERIALIZE_VAR_TO_BUF(p, colId, int16_t); SERIALIZE_VAR_TO_BUF(p, colId, int16_t);
SERIALIZE_MEM_TO_BUF(p, term, colType); SERIALIZE_MEM_TO_BUF(p, term, colType);
SERIALIZE_MEM_TO_BUF(p, term, nColVal); SERIALIZE_MEM_TO_BUF(p, term, nColVal);
SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal); SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal);
SERIALIZE_VAR_TO_BUF(p, version, int32_t); SERIALIZE_VAR_TO_BUF(p, version, int32_t);
SERIALIZE_VAR_TO_BUF(p, uid, uint64_t); SERIALIZE_VAR_TO_BUF(p, uid, uint64_t);
SERIALIZE_MEM_TO_BUF(p, term, operType); SERIALIZE_MEM_TO_BUF(p, term, operType);
tSkipListPut(pCache->skiplist, (void *)buf); tSkipListPut(pCache->skiplist, (void *)buf);
return 0; return 0;
// encode end // encode end
} }
int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
IndexCache *pCache = cache; IndexCache *pCache = cache;
return 0; return 0;
} }
int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { int indexCacheSearch(
if (cache == NULL) { return -1; } void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) {
IndexCache *pCache = cache; if (cache == NULL) {
SIndexTerm *term = query->term; return -1;
EIndexQueryType qtype = query->qType; }
IndexCache * pCache = cache;
int32_t keyLen = CACHE_KEY_LEN(term); SIndexTerm * term = query->term;
EIndexQueryType qtype = query->qType;
int32_t keyLen = CACHE_KEY_LEN(term);
char *buf = calloc(1, keyLen); char *buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) { if (qtype == QUERY_TERM) {
} else if (qtype == QUERY_PREFIX) { } else if (qtype == QUERY_PREFIX) {
} else if (qtype == QUERY_SUFFIX) { } else if (qtype == QUERY_SUFFIX) {
} else if (qtype == QUERY_REGEX) { } else if (qtype == QUERY_REGEX) {
} }
return 0; return 0;
} }

1308
source/libs/index/src/index_fst.c
View File

File diff suppressed because it is too large Load Diff

132
source/libs/index/src/index_fst_automation.c
View File

@ -15,44 +15,49 @@
#include "index_fst_automation.h" #include "index_fst_automation.h"
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; } if (nsv == NULL) {
return NULL;
}
nsv->kind = kind; nsv->kind = kind;
nsv->type = ty; nsv->type = ty;
if (ty == FST_INT) { if (ty == FST_INT) {
nsv->val = *(int *)val; nsv->val = *(int *)val;
} else if (ty == FST_CHAR) { } else if (ty == FST_CHAR) {
size_t len = strlen((char *)val); size_t len = strlen((char *)val);
nsv->ptr = (char *)calloc(1, len + 1); nsv->ptr = (char *)calloc(1, len + 1);
memcpy(nsv->ptr, val, len); memcpy(nsv->ptr, val, len);
} else if (ty == FST_ARRAY) { } else if (ty == FST_ARRAY) {
//TODO, // TODO,
//nsv->arr = taosArrayFromList() // nsv->arr = taosArrayFromList()
} }
return nsv; return nsv;
} }
void startWithStateValueDestroy(void *val) { void startWithStateValueDestroy(void *val) {
StartWithStateValue *sv = (StartWithStateValue *)val; StartWithStateValue *sv = (StartWithStateValue *)val;
if (sv == NULL) { return; } if (sv == NULL) {
return;
}
if (sv->type == FST_INT) { if (sv->type == FST_INT) {
// //
} else if (sv->type == FST_CHAR) { } else if (sv->type == FST_CHAR) {
free(sv->ptr); free(sv->ptr);
} else if (sv->type == FST_ARRAY) { } else if (sv->type == FST_ARRAY) {
taosArrayDestroy(sv->arr); taosArrayDestroy(sv->arr);
} }
free(sv); free(sv);
} }
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; } if (nsv == NULL) {
return NULL;
}
nsv->kind = sv->kind; nsv->kind = sv->kind;
nsv->type= sv->type; nsv->type = sv->type;
if (nsv->type == FST_INT) { if (nsv->type == FST_INT) {
nsv->val = sv->val; nsv->val = sv->val;
} else if (nsv->type == FST_CHAR) { } else if (nsv->type == FST_CHAR) {
@ -64,93 +69,67 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
return nsv; return nsv;
} }
// prefix query, impl later // prefix query, impl later
static void* prefixStart(AutomationCtx *ctx) { static void *prefixStart(AutomationCtx *ctx) {
StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata);
return startWithStateValueDump(data); return startWithStateValueDump(data);
}; };
static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { static bool prefixIsMatch(AutomationCtx *ctx, void *sv) {
StartWithStateValue* ssv = (StartWithStateValue *)sv; StartWithStateValue *ssv = (StartWithStateValue *)sv;
return ssv->val == strlen(ctx->data); return ssv->val == strlen(ctx->data);
} }
static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { static bool prefixCanMatch(AutomationCtx *ctx, void *sv) {
StartWithStateValue* ssv = (StartWithStateValue *)sv; StartWithStateValue *ssv = (StartWithStateValue *)sv;
return ssv->val >= 0; return ssv->val >= 0;
} }
static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; }
return true; static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
} StartWithStateValue *ssv = (StartWithStateValue *)state;
static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { if (ssv == NULL || ctx == NULL) {
StartWithStateValue* ssv = (StartWithStateValue *)state; return NULL;
if (ssv == NULL || ctx == NULL) {return NULL;} }
char *data = ctx->data; char *data = ctx->data;
if (ssv->kind == Done) { if (ssv->kind == Done) {
return startWithStateValueCreate(Done, FST_INT, &ssv->val); return startWithStateValueCreate(Done, FST_INT, &ssv->val);
} }
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1; int val = ssv->val + 1;
StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val);
if (prefixIsMatch(ctx, nsv)) { if (prefixIsMatch(ctx, nsv)) {
nsv->kind = Done; nsv->kind = Done;
} else { } else {
nsv->kind = Running; nsv->kind = Running;
} }
return nsv; return nsv;
} }
return NULL;
}
static void* prefixAcceptEof(AutomationCtx *ctx, void *state) {
return NULL; return NULL;
} }
static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; }
// pattern query, impl later // pattern query, impl later
static void* patternStart(AutomationCtx *ctx) { static void *patternStart(AutomationCtx *ctx) { return NULL; }
return NULL; static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; }
} static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; }
static bool patternIsMatch(AutomationCtx *ctx, void *data) { static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; }
return true;
}
static bool patternCanMatch(AutomationCtx *ctx, void *data) {
return true;
}
static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) {
return true;
}
static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; }
return NULL;
}
static void* patternAcceptEof(AutomationCtx *ctx, void *state) { static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; }
return NULL;
}
AutomationFunc automFuncs[] = {{ AutomationFunc automFuncs[] = {
prefixStart, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof},
prefixIsMatch, {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof}
prefixCanMatch, // add more search type
prefixWillAlwaysMatch,
prefixAccept,
prefixAcceptEof
},
{
patternStart,
patternIsMatch,
patternCanMatch,
patternWillAlwaysMatch,
patternAccept,
patternAcceptEof
}
// add more search type
}; };
AutomationCtx* automCtxCreate(void *data,AutomationType atype) { AutomationCtx *automCtxCreate(void *data, AutomationType atype) {
AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) {
return NULL;
}
StartWithStateValue *sv = NULL; StartWithStateValue *sv = NULL;
if (atype == AUTOMATION_PREFIX) { if (atype == AUTOMATION_PREFIX) {
@ -158,22 +137,21 @@ AutomationCtx* automCtxCreate(void *data,AutomationType atype) {
sv = startWithStateValueCreate(Running, FST_INT, &val); sv = startWithStateValueCreate(Running, FST_INT, &val);
ctx->stdata = (void *)sv; ctx->stdata = (void *)sv;
} else if (atype == AUTMMATION_MATCH) { } else if (atype == AUTMMATION_MATCH) {
} else { } else {
// add more search type // add more search type
} }
char* src = (char *)data; char * src = (char *)data;
size_t len = strlen(src); size_t len = strlen(src);
char* dst = (char *)malloc(len * sizeof(char) + 1); char * dst = (char *)malloc(len * sizeof(char) + 1);
memcpy(dst, src, len); memcpy(dst, src, len);
dst[len] = 0; dst[len] = 0;
ctx->data = dst; ctx->data = dst;
ctx->type = atype; ctx->type = atype;
ctx->stdata = (void *)sv; ctx->stdata = (void *)sv;
return ctx; return ctx;
} }
void automCtxDestroy(AutomationCtx *ctx) { void automCtxDestroy(AutomationCtx *ctx) {
startWithStateValueDestroy(ctx->stdata); startWithStateValueDestroy(ctx->stdata);
free(ctx->data); free(ctx->data);

800
source/libs/index/src/index_fst_common.c
View File

@ -12,296 +12,522 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "index_fst_common.h"
#include "tutil.h"
const uint8_t COMMON_INPUTS[] = { const uint8_t COMMON_INPUTS[] = {
84, // '\x00' 84, // '\x00'
85, // '\x01' 85, // '\x01'
86, // '\x02' 86, // '\x02'
87, // '\x03' 87, // '\x03'
88, // '\x04' 88, // '\x04'
89, // '\x05' 89, // '\x05'
90, // '\x06' 90, // '\x06'
91, // '\x07' 91, // '\x07'
92, // '\x08' 92, // '\x08'
93, // '\t' 93, // '\t'
94, // '\n' 94, // '\n'
95, // '\x0b' 95, // '\x0b'
96, // '\x0c' 96, // '\x0c'
97, // '\r' 97, // '\r'
98, // '\x0e' 98, // '\x0e'
99, // '\x0f' 99, // '\x0f'
100, // '\x10' 100, // '\x10'
101, // '\x11' 101, // '\x11'
102, // '\x12' 102, // '\x12'
103, // '\x13' 103, // '\x13'
104, // '\x14' 104, // '\x14'
105, // '\x15' 105, // '\x15'
106, // '\x16' 106, // '\x16'
107, // '\x17' 107, // '\x17'
108, // '\x18' 108, // '\x18'
109, // '\x19' 109, // '\x19'
110, // '\x1a' 110, // '\x1a'
111, // '\x1b' 111, // '\x1b'
112, // '\x1c' 112, // '\x1c'
113, // '\x1d' 113, // '\x1d'
114, // '\x1e' 114, // '\x1e'
115, // '\x1f' 115, // '\x1f'
116, // ' ' 116, // ' '
80, // '!' 80, // '!'
117, // '"' 117, // '"'
118, // '#' 118, // '#'
79, // '$' 79, // '$'
39, // '%' 39, // '%'
30, // '&' 30, // '&'
81, // "'" 81, // "'"
75, // '(' 75, // '('
74, // ')' 74, // ')'
82, // '*' 82, // '*'
57, // '+' 57, // '+'
66, // ',' 66, // ','
16, // '-' 16, // '-'
12, // '.' 12, // '.'
2, // '/' 2, // '/'
19, // '0' 19, // '0'
20, // '1' 20, // '1'
21, // '2' 21, // '2'
27, // '3' 27, // '3'
32, // '4' 32, // '4'
29, // '5' 29, // '5'
35, // '6' 35, // '6'
36, // '7' 36, // '7'
37, // '8' 37, // '8'
34, // '9' 34, // '9'
24, // ':' 24, // ':'
73, // ';' 73, // ';'
119, // '<' 119, // '<'
23, // '=' 23, // '='
120, // '>' 120, // '>'
40, // '?' 40, // '?'
83, // '@' 83, // '@'
44, // 'A' 44, // 'A'
48, // 'B' 48, // 'B'
42, // 'C' 42, // 'C'
43, // 'D' 43, // 'D'
49, // 'E' 49, // 'E'
46, // 'F' 46, // 'F'
62, // 'G' 62, // 'G'
61, // 'H' 61, // 'H'
47, // 'I' 47, // 'I'
69, // 'J' 69, // 'J'
68, // 'K' 68, // 'K'
58, // 'L' 58, // 'L'
56, // 'M' 56, // 'M'
55, // 'N' 55, // 'N'
59, // 'O' 59, // 'O'
51, // 'P' 51, // 'P'
72, // 'Q' 72, // 'Q'
54, // 'R' 54, // 'R'
45, // 'S' 45, // 'S'
52, // 'T' 52, // 'T'
64, // 'U' 64, // 'U'
65, // 'V' 65, // 'V'
63, // 'W' 63, // 'W'
71, // 'X' 71, // 'X'
67, // 'Y' 67, // 'Y'
70, // 'Z' 70, // 'Z'
77, // '[' 77, // '['
121, // '\\' 121, // '\\'
78, // ']' 78, // ']'
122, // '^' 122, // '^'
31, // '_' 31, // '_'
123, // '`' 123, // '`'
4, // 'a' 4, // 'a'
25, // 'b' 25, // 'b'
9, // 'c' 9, // 'c'
17, // 'd' 17, // 'd'
1, // 'e' 1, // 'e'
26, // 'f' 26, // 'f'
22, // 'g' 22, // 'g'
13, // 'h' 13, // 'h'
7, // 'i' 7, // 'i'
50, // 'j' 50, // 'j'
38, // 'k' 38, // 'k'
14, // 'l' 14, // 'l'
15, // 'm' 15, // 'm'
10, // 'n' 10, // 'n'
3, // 'o' 3, // 'o'
8, // 'p' 8, // 'p'
60, // 'q' 60, // 'q'
6, // 'r' 6, // 'r'
5, // 's' 5, // 's'
0, // 't' 0, // 't'
18, // 'u' 18, // 'u'
33, // 'v' 33, // 'v'
11, // 'w' 11, // 'w'
41, // 'x' 41, // 'x'
28, // 'y' 28, // 'y'
53, // 'z' 53, // 'z'
124, // '{' 124, // '{'
125, // '|' 125, // '|'
126, // '}' 126, // '}'
76, // '~' 76, // '~'
127, // '\x7f' 127, // '\x7f'
128, // '\x80' 128, // '\x80'
129, // '\x81' 129, // '\x81'
130, // '\x82' 130, // '\x82'
131, // '\x83' 131, // '\x83'
132, // '\x84' 132, // '\x84'
133, // '\x85' 133, // '\x85'
134, // '\x86' 134, // '\x86'
135, // '\x87' 135, // '\x87'
136, // '\x88' 136, // '\x88'
137, // '\x89' 137, // '\x89'
138, // '\x8a' 138, // '\x8a'
139, // '\x8b' 139, // '\x8b'
140, // '\x8c' 140, // '\x8c'
141, // '\x8d' 141, // '\x8d'
142, // '\x8e' 142, // '\x8e'
143, // '\x8f' 143, // '\x8f'
144, // '\x90' 144, // '\x90'
145, // '\x91' 145, // '\x91'
146, // '\x92' 146, // '\x92'
147, // '\x93' 147, // '\x93'
148, // '\x94' 148, // '\x94'
149, // '\x95' 149, // '\x95'
150, // '\x96' 150, // '\x96'
151, // '\x97' 151, // '\x97'
152, // '\x98' 152, // '\x98'
153, // '\x99' 153, // '\x99'
154, // '\x9a' 154, // '\x9a'
155, // '\x9b' 155, // '\x9b'
156, // '\x9c' 156, // '\x9c'
157, // '\x9d' 157, // '\x9d'
158, // '\x9e' 158, // '\x9e'
159, // '\x9f' 159, // '\x9f'
160, // '\xa0' 160, // '\xa0'
161, // '¡' 161, // '¡'
162, // '¢' 162, // '¢'
163, // '£' 163, // '£'
164, // '¤' 164, // '¤'
165, // '¥' 165, // '¥'
166, // '¦' 166, // '¦'
167, // '§' 167, // '§'
168, // '¨' 168, // '¨'
169, // '©' 169, // '©'
170, // 'ª' 170, // 'ª'
171, // '«' 171, // '«'
172, // '¬' 172, // '¬'
173, // '\xad' 173, // '\xad'
174, // '®' 174, // '®'
175, // '¯' 175, // '¯'
176, // '°' 176, // '°'
177, // '±' 177, // '±'
178, // '²' 178, // '²'
179, // '³' 179, // '³'
180, // '´' 180, // '´'
181, // 'µ' 181, // 'µ'
182, // '¶' 182, // '¶'
183, // '·' 183, // '·'
184, // '¸' 184, // '¸'
185, // '¹' 185, // '¹'
186, // 'º' 186, // 'º'
187, // '»' 187, // '»'
188, // '¼' 188, // '¼'
189, // '½' 189, // '½'
190, // '¾' 190, // '¾'
191, // '¿' 191, // '¿'
192, // 'À' 192, // 'À'
193, // 'Á' 193, // 'Á'
194, // 'Â' 194, // 'Â'
195, // 'Ã' 195, // 'Ã'
196, // 'Ä' 196, // 'Ä'
197, // 'Å' 197, // 'Å'
198, // 'Æ' 198, // 'Æ'
199, // 'Ç' 199, // 'Ç'
200, // 'È' 200, // 'È'
201, // 'É' 201, // 'É'
202, // 'Ê' 202, // 'Ê'
203, // 'Ë' 203, // 'Ë'
204, // 'Ì' 204, // 'Ì'
205, // 'Í' 205, // 'Í'
206, // 'Î' 206, // 'Î'
207, // 'Ï' 207, // 'Ï'
208, // 'Ð' 208, // 'Ð'
209, // 'Ñ' 209, // 'Ñ'
210, // 'Ò' 210, // 'Ò'
211, // 'Ó' 211, // 'Ó'
212, // 'Ô' 212, // 'Ô'
213, // 'Õ' 213, // 'Õ'
214, // 'Ö' 214, // 'Ö'
215, // '×' 215, // '×'
216, // 'Ø' 216, // 'Ø'
217, // 'Ù' 217, // 'Ù'
218, // 'Ú' 218, // 'Ú'
219, // 'Û' 219, // 'Û'
220, // 'Ü' 220, // 'Ü'
221, // 'Ý' 221, // 'Ý'
222, // 'Þ' 222, // 'Þ'
223, // 'ß' 223, // 'ß'
224, // 'à' 224, // 'à'
225, // 'á' 225, // 'á'
226, // 'â' 226, // 'â'
227, // 'ã' 227, // 'ã'
228, // 'ä' 228, // 'ä'
229, // 'å' 229, // 'å'
230, // 'æ' 230, // 'æ'
231, // 'ç' 231, // 'ç'
232, // 'è' 232, // 'è'
233, // 'é' 233, // 'é'
234, // 'ê' 234, // 'ê'
235, // 'ë' 235, // 'ë'
236, // 'ì' 236, // 'ì'
237, // 'í' 237, // 'í'
238, // 'î' 238, // 'î'
239, // 'ï' 239, // 'ï'
240, // 'ð' 240, // 'ð'
241, // 'ñ' 241, // 'ñ'
242, // 'ò' 242, // 'ò'
243, // 'ó' 243, // 'ó'
244, // 'ô' 244, // 'ô'
245, // 'õ' 245, // 'õ'
246, // 'ö' 246, // 'ö'
247, // '÷' 247, // '÷'
248, // 'ø' 248, // 'ø'
249, // 'ù' 249, // 'ù'
250, // 'ú' 250, // 'ú'
251, // 'û' 251, // 'û'
252, // 'ü' 252, // 'ü'
253, // 'ý' 253, // 'ý'
254, // 'þ' 254, // 'þ'
255, // 'ÿ' 255, // 'ÿ'
}; };
const char COMMON_INPUTS_INV[] = { const char COMMON_INPUTS_INV[] = {
't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', 't',
'.', 'h', 'l', 'm', '-', 'd', 'u', '0', '1', '2', 'g', '=', 'e',
':', 'b', 'f', '3', 'y', '5', '&', '_', '4', 'v', '9', '6', '/',
'7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', 'o',
'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'a',
'q', 'H', 'G', 'W', 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 's',
'Q', ';', ')', '(', '~', '[', ']', '$', '!', '\'', '*', '@', 'r',
'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', 'i',
'\x08', '\t', '\n', '\x0b', '\x0c', '\r', '\x0e', '\x0f', '\x10', 'p',
'\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', 'c',
'\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', 'n',
'#', '<', '>', '\\', '^', '`', '{', '|', '}','\x7f','\x80', 'w',
'\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '.',
'\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', '\x90', 'h',
'\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', 'l',
'\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', '\xa0', 'm',
'\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '-',
'\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', '\xb0', 'd',
'\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', 'u',
'\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', '\xc0', '0',
'\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '1',
'\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', '\xd0', '2',
'\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', 'g',
'\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', '\xe0', '=',
'\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', ':',
'\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', '\xf0', 'b',
'\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', 'f',
'\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', '3',
'y',
'5',
'&',
'_',
'4',
'v',
'9',
'6',
'7',
'8',
'k',
'%',
'?',
'x',
'C',
'D',
'A',
'S',
'F',
'I',
'B',
'E',
'j',
'P',
'T',
'z',
'R',
'N',
'M',
'+',
'L',
'O',
'q',
'H',
'G',
'W',
'U',
'V',
',',
'Y',
'K',
'J',
'Z',
'X',
'Q',
';',
')',
'(',
'~',
'[',
']',
'$',
'!',
'\'',
'*',
'@',
'\x00',
'\x01',
'\x02',
'\x03',
'\x04',
'\x05',
'\x06',
'\x07',
'\x08',
'\t',
'\n',
'\x0b',
'\x0c',
'\r',
'\x0e',
'\x0f',
'\x10',
'\x11',
'\x12',
'\x13',
'\x14',
'\x15',
'\x16',
'\x17',
'\x18',
'\x19',
'\x1a',
'\x1b',
'\x1c',
'\x1d',
'\x1e',
'\x1f',
' ',
'"',
'#',
'<',
'>',
'\\',
'^',
'`',
'{',
'|',
'}',
'\x7f',
'\x80',
'\x81',
'\x82',
'\x83',
'\x84',
'\x85',
'\x86',
'\x87',
'\x88',
'\x89',
'\x8a',
'\x8b',
'\x8c',
'\x8d',
'\x8e',
'\x8f',
'\x90',
'\x91',
'\x92',
'\x93',
'\x94',
'\x95',
'\x96',
'\x97',
'\x98',
'\x99',
'\x9a',
'\x9b',
'\x9c',
'\x9d',
'\x9e',
'\x9f',
'\xa0',
'\xa1',
'\xa2',
'\xa3',
'\xa4',
'\xa5',
'\xa6',
'\xa7',
'\xa8',
'\xa9',
'\xaa',
'\xab',
'\xac',
'\xad',
'\xae',
'\xaf',
'\xb0',
'\xb1',
'\xb2',
'\xb3',
'\xb4',
'\xb5',
'\xb6',
'\xb7',
'\xb8',
'\xb9',
'\xba',
'\xbb',
'\xbc',
'\xbd',
'\xbe',
'\xbf',
'\xc0',
'\xc1',
'\xc2',
'\xc3',
'\xc4',
'\xc5',
'\xc6',
'\xc7',
'\xc8',
'\xc9',
'\xca',
'\xcb',
'\xcc',
'\xcd',
'\xce',
'\xcf',
'\xd0',
'\xd1',
'\xd2',
'\xd3',
'\xd4',
'\xd5',
'\xd6',
'\xd7',
'\xd8',
'\xd9',
'\xda',
'\xdb',
'\xdc',
'\xdd',
'\xde',
'\xdf',
'\xe0',
'\xe1',
'\xe2',
'\xe3',
'\xe4',
'\xe5',
'\xe6',
'\xe7',
'\xe8',
'\xe9',
'\xea',
'\xeb',
'\xec',
'\xed',
'\xee',
'\xef',
'\xf0',
'\xf1',
'\xf2',
'\xf3',
'\xf4',
'\xf5',
'\xf6',
'\xf7',
'\xf8',
'\xf9',
'\xfa',
'\xfb',
'\xfc',
'\xfd',
'\xfe',
'\xff',
}; };

110
source/libs/index/src/index_fst_counting_writer.c
View File

@ -12,10 +12,10 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tutil.h" #include "index_fst_counting_writer.h"
#include "indexInt.h" #include "indexInt.h"
#include "index_fst_util.h" #include "index_fst_util.h"
#include "index_fst_counting_writer.h" #include "tutil.h"
static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
if (ctx->offset + len > ctx->limit) { if (ctx->offset + len > ctx->limit) {
@ -25,13 +25,13 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
if (ctx->type == TFile) { if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len)); assert(len == tfWrite(ctx->file.fd, buf, len));
} else { } else {
memcpy(ctx->mem.buf+ ctx->offset, buf, len); memcpy(ctx->mem.buf + ctx->offset, buf, len);
} }
ctx->offset += len; ctx->offset += len;
return len; return len;
} }
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
int nRead = 0; int nRead = 0;
if (ctx->type == TFile) { if (ctx->type == TFile) {
nRead = tfRead(ctx->file.fd, buf, len); nRead = tfRead(ctx->file.fd, buf, len);
} else { } else {
@ -40,110 +40,116 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
ctx->offset += nRead; ctx->offset += nRead;
return nRead; return nRead;
} }
static int writeCtxDoFlush(WriterCtx *ctx) { static int writeCtxDoFlush(WriterCtx *ctx) {
if (ctx->type == TFile) { if (ctx->type == TFile) {
//tfFsync(ctx->fd); // tfFsync(ctx->fd);
//tfFlush(ctx->file.fd); // tfFlush(ctx->file.fd);
} else { } else {
// do nothing // do nothing
} }
return 1; return 1;
} }
WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) {
WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); WriterCtx *ctx = calloc(1, sizeof(WriterCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) {
return NULL;
}
ctx->type = type; ctx->type = type;
if (ctx->type == TFile) { if (ctx->type == TFile) {
// ugly code, refactor later // ugly code, refactor later
ctx->file.readOnly = readOnly; ctx->file.readOnly = readOnly;
if (readOnly == false) { if (readOnly == false) {
ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); ctx->file.fd = tfOpenCreateWriteAppend(tmpFile);
} else { } else {
ctx->file.fd = tfOpenReadWrite(tmpFile); ctx->file.fd = tfOpenReadWrite(tmpFile);
} }
if (ctx->file.fd < 0) { if (ctx->file.fd < 0) {
goto END; goto END;
indexError("open file error %d", errno); indexError("open file error %d", errno);
} }
} else if (ctx->type == TMemory) { } else if (ctx->type == TMemory) {
ctx->mem.buf = calloc(1, sizeof(char) * capacity); ctx->mem.buf = calloc(1, sizeof(char) * capacity);
ctx->mem.capa = capacity; ctx->mem.capa = capacity;
} }
ctx->write = writeCtxDoWrite; ctx->write = writeCtxDoWrite;
ctx->read = writeCtxDoRead; ctx->read = writeCtxDoRead;
ctx->flush = writeCtxDoFlush; ctx->flush = writeCtxDoFlush;
ctx->offset = 0; ctx->offset = 0;
ctx->limit = capacity; ctx->limit = capacity;
return ctx; return ctx;
END: END:
if (ctx->type == TMemory) { free(ctx->mem.buf); } if (ctx->type == TMemory) {
free(ctx->mem.buf);
}
free(ctx); free(ctx);
} }
void writerCtxDestroy(WriterCtx *ctx) { void writerCtxDestroy(WriterCtx *ctx) {
if (ctx->type == TMemory) { if (ctx->type == TMemory) {
free(ctx->mem.buf); free(ctx->mem.buf);
} else { } else {
tfClose(ctx->file.fd); tfClose(ctx->file.fd);
} }
free(ctx); free(ctx);
} }
FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *fstCountingWriterCreate(void *wrt) {
FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter));
if (cw == NULL) { return NULL; } if (cw == NULL) {
return NULL;
}
cw->wrt = wrt; cw->wrt = wrt;
//(void *)(writerCtxCreate(TFile, readOnly)); //(void *)(writerCtxCreate(TFile, readOnly));
return cw; return cw;
} }
void fstCountingWriterDestroy(FstCountingWriter *cw) { void fstCountingWriterDestroy(FstCountingWriter *cw) {
// free wrt object: close fd or free mem // free wrt object: close fd or free mem
fstCountingWriterFlush(cw); fstCountingWriterFlush(cw);
//writerCtxDestroy((WriterCtx *)(cw->wrt)); // writerCtxDestroy((WriterCtx *)(cw->wrt));
free(cw); free(cw);
} }
int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) {
if (write == NULL) { return 0; } if (write == NULL) {
// update checksum return 0;
}
// update checksum
// write data to file/socket or mem // write data to file/socket or mem
WriterCtx *ctx = write->wrt; WriterCtx *ctx = write->wrt;
int nWrite = ctx->write(ctx, buf, len); int nWrite = ctx->write(ctx, buf, len);
assert(nWrite == len); assert(nWrite == len);
write->count += len; write->count += len;
return len; return len;
}
int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) {
if (write == NULL) { return 0; }
WriterCtx *ctx = write->wrt;
int nRead = ctx->read(ctx, buf, len);
//assert(nRead == len);
return nRead;
}
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) {
return 0;
} }
int fstCountingWriterFlush(FstCountingWriter *write) { int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) {
if (write == NULL) {
return 0;
}
WriterCtx *ctx = write->wrt;
int nRead = ctx->read(ctx, buf, len);
// assert(nRead == len);
return nRead;
}
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; }
int fstCountingWriterFlush(FstCountingWriter *write) {
WriterCtx *ctx = write->wrt; WriterCtx *ctx = write->wrt;
ctx->flush(ctx); ctx->flush(ctx);
//write->wtr->flush // write->wtr->flush
return 1; return 1;
} }
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) {
assert(1 <= nBytes && nBytes <= 8); assert(1 <= nBytes && nBytes <= 8);
uint8_t *buf = calloc(8, sizeof(uint8_t)); uint8_t *buf = calloc(8, sizeof(uint8_t));
for (uint8_t i = 0; i < nBytes; i++) { for (uint8_t i = 0; i < nBytes; i++) {
buf[i] = (uint8_t)n; buf[i] = (uint8_t)n;
n = n >> 8; n = n >> 8;
} }
fstCountingWriterWrite(writer, buf, nBytes); fstCountingWriterWrite(writer, buf, nBytes);
@ -154,7 +160,5 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t
uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) { uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) {
uint8_t nBytes = packSize(n); uint8_t nBytes = packSize(n);
fstCountingWriterPackUintIn(writer, n, nBytes); fstCountingWriterPackUintIn(writer, n, nBytes);
return nBytes; return nBytes;
} }

107
source/libs/index/src/index_fst_node.c
View File

@ -16,30 +16,34 @@
FstBuilderNode *fstBuilderNodeDefault() { FstBuilderNode *fstBuilderNodeDefault() {
FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); FstBuilderNode *bn = malloc(sizeof(FstBuilderNode));
bn->isFinal = false; bn->isFinal = false;
bn->finalOutput = 0; bn->finalOutput = 0;
bn->trans = taosArrayInit(16, sizeof(FstTransition)); bn->trans = taosArrayInit(16, sizeof(FstTransition));
return bn; return bn;
} }
void fstBuilderNodeDestroy(FstBuilderNode *node) { void fstBuilderNodeDestroy(FstBuilderNode *node) {
if (node == NULL) { return; } if (node == NULL) {
return;
}
taosArrayDestroy(node->trans); taosArrayDestroy(node->trans);
free(node); free(node);
} }
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) {
if (n1 == n2) { return true; } if (n1 == n2) {
if (n1 == NULL || n2 == NULL ) { return true;
}
if (n1 == NULL || n2 == NULL) {
return false; return false;
} }
if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) {
return false; return false;
} }
size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0; size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0;
size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0; size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0;
if (s1 != s2) { if (s1 != s2) {
return false; return false;
} }
for (size_t i = 0; i < s1; i++) { for (size_t i = 0; i < s1; i++) {
@ -47,69 +51,70 @@ bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) {
FstTransition *t2 = taosArrayGet(n2->trans, i); FstTransition *t2 = taosArrayGet(n2->trans, i);
if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) {
return false; return false;
} }
} }
return true; return true;
} }
FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); FstBuilderNode *node = malloc(sizeof(FstBuilderNode));
if (node == NULL) { return NULL; } if (node == NULL) {
return NULL;
}
// //
size_t sz = taosArrayGetSize(src->trans); size_t sz = taosArrayGetSize(src->trans);
SArray *trans = taosArrayInit(sz, sizeof(FstTransition)); SArray *trans = taosArrayInit(sz, sizeof(FstTransition));
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *tran = taosArrayGet(src->trans, i); FstTransition *tran = taosArrayGet(src->trans, i);
taosArrayPush(trans, tran); taosArrayPush(trans, tran);
} }
node->trans = trans; node->trans = trans;
node->isFinal = src->isFinal; node->isFinal = src->isFinal;
node->finalOutput = src->finalOutput; node->finalOutput = src->finalOutput;
return node; return node;
} }
// not destroy src, User's bussiness // not destroy src, User's bussiness
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
if (dst == NULL || src == NULL) { return; } if (dst == NULL || src == NULL) {
return;
}
dst->isFinal = src->isFinal; dst->isFinal = src->isFinal;
dst->finalOutput = src->finalOutput; dst->finalOutput = src->finalOutput;
//release free avoid mem leak // release free avoid mem leak
taosArrayDestroy(dst->trans); taosArrayDestroy(dst->trans);
size_t sz = taosArrayGetSize(src->trans); size_t sz = taosArrayGetSize(src->trans);
dst->trans = taosArrayInit(sz, sizeof(FstTransition)); dst->trans = taosArrayInit(sz, sizeof(FstTransition));
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition *trn = taosArrayGet(src->trans, i); FstTransition *trn = taosArrayGet(src->trans, i);
taosArrayPush(dst->trans, trn); taosArrayPush(dst->trans, trn);
} }
} }
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr
// startAddr) {
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { // size_t sz = taosArrayGetSize(b->trans);
// assert(sz < 256);
//size_t sz = taosArrayGetSize(b->trans); // if (FST_BUILDER_NODE_IS_FINAL(b)
//assert(sz < 256); // && FST_BUILDER_NODE_TRANS_ISEMPTY(b)
//if (FST_BUILDER_NODE_IS_FINAL(b) // && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) {
// && FST_BUILDER_NODE_TRANS_ISEMPTY(b) // return true;
// && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { //} else if (sz != 1 || b->isFinal) {
// return true; // // AnyTrans->Compile(w, addr, node);
//} else if (sz != 1 || b->isFinal) { //} else {
// // AnyTrans->Compile(w, addr, node); // FstTransition *tran = taosArrayGet(b->trans, 0);
//} else { // if (tran->addr == lastAddr && tran->out == 0) {
// FstTransition *tran = taosArrayGet(b->trans, 0); // //OneTransNext::compile(w, lastAddr, tran->inp);
// if (tran->addr == lastAddr && tran->out == 0) { // return true;
// //OneTransNext::compile(w, lastAddr, tran->inp); // } else {
// return true; // //OneTrans::Compile(w, lastAddr, *tran);
// } else { // return true;
// //OneTrans::Compile(w, lastAddr, *tran); // }
// return true; //}
// } // return true;
//} //}
//return true;
//}

162
source/libs/index/src/index_fst_registry.c
View File

@ -15,33 +15,33 @@
#include "index_fst_registry.h" #include "index_fst_registry.h"
uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
//TODO(yihaoDeng): refactor later // TODO(yihaoDeng): refactor later
const uint64_t FNV_PRIME = 1099511628211; const uint64_t FNV_PRIME = 1099511628211;
uint64_t h = 14695981039346656037u; uint64_t h = 14695981039346656037u;
h = (h ^ (uint64_t)bNode->isFinal) * FNV_PRIME; h = (h ^ (uint64_t)bNode->isFinal) * FNV_PRIME;
h = (h ^ (bNode)->finalOutput) * FNV_PRIME; h = (h ^ (bNode)->finalOutput) * FNV_PRIME;
uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans);
for (uint32_t i = 0; i < sz; i++) { for (uint32_t i = 0; i < sz; i++) {
FstTransition *trn = taosArrayGet(bNode->trans, i); FstTransition *trn = taosArrayGet(bNode->trans, i);
h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME;
h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME;
h = (h ^ (uint64_t)(trn->addr))* FNV_PRIME; h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME;
} }
return h %(registry->tableSize); return h % (registry->tableSize);
} }
static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (a >= sz || b >= sz) { return; } if (a >= sz || b >= sz) {
return;
}
FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a);
FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b);
FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node};
cell1->addr = cell2->addr; cell1->addr = cell2->addr;
cell1->node = cell2->node; cell1->node = cell2->node;
@ -52,49 +52,55 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
} }
static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
size_t sz = taosArrayGetSize(arr); size_t sz = taosArrayGetSize(arr);
if (start >= sz && end >= sz) {return; } if (start >= sz && end >= sz) {
return;
}
assert(start >= end); assert(start >= end);
int32_t s = (int32_t)start; int32_t s = (int32_t)start;
int32_t e = (int32_t)end; int32_t e = (int32_t)end;
while(s > e) { while (s > e) {
fstRegistryCellSwap(arr, s - 1, s); fstRegistryCellSwap(arr, s - 1, s);
s -= 1; s -= 1;
} }
} }
FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
FstRegistry *registry = malloc(sizeof(FstRegistry)); FstRegistry *registry = malloc(sizeof(FstRegistry));
if (registry == NULL) { return NULL ;} if (registry == NULL) {
return NULL;
}
uint64_t nCells = tableSize * mruSize; uint64_t nCells = tableSize * mruSize;
SArray* tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell));
if (NULL == tb) { if (NULL == tb) {
free(registry); free(registry);
return NULL; return NULL;
} }
for (uint64_t i = 0; i < nCells; i++) { for (uint64_t i = 0; i < nCells; i++) {
FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()};
taosArrayPush(tb, &cell); taosArrayPush(tb, &cell);
} }
registry->table = tb; registry->table = tb;
registry->tableSize = tableSize; registry->tableSize = tableSize;
registry->mruSize = mruSize; registry->mruSize = mruSize;
return registry; return registry;
} }
void fstRegistryDestroy(FstRegistry *registry) { void fstRegistryDestroy(FstRegistry *registry) {
if (registry == NULL) { return; } if (registry == NULL) {
return;
}
SArray *tb = registry->table; SArray *tb = registry->table;
size_t sz = taosArrayGetSize(tb); size_t sz = taosArrayGetSize(tb);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstRegistryCell *cell = taosArrayGet(tb, i); FstRegistryCell *cell = taosArrayGet(tb, i);
fstBuilderNodeDestroy(cell->node); fstBuilderNodeDestroy(cell->node);
} }
taosArrayDestroy(tb); taosArrayDestroy(tb);
free(registry); free(registry);
@ -102,74 +108,70 @@ void fstRegistryDestroy(FstRegistry *registry) {
FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) { FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) {
if (taosArrayGetSize(registry->table) <= 0) { if (taosArrayGetSize(registry->table) <= 0) {
return NULL; return NULL;
} }
uint64_t bucket = fstRegistryHash(registry, bNode); uint64_t bucket = fstRegistryHash(registry, bNode);
uint64_t start = registry->mruSize * bucket; uint64_t start = registry->mruSize * bucket;
uint64_t end = start + registry->mruSize; uint64_t end = start + registry->mruSize;
FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry)); FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry));
if (end - start == 1) { if (end - start == 1) {
FstRegistryCell *cell = taosArrayGet(registry->table, start); FstRegistryCell *cell = taosArrayGet(registry->table, start);
//cell->isNode && // cell->isNode &&
if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
entry->addr = cell->addr ; entry->addr = cell->addr;
return entry; return entry;
} else { } else {
fstBuilderNodeCloneFrom(cell->node, bNode); fstBuilderNodeCloneFrom(cell->node, bNode);
entry->state = NOTFOUND; entry->state = NOTFOUND;
entry->cell = cell; // copy or not entry->cell = cell; // copy or not
} }
} else if (end - start == 2) { } else if (end - start == 2) {
FstRegistryCell *cell1 = taosArrayGet(registry->table, start); FstRegistryCell *cell1 = taosArrayGet(registry->table, start);
if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
entry->addr = cell1->addr; entry->addr = cell1->addr;
return entry; return entry;
}
FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1);
if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) {
entry->state = FOUND;
entry->addr = cell2->addr;
// must swap here
fstRegistryCellSwap(registry->table, start, start + 1);
return entry;
} }
//clone from bNode, refactor later FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1);
if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) {
entry->state = FOUND;
entry->addr = cell2->addr;
// must swap here
fstRegistryCellSwap(registry->table, start, start + 1);
return entry;
}
// clone from bNode, refactor later
fstBuilderNodeCloneFrom(cell2->node, bNode); fstBuilderNodeCloneFrom(cell2->node, bNode);
fstRegistryCellSwap(registry->table, start, start + 1); fstRegistryCellSwap(registry->table, start, start + 1);
FstRegistryCell *cCell = taosArrayGet(registry->table, start); FstRegistryCell *cCell = taosArrayGet(registry->table, start);
entry->state = NOTFOUND; entry->state = NOTFOUND;
entry->cell = cCell; entry->cell = cCell;
} else { } else {
uint32_t i = start; uint32_t i = start;
for (; i < end; i++) { for (; i < end; i++) {
FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i);
if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) {
entry->state = FOUND; entry->state = FOUND;
entry->addr = cell->addr; entry->addr = cell->addr;
fstRegistryCellPromote(registry->table, i, start); fstRegistryCellPromote(registry->table, i, start);
break; break;
} }
} }
if (i >= end) { if (i >= end) {
uint64_t last = end - 1; uint64_t last = end - 1;
FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last);
//clone from bNode, refactor later // clone from bNode, refactor later
fstBuilderNodeCloneFrom(cell->node, bNode); fstBuilderNodeCloneFrom(cell->node, bNode);
fstRegistryCellPromote(registry->table, last, start); fstRegistryCellPromote(registry->table, last, start);
FstRegistryCell *cCell = taosArrayGet(registry->table, start); FstRegistryCell *cCell = taosArrayGet(registry->table, start);
entry->state = NOTFOUND; entry->state = NOTFOUND;
entry->cell = cCell; entry->cell = cCell;
} }
} }
return entry; return entry;
} }
void fstRegistryEntryDestroy(FstRegistryEntry *entry) { void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); }
free(entry);
}

137
source/libs/index/src/index_fst_util.c
View File

@ -15,37 +15,32 @@
#include "index_fst_util.h" #include "index_fst_util.h"
#include "index_fst_common.h" #include "index_fst_common.h"
// A sentinel value used to indicate an empty final state
const CompiledAddr EMPTY_ADDRESS = 0;
//A sentinel value used to indicate an empty final state
const CompiledAddr EMPTY_ADDRESS = 0;
/// A sentinel value used to indicate an invalid state. /// A sentinel value used to indicate an invalid state.
const CompiledAddr NONE_ADDRESS = 1; const CompiledAddr NONE_ADDRESS = 1;
// This version number is written to every finite state transducer created by // This version number is written to every finite state transducer created by
// this crate. When a finite state transducer is read, its version number is // this crate. When a finite state transducer is read, its version number is
// checked against this value. // checked against this value.
const uint64_t VERSION = 3; const uint64_t VERSION = 3;
// The threshold (in number of transitions) at which an index is created for // The threshold (in number of transitions) at which an index is created for
// a node's transitions. This speeds up lookup time at the expense of FST size // a node's transitions. This speeds up lookup time at the expense of FST size
const uint64_t TRANS_INDEX_THRESHOLD = 32; const uint64_t TRANS_INDEX_THRESHOLD = 32;
// uint8_t commonInput(uint8_t idx) {
//uint8_t commonInput(uint8_t idx) {
// if (idx == 0) { return -1; } // if (idx == 0) { return -1; }
// else { // else {
// return COMMON_INPUTS_INV[idx - 1]; // return COMMON_INPUTS_INV[idx - 1];
// } // }
//} //}
// //
//uint8_t commonIdx(uint8_t v, uint8_t max) { // uint8_t commonIdx(uint8_t v, uint8_t max) {
// uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256; // uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256;
// return v > max ? 0: v; // return v > max ? 0: v;
//} //}
uint8_t packSize(uint64_t n) { uint8_t packSize(uint64_t n) {
if (n < (1u << 8)) { if (n < (1u << 8)) {
return 1; return 1;
@ -71,17 +66,17 @@ uint64_t unpackUint64(uint8_t *ch, uint8_t sz) {
for (uint8_t i = 0; i < sz; i++) { for (uint8_t i = 0; i < sz; i++) {
n = n | (ch[i] << (8 * i)); n = n | (ch[i] << (8 * i));
} }
return n; return n;
} }
uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) { uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) {
if (transAddr == EMPTY_ADDRESS) { if (transAddr == EMPTY_ADDRESS) {
return packSize(EMPTY_ADDRESS); return packSize(EMPTY_ADDRESS);
} else { } else {
return packSize(nodeAddr - transAddr); return packSize(nodeAddr - transAddr);
} }
} }
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) {
uint64_t delta = unpackUint64(data, len); uint64_t delta = unpackUint64(data, len);
// delta_add = u64_to_usize // delta_add = u64_to_usize
if (delta == EMPTY_ADDRESS) { if (delta == EMPTY_ADDRESS) {
return EMPTY_ADDRESS; return EMPTY_ADDRESS;
@ -95,56 +90,53 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) {
FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { FstSlice fstSliceCreate(uint8_t *data, uint64_t len) {
FstString *str = (FstString *)malloc(sizeof(FstString)); FstString *str = (FstString *)malloc(sizeof(FstString));
str->ref = 1; str->ref = 1;
str->len = len; str->len = len;
str->data = malloc(len * sizeof(uint8_t)); str->data = malloc(len * sizeof(uint8_t));
memcpy(str->data, data, len); memcpy(str->data, data, len);
FstSlice s = {.str = str, .start = 0, .end = len - 1}; FstSlice s = {.str = str, .start = 0, .end = len - 1};
return s; return s;
} }
// just shallow copy // just shallow copy
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) {
FstString *str = s->str; FstString *str = s->str;
str->ref++; str->ref++;
//uint8_t *buf = fstSliceData(s, &alen); // uint8_t *buf = fstSliceData(s, &alen);
//start = buf + start - (buf - s->start); // start = buf + start - (buf - s->start);
//end = buf + end - (buf - s->start); // end = buf + end - (buf - s->start);
FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
return t; return t;
} }
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
int32_t tlen = end - start + 1;
int32_t tlen = end - start + 1; int32_t slen;
int32_t slen; uint8_t *data = fstSliceData(s, &slen);
uint8_t *data = fstSliceData(s, &slen);
assert(tlen <= slen); assert(tlen <= slen);
uint8_t *buf = malloc(sizeof(uint8_t) * tlen); uint8_t *buf = malloc(sizeof(uint8_t) * tlen);
memcpy(buf, data + start, tlen); memcpy(buf, data + start, tlen);
FstString *str = malloc(sizeof(FstString)); FstString *str = malloc(sizeof(FstString));
str->data = buf; str->data = buf;
str->len = tlen; str->len = tlen;
str->ref = 1; str->ref = 1;
FstSlice ans; FstSlice ans;
ans.str = str; ans.str = str;
ans.start = 0; ans.start = 0;
ans.end = tlen - 1; ans.end = tlen - 1;
return ans; return ans;
}
bool fstSliceIsEmpty(FstSlice *s) {
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
} }
bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; }
uint8_t *fstSliceData(FstSlice *s, int32_t *size) { uint8_t *fstSliceData(FstSlice *s, int32_t *size) {
FstString *str = s->str; FstString *str = s->str;
if (size != NULL) { if (size != NULL) {
*size = s->end - s->start + 1; *size = s->end - s->start + 1;
} }
return str->data + s->start; return str->data + s->start;
} }
void fstSliceDestroy(FstSlice *s) { void fstSliceDestroy(FstSlice *s) {
FstString *str = s->str; FstString *str = s->str;
@ -152,40 +144,45 @@ void fstSliceDestroy(FstSlice *s) {
if (str->ref <= 0) { if (str->ref <= 0) {
free(str->data); free(str->data);
free(str); free(str);
s->str = NULL; s->str = NULL;
} }
} }
int fstSliceCompare(FstSlice *a, FstSlice *b) { int fstSliceCompare(FstSlice *a, FstSlice *b) {
int32_t alen, blen; int32_t alen, blen;
uint8_t *aBuf = fstSliceData(a, &alen); uint8_t *aBuf = fstSliceData(a, &alen);
uint8_t *bBuf = fstSliceData(b, &blen); uint8_t *bBuf = fstSliceData(b, &blen);
uint32_t i, j; uint32_t i, j;
for (i = 0, j = 0; i < alen && j < blen; i++, j++) { for (i = 0, j = 0; i < alen && j < blen; i++, j++) {
uint8_t x = aBuf[i]; uint8_t x = aBuf[i];
uint8_t y = bBuf[j]; uint8_t y = bBuf[j];
if (x == y) { continue;} if (x == y) {
else if (x < y) { return -1; } continue;
else { return 1; }; } else if (x < y) {
return -1;
} else {
return 1;
};
} }
if (i < alen) { return 1; } if (i < alen) {
else if (j < blen) { return -1; } return 1;
else { return 0; } } else if (j < blen) {
} return -1;
} else {
return 0;
}
}
//FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { // FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) {
// FstStack *s = calloc(1, sizeof(FstStack)); // FstStack *s = calloc(1, sizeof(FstStack));
// if (s == NULL) { return NULL; } // if (s == NULL) { return NULL; }
// s-> // s->
// s->freeFn // s->freeFn
// //
//} //}
//void *fstStackPush(FstStack *s, void *elem); // void *fstStackPush(FstStack *s, void *elem);
//void *fstStackTop(FstStack *s); // void *fstStackTop(FstStack *s);
//size_t fstStackLen(FstStack *s); // size_t fstStackLen(FstStack *s);
//void *fstStackGetAt(FstStack *s, size_t i); // void *fstStackGetAt(FstStack *s, size_t i);
//void fstStackDestory(FstStack *); // void fstStackDestory(FstStack *);

203
source/libs/index/src/index_tfile.c
View File

@ -16,58 +16,60 @@
//#include <sys/types.h> //#include <sys/types.h>
//#include <dirent.h> //#include <dirent.h>
#include "index_tfile.h" #include "index_tfile.h"
#include "index.h"
#include "index_fst.h" #include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_util.h" #include "index_util.h"
#include "taosdef.h" #include "taosdef.h"
#include "index.h"
#include "index_fst_counting_writer.h"
static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) {
static FORCE_INLINE int tfileLoadHeader(WriterCtx *ctx, TFileReadHeader *header) { // TODO simple tfile header later
//TODO simple tfile header later char buf[TFILE_HADER_PRE_SIZE];
char buf[TFILE_HADER_PRE_SIZE]; char * p = buf;
char *p = buf; TFileReadHeader *header = &reader->header;
int64_t nread = ctx->read(ctx, buf, TFILE_HADER_PRE_SIZE); int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE);
assert(nread == TFILE_HADER_PRE_SIZE); assert(nread == TFILE_HADER_PRE_SIZE);
memcpy(&header->suid, p, sizeof(header->suid)); memcpy(&header->suid, p, sizeof(header->suid));
p += sizeof(header->suid); p += sizeof(header->suid);
memcpy(&header->version, p, sizeof(header->version)); memcpy(&header->version, p, sizeof(header->version));
p += sizeof(header->version); p += sizeof(header->version);
int32_t colLen = 0; int32_t colLen = 0;
memcpy(&colLen, p, sizeof(colLen)); memcpy(&colLen, p, sizeof(colLen));
assert(colLen < sizeof(header->colName)); assert(colLen < sizeof(header->colName));
nread = ctx->read(ctx, header->colName, colLen); nread = reader->ctx->read(reader->ctx, header->colName, colLen);
assert(nread == colLen); assert(nread == colLen);
nread = ctx->read(ctx, &header->colType, sizeof(header->colType)); nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType));
return 0; return 0;
}; };
static int tfileGetFileList(const char *path, SArray *result) { static int tfileGetFileList(const char *path, SArray *result) {
DIR *dir = opendir(path); DIR *dir = opendir(path);
if (NULL == dir) { return -1; } if (NULL == dir) {
return -1;
}
struct dirent *entry; struct dirent *entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
size_t len = strlen(entry->d_name); size_t len = strlen(entry->d_name);
char *buf = calloc(1, len + 1); char * buf = calloc(1, len + 1);
memcpy(buf, entry->d_name, len); memcpy(buf, entry->d_name, len);
taosArrayPush(result, &buf); taosArrayPush(result, &buf);
} }
closedir(dir); closedir(dir);
return 0; return 0;
} }
static void tfileDestroyFileName(void *elem) { static void tfileDestroyFileName(void *elem) {
char *p = *(char **)elem; char *p = *(char **)elem;
free(p); free(p);
} }
static int tfileCompare(const void *a, const void *b) { static int tfileCompare(const void *a, const void *b) {
const char *aName = *(char **)a; const char *aName = *(char **)a;
const char *bName = *(char **)b; const char *bName = *(char **)b;
size_t aLen = strlen(aName); size_t aLen = strlen(aName);
size_t bLen = strlen(bName); size_t bLen = strlen(bName);
return strncmp(aName, bName, aLen > bLen ? aLen : bLen); return strncmp(aName, bName, aLen > bLen ? aLen : bLen);
} }
// tfile name suid-colId-version.tindex // tfile name suid-colId-version.tindex
@ -75,126 +77,131 @@ static int tfileParseFileName(const char *filename, uint64_t *suid, int *colId,
if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) {
// read suid & colid & version success // read suid & colid & version success
return 0; return 0;
} }
return -1; return -1;
} }
static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) {
SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_MEM_TO_BUF(buf, key, suid);
SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_MEM_TO_BUF(buf, key, colType); SERIALIZE_MEM_TO_BUF(buf, key, colType);
SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_MEM_TO_BUF(buf, key, version); SERIALIZE_MEM_TO_BUF(buf, key, version);
SERIALIZE_VAR_TO_BUF(buf, '_', char); SERIALIZE_VAR_TO_BUF(buf, '_', char);
SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
} }
TFileCache *tfileCacheCreate(const char *path) { TFileCache *tfileCacheCreate(const char *path) {
TFileCache *tcache = calloc(1, sizeof(TFileCache)); TFileCache *tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; } if (tcache == NULL) {
return NULL;
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); }
tcache->capacity = 64;
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
SArray *files = taosArrayInit(4, sizeof(void *)); tcache->capacity = 64;
tfileGetFileList(path, files);
SArray *files = taosArrayInit(4, sizeof(void *));
tfileGetFileList(path, files);
taosArraySort(files, tfileCompare); taosArraySort(files, tfileCompare);
for (size_t i = 0; i < taosArrayGetSize(files); i++) { for (size_t i = 0; i < taosArrayGetSize(files); i++) {
char *file = taosArrayGetP(files, i); char * file = taosArrayGetP(files, i);
uint64_t suid; uint64_t suid;
int colId, version; int colId, version;
if (0 != tfileParseFileName(file, &suid, &colId, &version)) { if (0 != tfileParseFileName(file, &suid, &colId, &version)) {
goto End; goto End;
continue; continue;
}
TFileReader *reader = calloc(1, sizeof(TFileReader));
reader->ctx = writerCtxCreate(TFile, file, true, 1024 * 64);
if (reader->ctx == NULL) {
TFileReaderDestroy(reader);
indexError("failed to open index: %s", file);
goto End;
} }
TFileReadHeader header = {0};
if (0 != tfileLoadHeader(reader->ctx, &header)) { WriterCtx *wc = writerCtxCreate(TFile, file, true, 1024 * 64);
TFileReaderDestroy(reader); if (wc == NULL) {
indexError("failed to load index header, index Id: %s", file); indexError("failed to open index: %s", file);
goto End;
}
TFileReader *reader = tfileReaderCreate(wc);
if (0 != tfileReadLoadHeader(reader)) {
TFileReaderDestroy(reader);
indexError("failed to load index header, index Id: %s", file);
goto End;
} }
} }
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return tcache; return tcache;
End: End:
tfileCacheDestroy(tcache);
taosArrayDestroyEx(files, tfileDestroyFileName); taosArrayDestroyEx(files, tfileDestroyFileName);
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache *tcache) { void tfileCacheDestroy(TFileCache *tcache) {
if (tcache == NULL) {
free(tcache); return;
}
// free table cache
TFileReader **reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) {
TFileReader *p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName,
p->header.colType);
TFileReaderDestroy(p);
reader = taosHashIterate(tcache->tableCache, reader);
}
taosHashCleanup(tcache->tableCache);
free(tcache);
} }
TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) {
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(key, buf); tfileSerialCacheKey(key, buf);
TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
return reader; return reader;
} }
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) {
char buf[128] = {0}; char buf[128] = {0};
tfileSerialCacheKey(key, buf); tfileSerialCacheKey(key, buf);
taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *));
return; return;
} }
TFileReader *tfileReaderCreate(WriterCtx *ctx) {
TFileReader* tfileReaderCreate() { TFileReader *reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) {
return NULL;
}
reader->ctx = ctx;
// T_REF_INC(reader);
return reader;
} }
void TFileReaderDestroy(TFileReader *reader) { void TFileReaderDestroy(TFileReader *reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
writerCtxDestroy(reader->ctx); }
// T_REF_INC(reader);
writerCtxDestroy(reader->ctx);
free(reader); free(reader);
} }
TFileWriter *tfileWriterCreate(const char *suid, const char *colName); TFileWriter *tfileWriterCreate(const char *suid, const char *colName);
void tfileWriterDestroy(TFileWriter *tw); void tfileWriterDestroy(TFileWriter *tw);
IndexTFile *indexTFileCreate(const char *path) { IndexTFile *indexTFileCreate(const char *path) {
IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); IndexTFile *tfile = calloc(1, sizeof(IndexTFile));
tfile->cache = tfileCacheCreate(path); tfile->cache = tfileCacheCreate(path);
return tfile; return tfile;
} }
void IndexTFileDestroy(IndexTFile *tfile) { void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); }
free(tfile);
}
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) {
IndexTFile *pTfile = (IndexTFile *)tfile; IndexTFile *pTfile = (IndexTFile *)tfile;
SIndexTerm *term = query->term; SIndexTerm * term = query->term;
TFileCacheKey key = {.suid = term->suid, TFileCacheKey key = {
.colType = term->colType, .suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName};
.version = 0, TFileReader *reader = tfileCacheGet(pTfile->cache, &key);
.colName = term->colName,
.nColName= term->nColName};
TFileReader *reader = tfileCacheGet(pTfile->cache, &key);
return 0; return 0;
} }
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) {
TFileWriterOpt wOpt = {.suid = term->suid, TFileWriterOpt wOpt = {
.colType = term->colType, .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1};
.colName = term->colName,
.nColName= term->nColName,
.version = 1};
return 0;
return 0;
} }