Merge pull request #11137 from taosdata/feature/fst_update_query
add fuzzy search
This commit is contained in:
Yihao Deng
GitHub
commit
6ce2556391
|
|
@ -0,0 +1,74 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef __INDEX_FST_DFA_H__
|
||||
#define __INDEX_FST_DFA_H__
|
||||
|
||||
#include "indexFstRegex.h"
|
||||
#include "indexFstSparse.h"
|
||||
#include "tarray.h"
|
||||
#include "thash.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct FstDfa FstDfa;
|
||||
|
||||
typedef struct {
|
||||
SArray * insts;
|
||||
uint32_t next[256];
|
||||
bool isMatch;
|
||||
} State;
|
||||
|
||||
/*
|
||||
* dfa builder related func
|
||||
**/
|
||||
typedef struct FstDfaBuilder {
|
||||
FstDfa * dfa;
|
||||
SHashObj *cache;
|
||||
} FstDfaBuilder;
|
||||
|
||||
FstDfaBuilder *dfaBuilderCreate(SArray *insts);
|
||||
|
||||
void dfaBuilderDestroy(FstDfaBuilder *builder);
|
||||
|
||||
FstDfa *dfaBuilderBuild(FstDfaBuilder *builder);
|
||||
|
||||
bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t bytes,
|
||||
uint32_t *result);
|
||||
|
||||
bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result);
|
||||
|
||||
/*
|
||||
* dfa related func
|
||||
**/
|
||||
typedef struct FstDfa {
|
||||
SArray *insts;
|
||||
SArray *states;
|
||||
} FstDfa;
|
||||
|
||||
FstDfa *dfaCreate(SArray *insts, SArray *states);
|
||||
bool dfaIsMatch(FstDfa *dfa, uint32_t si);
|
||||
bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result);
|
||||
void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip);
|
||||
bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_INDEX_FST_REGEX_H_
|
||||
#define _TD_INDEX_FST_REGEX_H_
|
||||
|
||||
//#include "indexFstDfa.h"
|
||||
#include "taos.h"
|
||||
#include "tarray.h"
|
||||
#include "tchecksum.h"
|
||||
#include "thash.h"
|
||||
#include "tlog.h"
|
||||
#include "tutil.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType;
|
||||
|
||||
typedef struct MatchValue {
|
||||
} MatchValue;
|
||||
typedef struct JumpValue {
|
||||
uint32_t step;
|
||||
} JumpValue;
|
||||
|
||||
typedef struct SplitValue {
|
||||
uint32_t len1;
|
||||
uint32_t len2;
|
||||
} SplitValue;
|
||||
|
||||
typedef struct RangeValue {
|
||||
uint8_t start;
|
||||
uint8_t end;
|
||||
} RangeValue;
|
||||
|
||||
typedef struct {
|
||||
InstType ty;
|
||||
union {
|
||||
MatchValue mv;
|
||||
JumpValue jv;
|
||||
SplitValue sv;
|
||||
RangeValue rv;
|
||||
};
|
||||
} Inst;
|
||||
|
||||
typedef struct {
|
||||
char *orig;
|
||||
void *dfa;
|
||||
} FstRegex;
|
||||
|
||||
FstRegex *regexCreate(const char *str);
|
||||
|
||||
void regexSetup(FstRegex *regex, uint32_t size, const char *str);
|
||||
|
||||
// uint32_t regexStart()
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -13,8 +13,8 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_INDEX_SPARSE_H_
|
||||
#define _TD_INDEX_SPARSE_H_
|
||||
#ifndef _TD_INDEX_FST_SPARSE_H_
|
||||
#define _TD_INDEX_FST_SPARSE_H_
|
||||
|
||||
#include "tarray.h"
|
||||
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "indexFstDfa.h"
|
||||
#include "thash.h"
|
||||
|
||||
const static uint32_t STATE_LIMIT = 1000;
|
||||
|
||||
static int dfaInstsEqual(const void *a, const void *b, size_t size) {
|
||||
SArray *ar = (SArray *)a;
|
||||
SArray *br = (SArray *)b;
|
||||
size_t al = ar != NULL ? taosArrayGetSize(ar) : 0;
|
||||
size_t bl = br != NULL ? taosArrayGetSize(br) : 0;
|
||||
if (al != bl) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < al; i++) {
|
||||
uint32_t v1 = *(uint32_t *)taosArrayGet(ar, i);
|
||||
uint32_t v2 = *(uint32_t *)taosArrayGet(br, i);
|
||||
if (v1 != v2) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
FstDfaBuilder *dfaBuilderCreate(SArray *insts) {
|
||||
FstDfaBuilder *builder = taosMemoryCalloc(1, sizeof(FstDfaBuilder));
|
||||
if (builder == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
SArray *states = taosArrayInit(4, sizeof(State));
|
||||
|
||||
builder->dfa = dfaCreate(insts, states);
|
||||
builder->cache = taosHashInit(
|
||||
4, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT),
|
||||
false, HASH_NO_LOCK);
|
||||
taosHashSetEqualFp(builder->cache, dfaInstsEqual);
|
||||
return builder;
|
||||
}
|
||||
void dfaBuilderDestroy(FstDfaBuilder *builder) {
|
||||
if (builder == NULL) {
|
||||
return;
|
||||
}
|
||||
void *pIter = builder->cache != NULL ? taosHashIterate(builder->cache, NULL) : NULL;
|
||||
while (pIter) {
|
||||
SArray **key = pIter;
|
||||
taosArrayDestroy(*key);
|
||||
pIter = taosHashIterate(builder->cache, pIter);
|
||||
}
|
||||
taosHashCleanup(builder->cache);
|
||||
}
|
||||
|
||||
FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
|
||||
uint32_t sz = taosArrayGetSize(builder->dfa->insts);
|
||||
FstSparseSet *cur = sparSetCreate(sz);
|
||||
FstSparseSet *nxt = sparSetCreate(sz);
|
||||
|
||||
dfaAdd(builder->dfa, cur, 0);
|
||||
|
||||
SArray * states = taosArrayInit(0, sizeof(uint32_t));
|
||||
uint32_t result;
|
||||
if (dfaBuilderCachedState(builder, cur, &result)) {
|
||||
taosArrayPush(states, &result);
|
||||
}
|
||||
SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
|
||||
while (taosArrayGetSize(states) != 0) {
|
||||
result = *(uint32_t *)taosArrayPop(states);
|
||||
for (int i = 0; i < 256; i++) {
|
||||
uint32_t ns, dummpy = 0;
|
||||
if (dfaBuilderRunState(builder, cur, nxt, result, i, &ns)) {
|
||||
if (taosHashGet(seen, &ns, sizeof(ns)) == NULL) {
|
||||
taosHashPut(seen, &ns, sizeof(ns), &dummpy, sizeof(dummpy));
|
||||
taosArrayPush(states, &ns);
|
||||
}
|
||||
}
|
||||
if (taosArrayGetSize(builder->dfa->states) > STATE_LIMIT) {
|
||||
// Too many state;
|
||||
//
|
||||
}
|
||||
}
|
||||
}
|
||||
taosArrayDestroy(states);
|
||||
taosHashCleanup(seen);
|
||||
return builder->dfa;
|
||||
}
|
||||
|
||||
bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte,
|
||||
uint32_t *result) {
|
||||
sparSetClear(cur);
|
||||
State *t = taosArrayGet(builder->dfa->states, state);
|
||||
for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
|
||||
uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
|
||||
sparSetAdd(cur, ip);
|
||||
}
|
||||
dfaRun(builder->dfa, cur, next, byte);
|
||||
|
||||
t = taosArrayGet(builder->dfa->states, state);
|
||||
|
||||
uint32_t nxtState;
|
||||
if (dfaBuilderCachedState(builder, next, &nxtState)) {
|
||||
t->next[byte] = nxtState;
|
||||
*result = nxtState;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) {
|
||||
SArray *tinsts = taosArrayInit(4, sizeof(uint32_t));
|
||||
bool isMatch = false;
|
||||
|
||||
for (int i = 0; i < sparSetLen(set); i++) {
|
||||
uint32_t ip = sparSetGet(set, i);
|
||||
|
||||
Inst *inst = taosArrayGet(builder->dfa->insts, ip);
|
||||
if (inst->ty == JUMP || inst->ty == SPLIT) {
|
||||
continue;
|
||||
} else if (inst->ty == RANGE) {
|
||||
taosArrayPush(tinsts, &ip);
|
||||
} else if (inst->ty == MATCH) {
|
||||
isMatch = true;
|
||||
taosArrayPush(tinsts, &ip);
|
||||
}
|
||||
}
|
||||
if (taosArrayGetSize(tinsts) == 0) {
|
||||
return false;
|
||||
}
|
||||
uint32_t *v = taosHashGet(builder->cache, &tinsts, sizeof(POINTER_BYTES));
|
||||
if (v != NULL) {
|
||||
*result = *v;
|
||||
taosArrayDestroy(tinsts);
|
||||
} else {
|
||||
State st;
|
||||
st.insts = tinsts;
|
||||
st.isMatch = isMatch;
|
||||
taosArrayPush(builder->dfa->states, &st);
|
||||
int32_t sz = taosArrayGetSize(builder->dfa->states) - 1;
|
||||
taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz));
|
||||
*result = sz;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
FstDfa *dfaCreate(SArray *insts, SArray *states) {
|
||||
FstDfa *dfa = taosMemoryCalloc(1, sizeof(FstDfa));
|
||||
if (dfa == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
dfa->insts = insts;
|
||||
dfa->states = states;
|
||||
return dfa;
|
||||
}
|
||||
bool dfaIsMatch(FstDfa *dfa, uint32_t si) {
|
||||
if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
|
||||
return false;
|
||||
}
|
||||
State *st = taosArrayGet(dfa->states, si);
|
||||
return st != NULL ? st->isMatch : false;
|
||||
}
|
||||
bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) {
|
||||
if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) {
|
||||
return false;
|
||||
}
|
||||
State *st = taosArrayGet(dfa->states, si);
|
||||
*result = st->next[byte];
|
||||
return true;
|
||||
}
|
||||
void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) {
|
||||
if (sparSetContains(set, ip)) {
|
||||
return;
|
||||
}
|
||||
sparSetAdd(set, ip);
|
||||
Inst *inst = taosArrayGet(dfa->insts, ip);
|
||||
if (inst->ty == MATCH || inst->ty == RANGE) {
|
||||
// do nothing
|
||||
} else if (inst->ty == JUMP) {
|
||||
dfaAdd(dfa, set, inst->jv.step);
|
||||
} else if (inst->ty == SPLIT) {
|
||||
dfaAdd(dfa, set, inst->sv.len1);
|
||||
dfaAdd(dfa, set, inst->sv.len2);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) {
|
||||
bool isMatch = false;
|
||||
sparSetClear(to);
|
||||
for (int i = 0; i < sparSetLen(from); i++) {
|
||||
uint32_t ip = sparSetGet(from, i);
|
||||
|
||||
Inst *inst = taosArrayGet(dfa->insts, ip);
|
||||
if (inst->ty == JUMP || inst->ty == SPLIT) {
|
||||
continue;
|
||||
} else if (inst->ty == MATCH) {
|
||||
isMatch = true;
|
||||
} else if (inst->ty == RANGE) {
|
||||
if (inst->rv.start <= byte && byte <= inst->rv.end) {
|
||||
dfaAdd(dfa, to, ip + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return isMatch;
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "indexFstRegex.h"
|
||||
#include "indexFstSparse.h"
|
||||
|
||||
FstRegex *regexCreate(const char *str) {
|
||||
FstRegex *regex = taosMemoryCalloc(1, sizeof(FstRegex));
|
||||
if (regex == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
int32_t sz = (int32_t)strlen(str);
|
||||
char * orig = taosMemoryCalloc(1, sz);
|
||||
memcpy(orig, str, sz);
|
||||
|
||||
regex->orig = orig;
|
||||
}
|
||||
|
||||
void regexSetup(FstRegex *regex, uint32_t size, const char *str) {
|
||||
// return
|
||||
// return;
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "indexSparse.h"
|
||||
#include "indexFstSparse.h"
|
||||
|
||||
FstSparseSet *sparSetCreate(int32_t sz) {
|
||||
FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
|
||||
|
|
|
|||
Loading…
Reference in New Issue