xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #10390 from taosdata/feature/indexUpdate 

add multi tag query
This commit is contained in:
Shengliang Guan 2022-02-24 20:51:30 +08:00 committed by GitHub
parent e1943fd626 930d09dc0b
commit fac3fad42b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 273 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
source/libs/index/inc/index_util.h
View File

@ -15,36 +15,46 @@
#ifndef __INDEX_UTIL_H__
#define __INDEX_UTIL_H__
#include "tarray.h"
#ifdef __cplusplus
extern "C" {
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void*)buf, (void*)key->mem, len); \
memcpy((void *)buf, (void *)key->mem, len); \
buf += len; \
} while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void*)buf, (void*)&c, sizeof(c)); \
buf += sizeof(c); \
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void*)buf, (void*)var, len); \
memcpy((void *)buf, (void *)var, len); \
buf += len; \
} while (0)
/* multi sorted result intersection
* input: [1, 2, 4, 5]
* [2, 3, 4, 5]
* [1, 4, 5]
* output:[4, 5]
*/
void iIntersection(SArray *interResults, SArray *finalResult);
#ifdef __cplusplus
}
#endif

17
source/libs/index/src/index.c
View File

@ -370,22 +370,27 @@ static void indexInterResultsDestroy(SArray* results) {
}
taosArrayDestroy(results);
}
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType
SArray* first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL);
for (int i = 0; i < taosArrayGetSize(interResults); i--) {
SArray* t = taosArrayGetP(interResults, i);
taosArraySort(t, uidCompare);
taosArrayRemoveDuplicate(t, uidCompare, NULL);
}
if (oType == MUST) {
iIntersection(interResults, fResults);
// just one column index, enhance later
taosArrayAddAll(fResults, first);
// taosArrayAddAll(fResults, interResults);
} else if (oType == SHOULD) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
taosArrayAddAll(fResults, interResults);
// tag1 condistion || tag2 condition
} else if (oType == NOT) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
taosArrayAddAll(fResults, interResults);
// not use currently
}
return 0;

2
source/libs/index/src/index_tfile.c
View File

@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile;
IndexTFile* pTfile = tfile;
SIndexTerm* term = query->term;
ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};

70
source/libs/index/src/index_util.c Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index_util.h"
#include "index.h"
typedef struct MergeIndex {
int idx;
int len;
} MergeIndex;
static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
uint64_t v;
int32_t m;
while (s <= e) {
m = s + (e - s) / 2;
v = *(uint64_t *)taosArrayGet(arr, m);
if (v >= k) {
e = m - 1;
} else {
s = m + 1;
}
}
return s;
}
void iIntersection(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
SArray *base = taosArrayGetP(inters, 0);
for (int i = 0; i < taosArrayGetSize(base); i++) {
uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
bool has = true;
for (int j = 1; j < taosArrayGetSize(inters); j++) {
SArray *oth = taosArrayGetP(inters, j);
int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
if (mid >= 0 && mid < mi[j].len) {
uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
has = (val == tgt ? true : false);
mi[j].idx = mid;
} else {
has = false;
}
}
if (has == true) {
taosArrayPush(final, &tgt);
}
}
tfree(mi);
}

20
source/libs/index/test/CMakeLists.txt
View File

@ -1,6 +1,7 @@
add_executable(indexTest "")
add_executable(fstTest "")
add_executable(fstUT "")
add_executable(UtilUT "")
target_sources(indexTest
PRIVATE
@ -15,6 +16,11 @@ target_sources(fstUT
PRIVATE
"fstUT.cc"
)
target_sources(UtilUT
PRIVATE
"utilUT.cc"
)
target_include_directories ( indexTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
@ -31,6 +37,12 @@ target_include_directories ( fstUT
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories ( UtilUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest
os
util
@ -53,6 +65,14 @@ target_link_libraries (fstUT
index
)
target_link_libraries (UtilUT
os
util
common
gtest_main
index
)
#add_test(
# NAME index_test

149
source/libs/index/test/utilUT.cc Normal file
View File

@ -0,0 +1,149 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "index_util.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
class UtilEnv : public ::testing::Test {
protected:
virtual void SetUp() {
src = (SArray *)taosArrayInit(2, sizeof(void *));
for (int i = 0; i < 3; i++) {
SArray *m = taosArrayInit(10, sizeof(uint64_t));
taosArrayPush(src, &m);
}
rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
}
virtual void TearDown() {
for (int i = 0; i < taosArrayGetSize(src); i++) {
SArray *m = (SArray *)taosArrayGetP(src, i);
taosArrayDestroy(m);
}
taosArrayDestroy(src);
}
SArray *src;
SArray *rslt;
};
static void clearSourceArray(SArray *p) {
for (int i = 0; i < taosArrayGetSize(p); i++) {
SArray *m = (SArray *)taosArrayGetP(p, i);
taosArrayClear(m);
}
}
static void clearFinalArray(SArray *p) { taosArrayClear(p); }
TEST_F(UtilEnv, intersectionSimpleResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 10);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectMultiEmptyResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 10; i < 20; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 20; i < 30; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
// empty source
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectSimpleEmpty) {
clearSourceArray(src);
clearFinalArray(rslt);
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, intersect01) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {2, 3, 4, 5};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {1, 2, 3, 5};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {3, 5, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 2);
}
TEST_F(UtilEnv, intersect02) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {8, 10, 12, 13};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {9, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}