xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #10563 from taosdata/feature/index_oper 

add update on index
This commit is contained in:
Yihao Deng 2022-03-05 11:51:27 +08:00 committed by GitHub
parent 74fb9433a6 c368e59689
commit 23521373aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
source/libs/index/src/index.c
View File

@ -16,6 +16,7 @@
#include "index.h" #include "index.h"
#include "indexInt.h" #include "indexInt.h"
#include "index_cache.h" #include "index_cache.h"
#include "index_comm.h"
#include "index_tfile.h" #include "index_tfile.h"
#include "index_util.h" #include "index_util.h"
#include "tdef.h" #include "tdef.h"
@ -30,8 +31,6 @@
void* indexQhandle = NULL; void* indexQhandle = NULL;
static char JSON_COLUMN[] = "JSON";
#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ #define INDEX_MERGE_ADD_DEL(src, dst, tgt) \
{ \ { \
bool f = false; \ bool f = false; \
@ -64,13 +63,11 @@ typedef struct SIdxColInfo {
int cVersion; int cVersion;
} SIdxColInfo; } SIdxColInfo;
typedef struct SIdxMergeHelper { typedef struct SIdxTempResult {
char* colVal;
SArray* total; SArray* total;
SArray* added; SArray* added;
SArray* deled; SArray* deled;
bool reset; } SIdxTempResult;
} SIdxMergeHelper;
static pthread_once_t isInit = PTHREAD_ONCE_INIT; static pthread_once_t isInit = PTHREAD_ONCE_INIT;
// static void indexInit(); // static void indexInit();
@ -82,8 +79,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp
static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
// merge cache and tfile by opera type // merge cache and tfile by opera type
static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxMergeHelper* helper); static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTempResult* helper);
static void indexMergeSameKey(SArray* result, TFileValue* tv, SIdxMergeHelper* helper);
// static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf);
@ -399,7 +395,6 @@ static void indexInterResultsDestroy(SArray* results) {
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType // refactor, merge interResults into fResults by oType
for (int i = 0; i < taosArrayGetSize(interResults); i--) { for (int i = 0; i < taosArrayGetSize(interResults); i--) {
SArray* t = taosArrayGetP(interResults, i); SArray* t = taosArrayGetP(interResults, i);
taosArraySort(t, uidCompare); taosArraySort(t, uidCompare);
@ -418,98 +413,82 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType
return 0; return 0;
} }
SIdxMergeHelper* sIdxMergeHelperCreate() { SIdxTempResult* sIdxTempResultCreate() {
SIdxMergeHelper* hp = calloc(1, sizeof(SIdxMergeHelper)); SIdxTempResult* tr = calloc(1, sizeof(SIdxTempResult));
hp->total = taosArrayInit(4, sizeof(uint64_t)); tr->total = taosArrayInit(4, sizeof(uint64_t));
hp->added = taosArrayInit(4, sizeof(uint64_t)); tr->added = taosArrayInit(4, sizeof(uint64_t));
hp->deled = taosArrayInit(4, sizeof(uint64_t)); tr->deled = taosArrayInit(4, sizeof(uint64_t));
hp->reset = false; return tr;
return hp;
} }
void sIdxMergeHelperClear(SIdxMergeHelper* hp) { void sIdxTempResultClear(SIdxTempResult* tr) {
if (hp == NULL) { if (tr == NULL) {
return; return;
} }
hp->reset = false; taosArrayClear(tr->total);
taosArrayClear(hp->total); taosArrayClear(tr->added);
taosArrayClear(hp->added); taosArrayClear(tr->deled);
taosArrayClear(hp->deled);
} }
void sIdxMergeHelperDestroy(SIdxMergeHelper* hp) { void sIdxTempResultDestroy(SIdxTempResult* tr) {
if (hp == NULL) { if (tr == NULL) {
return; return;
} }
taosArrayDestroy(hp->total); taosArrayDestroy(tr->total);
taosArrayDestroy(hp->added); taosArrayDestroy(tr->added);
taosArrayDestroy(hp->deled); taosArrayDestroy(tr->deled);
} }
static void indexMergeSameKey(SArray* result, TFileValue* tv, SIdxMergeHelper* helper) { static void sIdxTempResultMergeTo(SArray* result, SIdxTempResult* tr) {
int32_t sz = result ? taosArrayGetSize(result) : 0; taosArraySort(tr->total, uidCompare);
if (sz > 0) { taosArraySort(tr->added, uidCompare);
// TODO(yihao): remove duplicate tableid taosArraySort(tr->deled, uidCompare);
TFileValue* lv = taosArrayGetP(result, sz - 1);
// indexError("merge colVal: %s", lv->colVal);
if (strcmp(lv->colVal, tv->colVal) == 0) {
taosArrayAddAll(lv->tableId, tv->tableId);
tfileValueDestroy(tv);
} else {
taosArrayPush(result, &tv);
}
} else {
taosArrayPush(result, &tv);
}
}
static void sIdxMergeResult(SArray* result, SIdxMergeHelper* mh) {
taosArraySort(mh->total, uidCompare);
taosArraySort(mh->added, uidCompare);
taosArraySort(mh->deled, uidCompare);
SArray* arrs = taosArrayInit(2, sizeof(void*)); SArray* arrs = taosArrayInit(2, sizeof(void*));
taosArrayPush(arrs, &mh->total); taosArrayPush(arrs, &tr->total);
taosArrayPush(arrs, &mh->added); taosArrayPush(arrs, &tr->added);
iUnion(arrs, result); iUnion(arrs, result);
taosArrayDestroy(arrs); taosArrayDestroy(arrs);
iExcept(result, mh->deled); iExcept(result, tr->deled);
} }
static void indexMayMergeToFinalResult(SArray* result, TFileValue* tfv, SIdxMergeHelper* help) { static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTempResult* tr) {
int32_t sz = taosArrayGetSize(result); int32_t sz = taosArrayGetSize(result);
if (sz > 0) { if (sz > 0) {
TFileValue* lv = taosArrayGetP(result, sz - 1); TFileValue* lv = taosArrayGetP(result, sz - 1);
if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) { if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) {
sIdxMergeResult(lv->tableId, help); sIdxTempResultMergeTo(lv->tableId, tr);
sIdxMergeHelperClear(help); sIdxTempResultClear(tr);
taosArrayPush(result, &tfv); taosArrayPush(result, &tfv);
} else if (tfv == NULL) { } else if (tfv == NULL) {
sIdxMergeResult(lv->tableId, help); // handle last iterator
sIdxTempResultMergeTo(lv->tableId, tr);
} else { } else {
// temp result saved in help
tfileValueDestroy(tfv); tfileValueDestroy(tfv);
} }
} else { } else {
taosArrayPush(result, &tfv); taosArrayPush(result, &tfv);
} }
} }
static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxMergeHelper* mh) { static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTempResult* tr) {
char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; char* colVal = (cv != NULL) ? cv->colVal : tv->colVal;
TFileValue* tfv = tfileValueCreate(colVal); TFileValue* tfv = tfileValueCreate(colVal);
indexMayMergeToFinalResult(result, tfv, mh); indexMayMergeTempToFinalResult(result, tfv, tr);
if (cv != NULL) { if (cv != NULL) {
uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0); uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0);
if (cv->type == ADD_VALUE) { if (cv->type == ADD_VALUE) {
INDEX_MERGE_ADD_DEL(mh->deled, mh->added, id) INDEX_MERGE_ADD_DEL(tr->deled, tr->added, id)
} else if (cv->type == DEL_VALUE) { } else if (cv->type == DEL_VALUE) {
INDEX_MERGE_ADD_DEL(mh->added, mh->deled, id) INDEX_MERGE_ADD_DEL(tr->added, tr->deled, id)
} }
} }
if (tv != NULL) { if (tv != NULL) {
taosArrayAddAll(mh->total, tv->val); taosArrayAddAll(tr->total, tv->val);
} }
} }
static void indexDestroyTempResult(SArray* result) { static void indexDestroyFinalResult(SArray* result) {
int32_t sz = result ? taosArrayGetSize(result) : 0; int32_t sz = result ? taosArrayGetSize(result) : 0;
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
TFileValue* tv = taosArrayGetP(result, i); TFileValue* tv = taosArrayGetP(result, i);
@ -543,7 +522,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
bool cn = cacheIter ? cacheIter->next(cacheIter) : false; bool cn = cacheIter ? cacheIter->next(cacheIter) : false;
bool tn = tfileIter ? tfileIter->next(tfileIter) : false; bool tn = tfileIter ? tfileIter->next(tfileIter) : false;
SIdxMergeHelper* help = sIdxMergeHelperCreate(); SIdxTempResult* tr = sIdxTempResultCreate();
while (cn == true || tn == true) { while (cn == true || tn == true) {
IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL; IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL;
IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL; IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL;
@ -557,21 +536,22 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
comp = 1; comp = 1;
} }
if (comp == 0) { if (comp == 0) {
indexMergeCacheAndTFile(result, cv, tv, help); indexMergeCacheAndTFile(result, cv, tv, tr);
cn = cacheIter->next(cacheIter); cn = cacheIter->next(cacheIter);
tn = tfileIter->next(tfileIter); tn = tfileIter->next(tfileIter);
} else if (comp < 0) { } else if (comp < 0) {
indexMergeCacheAndTFile(result, cv, NULL, help); indexMergeCacheAndTFile(result, cv, NULL, tr);
cn = cacheIter->next(cacheIter); cn = cacheIter->next(cacheIter);
} else { } else {
indexMergeCacheAndTFile(result, NULL, tv, help); indexMergeCacheAndTFile(result, NULL, tv, tr);
tn = tfileIter->next(tfileIter); tn = tfileIter->next(tfileIter);
} }
} }
indexMayMergeToFinalResult(result, NULL, help); indexMayMergeTempToFinalResult(result, NULL, tr);
sIdxTempResultDestroy(tr);
int ret = indexGenTFile(sIdx, pCache, result); int ret = indexGenTFile(sIdx, pCache, result);
indexDestroyTempResult(result); indexDestroyFinalResult(result);
indexCacheDestroyImm(pCache); indexCacheDestroyImm(pCache);
@ -581,8 +561,6 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
tfileReaderUnRef(pReader); tfileReaderUnRef(pReader);
indexCacheUnRef(pCache); indexCacheUnRef(pCache);
sIdxMergeHelperDestroy(help);
int64_t cost = taosGetTimestampUs() - st; int64_t cost = taosGetTimestampUs() - st;
if (ret != 0) { if (ret != 0) {
indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000); indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000);

1
source/libs/index/src/index_tfile.c
View File

@ -102,7 +102,6 @@ void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { if (tcache == NULL) {
return; return;
} }
// free table cache // free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) { while (reader) {

16
source/libs/index/test/jsonUT.cc
View File

@ -105,6 +105,22 @@ TEST_F(JsonEnv, testWriteMillonData) {
} }
indexMultiTermDestroy(terms); indexMultiTermDestroy(terms);
} }
{
std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx");
for (uint i = 0; i < 1000; i++) {
colVal[i % colVal.size()] = '0' + i % 128;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
}
{ {
std::string colName("voltagefdadfa"); std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx"); std::string colVal("abxxxxxxxxxxxx");