From f5c13c13c6d1a6ddfd3a3ff18ae979cba342bd9a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 1 Mar 2022 22:36:19 +0800 Subject: [PATCH 01/13] support json --- include/libs/index/index.h | 114 +++++++++++++++++++++++++--- source/libs/index/inc/indexInt.h | 57 +++++++++----- source/libs/index/src/index.c | 49 +++++++++--- source/libs/index/src/index_cache.c | 4 +- source/libs/index/src/index_json.c | 44 +++++++++++ source/libs/index/src/index_tfile.c | 13 ++-- source/libs/index/test/fstUT.cc | 10 +-- source/libs/index/test/jsonDemo.cc | 0 8 files changed, 241 insertions(+), 50 deletions(-) create mode 100644 source/libs/index/src/index_json.c create mode 100644 source/libs/index/test/jsonDemo.cc diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 47eb97cc3a..9424eae1b7 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -29,6 +29,12 @@ typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; +typedef struct SIndex SIndexJson; +typedef struct SIndexTerm SIndexJsonTerm; +typedef struct SIndexOpts SIndexJsonOpts; +typedef struct SIndexMultiTermQuery SIndexJsonMultiTermQuery; +typedef struct SArray SIndexJsonMultiTerm; + typedef enum { ADD_VALUE, // add index colume value DEL_VALUE, // delete index column value @@ -39,24 +45,108 @@ typedef enum { } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType; + /* - * @param: oper - * + * create multi query + * @param oper (input, relation between querys) */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); + /* - * @param: - * @param: + * destroy multi query + * @param pQuery (input, multi-query-object to be destory) + */ + +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); +/* + * add query to multi query + * @param pQuery (input, multi-query-object) + * @param term (input, single query term) + * @param type (input, single query type) + * @return error code + */ +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); +/* + * open index + * @param opt (input, index opt) + * @param path (input, index path) + * @param index (output, index object) + * @return error code + */ +int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); +/* + * close index + * @param index (input, index to be closed) + * @return error code */ -int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); void indexClose(SIndex* index); -int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); -int indexDelete(SIndex* index, SIndexMultiTermQuery* query); -int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); -int indexRebuild(SIndex* index, SIndexOpts* opt); + +/* + * insert terms into index + * @param index (input, index object) + * @param term (input, terms inserted into index) + * @param uid (input, uid of terms) + * @return error code + */ +int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); +/* + * delete terms that meet query condition + * @param index (input, index object) + * @param query (input, condition query to deleted) + * @return error code + */ + +int indexDelete(SIndex* index, SIndexMultiTermQuery* query); +/* + * search index + * @param index (input, index object) + * @param query (input, multi query condition) + * @param result(output, query result) + * @return error code + */ +int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); +/* + * rebuild index + * @param index (input, index object) + * @parma opt (input, rebuild index opts) + * @return error code + */ +int indexRebuild(SIndex* index, SIndexOpts* opt); + +/* + * open index + * @param opt (input,index json opt) + * @param path (input, index json path) + * @param index (output, index json object) + * @return error code + */ +int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index); +/* + * close index + * @param index (input, index to be closed) + * @return error code + */ + +int tIndexJsonClose(SIndexJson* index); + +/* + * insert terms into index + * @param index (input, index object) + * @param term (input, terms inserted into index) + * @param uid (input, uid of terms) + * @return error code + */ +int tIndexJsonPut(SIndexJson* index, SIndexJsonMultiTerm* terms, uint64_t uid); +/* + * search index + * @param index (input, index object) + * @param query (input, multi query condition) + * @param result(output, query result) + * @return error code + */ + +int tIndexJsonSearch(SIndexJson* index, SIndexJsonMultiTermQuery* query, SArray* result); /* * @param * @param diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 90ad1e15f4..57d587d297 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -120,29 +120,50 @@ int indexFlushCacheToTFile(SIndex* sIdx, void*); int32_t indexSerialCacheKey(ICacheKey* key, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \ +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); } \ +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); } \ +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); } \ +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", 255, __VA_ARGS__); \ + } \ } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); } \ +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ + do { \ + uint8_t oldTy = ty; \ + ty = (ty >> 4) | exTy; \ + ty = (ty << 4) | oldTy; \ } while (0) #ifdef __cplusplus diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 5147734a85..b518df2ea2 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -2,8 +2,8 @@ * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or @@ -30,6 +30,8 @@ void* indexQhandle = NULL; +static char JSON_COLUMN[] = "JSON"; + void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); @@ -63,6 +65,9 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); static void indexMergeSameKey(SArray* result, TFileValue* tv); +static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); +int32_t indexSerialKey(ICacheKey* key, char* buf); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); @@ -147,9 +152,8 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + char buf[128] = {0}; + int32_t sz = indexSerialTermKey(p, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { @@ -162,9 +166,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + char buf[128] = {0}; + // ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; + int32_t sz = indexSerialTermKey(p, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -554,7 +558,24 @@ END: return -1; } -int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +int32_t indexSerialTermKeyOfTag(SIndexTerm* p, char* buf) { + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; + return indexSerialKey(&key, buf); +} +int32_t indexSerilaTermKeyOfJson(SIndexTerm* p, char* buf) { + ICacheKey key = {.suid = p->suid, .colName = JSON_COLUMN, .nColName = strlen(JSON_COLUMN)}; + return indexSerialKey(&key, buf); +} + +int32_t indexSerialTermKey(SIndexTerm* itm, char* buf) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(itm->colType, TSDB_DATA_TYPE_JSON); + if (hasJson) { + return indexSerilaTermKeyOfJson(itm, buf); + } else { + return indexSerialTermKeyOfTag(itm, buf); + } +} +int32_t indexSerialKey(ICacheKey* key, char* buf) { char* p = buf; SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_VAR_TO_BUF(buf, '_', char); @@ -563,3 +584,13 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); return buf - p; } + +// int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +// char* p = buf; +// SERIALIZE_MEM_TO_BUF(buf, key, suid); +// SERIALIZE_VAR_TO_BUF(buf, '_', char); +// // SERIALIZE_MEM_TO_BUF(buf, key, colType); +// // SERIALIZE_VAR_TO_BUF(buf, '_', char); +// SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +// return buf - p; +//} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index d6a7141825..bb29a78cb1 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -24,6 +24,8 @@ #define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 +static char JSON_COLUMN[] = "JSON"; + static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -45,7 +47,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in return NULL; }; cache->mem = indexInternalCacheCreate(type); - cache->colName = tstrdup(colName); + cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c new file mode 100644 index 0000000000..f4d28b45b0 --- /dev/null +++ b/source/libs/index/src/index_json.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "indeInt.h" +#include "index.h" + +int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { + // handle + return tIndexOpen(opts, path, index); +} +// k +int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { + for (int i = 0; i < taosArrayGetSize(terms); i++) { + SIndexJsonTerm *p = taosArrayGetP(terms, i); + INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + } + return indexPut(index, terms, uid); + // handle put +} + +int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *query, SArray *result) { + for (int i = 0; i < taosArrayGetSize(terms); i++) { + SIndexJsonTerm *p = taosArrayGetP(terms, i); + INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + } + return indexSearch(index, query, result); + // handle search +} + +int tIndexJsonClose(SIndexJson *index) { + return tIndexClose(index); + // handle close +} diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index e44f8fc1c3..a05884b790 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -205,7 +205,11 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul } else if (qtype == QUERY_PREFIX) { // handle later // - } else { + } else if (qtype == QUERY_SUFFIX) { + // handle later + } else if (qtype == QUERY_REGEX) { + // handle later + } else if (qtype == QUERY_RANGE) { // handle later } tfileReaderUnRef(reader); @@ -586,11 +590,10 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno, + reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d59a3428da..b34fd71e9c 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -216,21 +216,21 @@ class FstEnv : public ::testing::Test { TEST_F(FstEnv, writeNormal) { fst->CreateWriter(); - std::string str("aa"); + std::string str("11"); for (int i = 0; i < 10; i++) { - str[0] = 'a' + i; + str[0] = '1' + i; str.resize(2); assert(fst->Put(str, i) == true); } // order failed - assert(fst->Put("aa", 1) == false); + assert(fst->Put("11", 1) == false); fst->DestroyWriter(); fst->CreateReader(); uint64_t val; - assert(fst->Get("a", &val) == false); - assert(fst->Get("aa", &val) == true); + assert(fst->Get("1", &val) == false); + assert(fst->Get("11", &val) == true); assert(val == 0); std::vector rlt; diff --git a/source/libs/index/test/jsonDemo.cc b/source/libs/index/test/jsonDemo.cc new file mode 100644 index 0000000000..e69de29bb2 From d7c3415ae0a5806f3a8710dc6fb13f4816197c25 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 09:24:17 +0800 Subject: [PATCH 02/13] support json --- source/libs/index/src/index_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c index f4d28b45b0..9ffab3fad1 100644 --- a/source/libs/index/src/index_json.c +++ b/source/libs/index/src/index_json.c @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "indeInt.h" #include "index.h" +#include "indexInt.h" int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { // handle From 4be79789ca9d044df45cd465b85fad63146b647e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 15:05:16 +0800 Subject: [PATCH 03/13] support json --- include/libs/index/index.h | 4 +-- source/libs/index/inc/indexInt.h | 2 ++ source/libs/index/src/index.c | 55 ++++++++++------------------- source/libs/index/src/index_cache.c | 46 +++++++++++++++++++----- source/libs/index/src/index_json.c | 12 +++---- 5 files changed, 65 insertions(+), 54 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 9424eae1b7..453b49e4c6 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -125,10 +125,10 @@ int tIndexJsonOpen(SIndexJsonOpts* opts, const char* path, SIndexJson** index); /* * close index * @param index (input, index to be closed) - * @return error code + * @return void */ -int tIndexJsonClose(SIndexJson* index); +void tIndexJsonClose(SIndexJson* index); /* * insert terms into index diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 57d587d297..928e8b6102 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -119,6 +119,8 @@ typedef struct TFileCacheKey { int indexFlushCacheToTFile(SIndex* sIdx, void*); int32_t indexSerialCacheKey(ICacheKey* key, char* buf); +// int32_t indexSerialKey(ICacheKey* key, char* buf); +// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); #define indexFatal(...) \ do { \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b518df2ea2..168e819073 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -65,8 +65,8 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); static void indexMergeSameKey(SArray* result, TFileValue* tv); -static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -int32_t indexSerialKey(ICacheKey* key, char* buf); +// static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); +// int32_t indexSerialKey(ICacheKey* key, char* buf); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); @@ -152,8 +152,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - int32_t sz = indexSerialTermKey(p, buf); + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { @@ -166,9 +167,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); - char buf[128] = {0}; - // ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - int32_t sz = indexSerialTermKey(p, buf); + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -334,8 +335,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result IndexCache* cache = NULL; char buf[128] = {0}; - ICacheKey key = {.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName)}; - int32_t sz = indexSerialCacheKey(&key, buf); + ICacheKey key = { + .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); pthread_mutex_lock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); @@ -558,39 +560,18 @@ END: return -1; } -int32_t indexSerialTermKeyOfTag(SIndexTerm* p, char* buf) { - ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; - return indexSerialKey(&key, buf); -} -int32_t indexSerilaTermKeyOfJson(SIndexTerm* p, char* buf) { - ICacheKey key = {.suid = p->suid, .colName = JSON_COLUMN, .nColName = strlen(JSON_COLUMN)}; - return indexSerialKey(&key, buf); -} +int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); -int32_t indexSerialTermKey(SIndexTerm* itm, char* buf) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(itm->colType, TSDB_DATA_TYPE_JSON); - if (hasJson) { - return indexSerilaTermKeyOfJson(itm, buf); - } else { - return indexSerialTermKeyOfTag(itm, buf); - } -} -int32_t indexSerialKey(ICacheKey* key, char* buf) { char* p = buf; SERIALIZE_MEM_TO_BUF(buf, key, suid); SERIALIZE_VAR_TO_BUF(buf, '_', char); // SERIALIZE_MEM_TO_BUF(buf, key, colType); // SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); + if (hasJson) { + SERIALIZE_STR_VAR_TO_BUF(buf, JSON_COLUMN, strlen(JSON_COLUMN)); + } else { + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); + } return buf - p; } - -// int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { -// char* p = buf; -// SERIALIZE_MEM_TO_BUF(buf, key, suid); -// SERIALIZE_VAR_TO_BUF(buf, '_', char); -// // SERIALIZE_MEM_TO_BUF(buf, key, colType); -// // SERIALIZE_VAR_TO_BUF(buf, '_', char); -// SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); -// return buf - p; -//} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index bb29a78cb1..a1f074feae 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -25,6 +25,7 @@ #define MEM_ESTIMATE_RADIO 1.5 static char JSON_COLUMN[] = "JSON"; +static char JSON_VALUE_DELIM = '&'; static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -46,6 +47,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in indexError("failed to create index cache"); return NULL; }; + cache->mem = indexInternalCacheCreate(type); cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; @@ -209,11 +211,38 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } } } +static char* indexCachePackJsonData(SIndexTerm* itm) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)calloc(1, sz); + char* p = buf; + + memcpy(p, itm->colVal, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, itm->colVal, itm->nColVal); + + return buf; +} int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); IndexCache* pCache = cache; indexCacheRef(pCache); @@ -224,8 +253,12 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { } // set up key ct->colType = term->colType; - ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); - memcpy(ct->colVal, term->colVal, term->nColVal); + if (hasJson) { + ct->colVal = indexCachePackJsonData(term); + } else { + ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); + memcpy(ct->colVal, term->colVal, term->nColVal); + } ct->version = atomic_add_fetch_32(&pCache->version, 1); // set value ct->uid = uid; @@ -369,6 +402,8 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { } static MemTable* indexInternalCacheCreate(int8_t type) { + type = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type; + MemTable* tbl = calloc(1, sizeof(MemTable)); indexMemRef(tbl); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -391,9 +426,6 @@ static bool indexCacheIteratorNext(Iterate* itera) { IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); - // IterateValue* iv = &itera->val; - // IterateValue tIterVal = {.colVal = NULL, .val = taosArrayInit(1, sizeof(uint64_t))}; - bool next = tSkipListIterNext(iter); if (next) { SSkipListNode* node = tSkipListIterGet(iter); @@ -413,10 +445,6 @@ static bool indexCacheIteratorNext(Iterate* itera) { taosArrayPush(iv->val, &ct->uid); } - // IterateValue* iv = &itera->val; - // iterateValueDestroy(iv, true); - //*iv = tIterVal; - return next; } diff --git a/source/libs/index/src/index_json.c b/source/libs/index/src/index_json.c index 9ffab3fad1..de88ff3c8a 100644 --- a/source/libs/index/src/index_json.c +++ b/source/libs/index/src/index_json.c @@ -17,9 +17,8 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) { // handle - return tIndexOpen(opts, path, index); + return indexOpen(opts, path, index); } -// k int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { for (int i = 0; i < taosArrayGetSize(terms); i++) { SIndexJsonTerm *p = taosArrayGetP(terms, i); @@ -29,16 +28,17 @@ int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { // handle put } -int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *query, SArray *result) { +int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *result) { + SArray *terms = tq->query; for (int i = 0; i < taosArrayGetSize(terms); i++) { SIndexJsonTerm *p = taosArrayGetP(terms, i); INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } - return indexSearch(index, query, result); + return indexSearch(index, tq, result); // handle search } -int tIndexJsonClose(SIndexJson *index) { - return tIndexClose(index); +void tIndexJsonClose(SIndexJson *index) { + return indexClose(index); // handle close } From 6a6f31c4a75dd30d916b7b31512f7620b6af1d02 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 17:09:22 +0800 Subject: [PATCH 04/13] support json --- source/libs/index/src/index_cache.c | 13 +++- source/libs/index/test/CMakeLists.txt | 18 ++++++ source/libs/index/test/jsonUT.cc | 92 +++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 source/libs/index/test/jsonUT.cc diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index a1f074feae..2742830d3b 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -222,7 +222,7 @@ static char* indexCachePackJsonData(SIndexTerm* itm) { char* buf = (char*)calloc(1, sz); char* p = buf; - memcpy(p, itm->colVal, itm->nColName); + memcpy(p, itm->colName, itm->nColName); p += itm->nColName; memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); @@ -329,13 +329,22 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + char* p = term->colVal; + if (hasJson) { + p = indexCachePackJsonData(term); + } + CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; int ret = indexQueryMem(mem, &ct, qtype, result, s); if (ret == 0 && *s != kTypeDeletion) { // continue search in imm ret = indexQueryMem(imm, &ct, qtype, result, s); } + if (hasJson) { + tfree(p); + } indexMemUnRef(mem); indexMemUnRef(imm); diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index bed42be3e5..1ebf85368b 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -2,6 +2,7 @@ add_executable(indexTest "") add_executable(fstTest "") add_executable(fstUT "") add_executable(UtilUT "") +add_executable(jsonUT "") target_sources(indexTest PRIVATE @@ -21,6 +22,10 @@ target_sources(UtilUT "utilUT.cc" ) +target_sources(jsonUT + PRIVATE + "jsonUT.cc" +) target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -43,6 +48,12 @@ target_include_directories ( UtilUT "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories (jsonUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -73,6 +84,13 @@ target_link_libraries (UtilUT index ) +target_link_libraries (jsonUT + os + util + common + gtest_main + index +) #add_test( # NAME index_test diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc new file mode 100644 index 0000000000..20b59add9f --- /dev/null +++ b/source/libs/index/test/jsonUT.cc @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +static std::string dir = "/tmp/json"; +class JsonEnv : public ::testing::Test { + protected: + virtual void SetUp() { + taosRemoveDir(dir.c_str()); + opts = indexOptsCreate(); + int ret = tIndexJsonOpen(opts, dir.c_str(), &index); + assert(ret == 0); + } + virtual void TearDown() { + tIndexJsonClose(index); + indexOptsDestroy(opts); + } + SIndexJsonOpts* opts; + SIndexJson* index; +}; + +TEST_F(JsonEnv, testWrite) { + { + std::string colName("voltage"); + std::string colVal("ab"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("ab1"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("123"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("ab"); + + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SArray* result = taosArrayInit(1, sizeof(uint64_t)); + indexMultiTermQueryAdd(mq, q, QUERY_TERM); + tIndexJsonSearch(index, mq, result); + assert(100 == taosArrayGetSize(result)); + indexMultiTermQueryDestroy(mq); + } + + // SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; +} From 7ae87c10103c801d06092ea3ddea00b9e9b671bd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 2 Mar 2022 22:06:02 +0800 Subject: [PATCH 05/13] add UT --- source/libs/index/inc/index_comm.h | 32 +++++++++++++++++++ source/libs/index/src/index_cache.c | 35 ++------------------ source/libs/index/src/index_comm.c | 48 ++++++++++++++++++++++++++++ source/libs/index/src/index_tfile.c | 13 +++++++- source/libs/index/test/fstUT.cc | 16 ++++++++++ source/libs/index/test/jsonDemo.cc | 0 source/libs/index/test/jsonUT.cc | 33 +++++++++++++++++-- source/libs/transport/src/transSrv.c | 20 ++++++------ 8 files changed, 153 insertions(+), 44 deletions(-) create mode 100644 source/libs/index/inc/index_comm.h create mode 100644 source/libs/index/src/index_comm.c delete mode 100644 source/libs/index/test/jsonDemo.cc diff --git a/source/libs/index/inc/index_comm.h b/source/libs/index/inc/index_comm.h new file mode 100644 index 0000000000..0d8418ba65 --- /dev/null +++ b/source/libs/index/inc/index_comm.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_INDEX_COMM_H_ +#define _TD_INDEX_COMM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char JSON_COLUMN[]; +extern char JSON_VALUE_DELIM; + +char* indexPackJsonData(SIndexTerm* itm); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 2742830d3b..599bac3fe6 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -14,6 +14,7 @@ */ #include "index_cache.h" +#include "index_comm.h" #include "index_util.h" #include "tcompare.h" #include "tsched.h" @@ -24,9 +25,6 @@ #define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 -static char JSON_COLUMN[] = "JSON"; -static char JSON_VALUE_DELIM = '&'; - static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -211,33 +209,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } } } -static char* indexCachePackJsonData(SIndexTerm* itm) { - /* - * |<-----colname---->|<-----dataType---->|<--------colVal---------->| - * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| - */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); - - int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; - char* buf = (char*)calloc(1, sz); - char* p = buf; - - memcpy(p, itm->colName, itm->nColName); - p += itm->nColName; - - memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); - p += sizeof(JSON_VALUE_DELIM); - - memcpy(p, &ty, sizeof(ty)); - p += sizeof(ty); - - memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); - p += sizeof(JSON_VALUE_DELIM); - - memcpy(p, itm->colVal, itm->nColVal); - - return buf; -} int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; @@ -254,7 +225,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // set up key ct->colType = term->colType; if (hasJson) { - ct->colVal = indexCachePackJsonData(term); + ct->colVal = indexPackJsonData(term); } else { ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); memcpy(ct->colVal, term->colVal, term->nColVal); @@ -333,7 +304,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); char* p = term->colVal; if (hasJson) { - p = indexCachePackJsonData(term); + p = indexPackJsonData(term); } CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; diff --git a/source/libs/index/src/index_comm.c b/source/libs/index/src/index_comm.c new file mode 100644 index 0000000000..4f3cbaa4da --- /dev/null +++ b/source/libs/index/src/index_comm.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "index.h" +#include "indexInt.h" + +char JSON_COLUMN[] = "JSON"; +char JSON_VALUE_DELIM = '&'; + +char* indexPackJsonData(SIndexTerm* itm) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)calloc(1, sz); + char* p = buf; + + memcpy(p, itm->colName, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, itm->colVal, itm->nColVal); + + return buf; +} diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a05884b790..a2089e8eee 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -15,6 +15,7 @@ p * #include "index_tfile.h" #include "index.h" +#include "index_comm.h" #include "index_fst.h" #include "index_fst_counting_writer.h" #include "index_util.h" @@ -186,13 +187,20 @@ void tfileReaderDestroy(TFileReader* reader) { int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) { SIndexTerm* term = query->term; + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); EIndexQueryType qtype = query->qType; int ret = -1; // refactor to callback later if (qtype == QUERY_TERM) { uint64_t offset; - FstSlice key = fstSliceCreate(term->colVal, term->nColVal); + char* p = term->colVal; + uint64_t sz = term->nColVal; + if (hasJson) { + p = indexPackJsonData(term); + sz = strlen(p); + } + FstSlice key = fstSliceCreate(p, sz); if (fstGet(reader->fst, &key, &offset)) { indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); @@ -202,6 +210,9 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul term->colVal); } fstSliceDestroy(&key); + if (hasJson) { + free(p); + } } else if (qtype == QUERY_PREFIX) { // handle later // diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index b34fd71e9c..3bc3b6da6a 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -238,3 +238,19 @@ TEST_F(FstEnv, writeNormal) { assert(fst->Search(ctx, rlt) == true); } TEST_F(FstEnv, WriteMillonrRecord) {} +TEST_F(FstEnv, writeAbNormal) { + fst->CreateWriter(); + std::string str1("voltage&\b&ab"); + std::string str2("voltbge&\b&ab"); + + fst->Put(str1, 1); + fst->Put(str2, 2); + + fst->DestroyWriter(); + + fst->CreateReader(); + uint64_t val; + assert(fst->Get("1", &val) == false); + assert(fst->Get("voltage&\b&ab", &val) == true); + assert(val == 1); +} diff --git a/source/libs/index/test/jsonDemo.cc b/source/libs/index/test/jsonDemo.cc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 20b59add9f..e184bc49ae 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -21,6 +21,8 @@ class JsonEnv : public ::testing::Test { protected: virtual void SetUp() { taosRemoveDir(dir.c_str()); + taosMkDir(dir.c_str()); + opts = indexOptsCreate(); int ret = tIndexJsonOpen(opts, dir.c_str(), &index); assert(ret == 0); @@ -87,6 +89,33 @@ TEST_F(JsonEnv, testWrite) { assert(100 == taosArrayGetSize(result)); indexMultiTermQueryDestroy(mq); } - - // SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; +} +TEST_F(JsonEnv, testWriteMillonData) { + { + std::string colName("voltagefdadfa"); + std::string colVal("abxxxxxxxxxxxx"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + { + std::string colName("voltage"); + std::string colVal("ab"); + + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SArray* result = taosArrayInit(1, sizeof(uint64_t)); + indexMultiTermQueryAdd(mq, q, QUERY_TERM); + tIndexJsonSearch(index, mq, result); + assert(100 == taosArrayGetSize(result)); + indexMultiTermQueryDestroy(mq); + } } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index f0db054797..c7b6ca2a2c 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -286,15 +286,17 @@ void uvOnWriteCb(uv_write_t* req, int status) { transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); - assert(taosArrayGetSize(conn->srvMsgs) >= 1); - SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); - taosArrayRemove(conn->srvMsgs, 0); - destroySmsg(msg); + if (conn->srvMsgs != NULL) { + assert(taosArrayGetSize(conn->srvMsgs) >= 1); + SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, 0); + taosArrayRemove(conn->srvMsgs, 0); + destroySmsg(msg); - // send second data, just use for push - if (taosArrayGetSize(conn->srvMsgs) > 0) { - msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); - uvStartSendRespInternal(msg); + // send second data, just use for push + if (taosArrayGetSize(conn->srvMsgs) > 0) { + msg = (SSrvMsg*)taosArrayGetP(conn->srvMsgs, 0); + uvStartSendRespInternal(msg); + } } } else { tError("server conn %p failed to write data, %s", conn, uv_err_name(status)); @@ -615,7 +617,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { SSrvMsg* msg = taosArrayGetP(conn->srvMsgs, i); destroySmsg(msg); } - taosArrayDestroy(conn->srvMsgs); + conn->srvMsgs = taosArrayDestroy(conn->srvMsgs); QUEUE_REMOVE(&conn->queue); if (clear) { From 09e6462c5fad9a5612ae64e96b93898e2fbe25a9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 3 Mar 2022 09:09:11 +0800 Subject: [PATCH 06/13] feature/qnode --- include/common/tmsg.h | 4 + source/common/src/tmsg.c | 6 + source/dnode/mgmt/impl/src/dndVnodes.c | 3 + source/dnode/mnode/impl/src/mndVgroup.c | 3 + source/dnode/vnode/inc/vnode.h | 3 + source/dnode/vnode/src/vnd/vnodeMain.c | 3 + source/libs/catalog/inc/catalogInt.h | 39 +- source/libs/catalog/src/catalog.c | 307 ++++++++---- source/libs/catalog/test/catalogTests.cpp | 580 ++++++++++++++++++++-- 9 files changed, 812 insertions(+), 136 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ab0472a575..25d668e788 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -739,6 +739,9 @@ typedef struct { int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; + uint32_t hashBegin; + uint32_t hashEnd; + int8_t hashMethod; int8_t walLevel; int8_t precision; int8_t compression; @@ -749,6 +752,7 @@ typedef struct { int8_t selfIndex; int8_t streamMode; SReplica replicas[TSDB_MAX_REPLICA]; + } SCreateVnodeReq, SAlterVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d8c850c6af..ae31dff310 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2112,6 +2112,9 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeU32(&encoder, pReq->hashBegin) < 0) return -1; + if (tEncodeU32(&encoder, pReq->hashEnd) < 0) return -1; + if (tEncodeI8(&encoder, pReq->hashMethod) < 0) return -1; if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; @@ -2152,6 +2155,9 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->hashBegin) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->hashEnd) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->hashMethod) < 0) return -1; if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index f20493aa7f..2767461f12 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -523,6 +523,9 @@ static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.rollPeriod = 128; pCfg->walCfg.segSize = 128; pCfg->walCfg.vgId = pCreate->vgId; + pCfg->hashBegin = pCreate->hashBegin; + pCfg->hashEnd = pCreate->hashEnd; + pCfg->hashMethod = pCreate->hashMethod; } static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b437b44417..f7b177f170 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -215,6 +215,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.replica = pVgroup->replica; createReq.selfIndex = -1; createReq.streamMode = pVgroup->streamMode; + createReq.hashBegin = pVgroup->hashBegin; + createReq.hashEnd = pVgroup->hashEnd; + createReq.hashMethod = pDb->hashMethod; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 31f04e840a..9a4f920499 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -57,6 +57,9 @@ typedef struct { SMetaCfg metaCfg; STqCfg tqCfg; SWalCfg walCfg; + uint32_t hashBegin; + uint32_t hashEnd; + int8_t hashMethod; } SVnodeCfg; typedef struct { diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index c748907d6c..ba346064ae 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -30,6 +30,9 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { cfg.pDnode = pVnodeCfg->pDnode; cfg.pTfs = pVnodeCfg->pTfs; cfg.dbId = pVnodeCfg->dbId; + cfg.hashBegin = pVnodeCfg->hashBegin; + cfg.hashEnd = pVnodeCfg->hashEnd; + cfg.hashMethod = pVnodeCfg->hashMethod; } // Validate options diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index e71f559ad9..c4f1a117fe 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -60,6 +60,7 @@ typedef struct SCtgDebug { bool lockDebug; bool cacheDebug; bool apiDebug; + bool metaDebug; uint32_t showCachePeriodSec; } SCtgDebug; @@ -119,6 +120,10 @@ typedef struct SCatalogStat { SCtgCacheStat cache; } SCatalogStat; +typedef struct SCtgUpdateMsgHeader { + SCatalog* pCtg; +} SCtgUpdateMsgHeader; + typedef struct SCtgUpdateVgMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -145,6 +150,14 @@ typedef struct SCtgRemoveStbMsg { uint64_t suid; } SCtgRemoveStbMsg; +typedef struct SCtgRemoveTblMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + uint64_t dbId; +} SCtgRemoveTblMsg; + + typedef struct SCtgMetaAction { int32_t act; void *data; @@ -189,19 +202,21 @@ typedef struct SCtgAction { #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) #define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE) -#define CTG_FLAG_STB 0x1 -#define CTG_FLAG_NOT_STB 0x2 -#define CTG_FLAG_UNKNOWN_STB 0x4 -#define CTG_FLAG_INF_DB 0x8 +#define CTG_FLAG_STB 0x1 +#define CTG_FLAG_NOT_STB 0x2 +#define CTG_FLAG_UNKNOWN_STB 0x4 +#define CTG_FLAG_INF_DB 0x8 +#define CTG_FLAG_FORCE_UPDATE 0x10 -#define CTG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) -#define CTG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) -#define CTG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) -#define CTG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) -#define CTG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) -#define CTG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) -#define CTG_GEN_STB_FLAG(_isStb) ((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB) -#define CTG_TBTYPE_MATCH(_flag, tbType) (CTG_IS_UNKNOWN_STB(_flag) || (CTG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) +#define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) +#define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) +#define CTG_FLAG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) +#define CTG_FLAG_IS_FORCE_UPDATE(_flag) ((_flag) & CTG_FLAG_FORCE_UPDATE) +#define CTG_FLAG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) +#define CTG_FLAG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) +#define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) +#define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) #define CTG_IS_INF_DBNAME(_dbname) ((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f0ea51c2f9..5779906761 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -72,6 +72,12 @@ int32_t ctgDbgEnableDebug(char *option) { return TSDB_CODE_SUCCESS; } + if (0 == strcasecmp(option, "meta")) { + gCTGDebug.metaDebug = true; + qDebug("api debug enabled"); + return TSDB_CODE_SUCCESS; + } + qError("invalid debug option:%s", option); return TSDB_CODE_CTG_INTERNAL_ERROR; @@ -148,9 +154,30 @@ int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) { return num; } +void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { + if (!gCTGDebug.metaDebug) { + return; + } -void ctgDbgShowDBCache(SHashObj *dbHash) { - if (NULL == dbHash) { + STableComInfo *c = &p->tableInfo; + + if (TSDB_CHILD_TABLE == p->tableType) { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); + return; + } else { + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); + } + + int32_t colNum = c->numOfColumns + c->numOfTags; + for (int32_t i = 0; i < colNum; ++i) { + SSchema *s = &p->schema[i]; + ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + } +} + +void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { + if (NULL == dbHash || !gCTGDebug.cacheDebug) { return; } @@ -164,31 +191,24 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { dbCache = (SCtgDBCache *)pIter; taosHashGetKey(dbCache, (void **)&dbFName, &len); - - CTG_CACHE_DEBUG("** %dth db [%.*s][%"PRIx64"] **", i, (int32_t)len, dbFName, dbCache->dbId); - CTG_CACHE_DEBUG("deleted: %d", dbCache->deleted); + int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; + int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; + int32_t hashMethod = -1; + int32_t vgNum = 0; + if (dbCache->vgInfo) { - CTG_CACHE_DEBUG("vgVersion: %d", dbCache->vgInfo->vgVersion); - CTG_CACHE_DEBUG("hashMethod: %d", dbCache->vgInfo->hashMethod); + vgVersion = dbCache->vgInfo->vgVersion; + hashMethod = dbCache->vgInfo->hashMethod; if (dbCache->vgInfo->vgHash) { - CTG_CACHE_DEBUG("vgNum: %d", taosHashGetSize(dbCache->vgInfo->vgHash)); - //TODO - } else { - CTG_CACHE_DEBUG("vgHash: %p", dbCache->vgInfo->vgHash); + vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); } - } else { - CTG_CACHE_DEBUG("vgInfo: %p", dbCache->vgInfo); - } - - if (dbCache->tbCache.metaCache) { - CTG_CACHE_DEBUG("metaNum: %d", taosHashGetSize(dbCache->tbCache.metaCache)); - } - - if (dbCache->tbCache.stbCache) { - CTG_CACHE_DEBUG("stbNum: %d", taosHashGetSize(dbCache->tbCache.stbCache)); - } + } + ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); + pIter = taosHashIterate(dbHash, pIter); } } @@ -197,15 +217,15 @@ void ctgDbgShowDBCache(SHashObj *dbHash) { void ctgDbgShowClusterCache(SCatalog* pCtg) { - if (NULL == pCtg) { + if (!gCTGDebug.cacheDebug || NULL == pCtg) { return; } - CTG_CACHE_DEBUG("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); - CTG_CACHE_DEBUG("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); + ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); - ctgDbgShowDBCache(pCtg->dbCache); + ctgDbgShowDBCache(pCtg, pCtg->dbCache); } @@ -292,6 +312,66 @@ _return: } +int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; + SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + msg->dbId = dbId; + msg->suid = suid; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(action.data); + CTG_RET(code); +} + + + +int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL}; + SCtgRemoveTblMsg *msg = malloc(sizeof(SCtgRemoveTblMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + strncpy(msg->tbName, tbName, sizeof(msg->tbName)); + msg->dbId = dbId; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(&action)); + + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(action.data); + CTG_RET(code); +} + + void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { @@ -554,7 +634,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); @@ -562,7 +642,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_IS_INF_DB(flag)) { + if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); } else { tNameGetFullDbName(pTableName, dbFName); @@ -590,6 +670,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } *exist = 1; + if (dbId) { + *dbId = dbCache->dbId; + } tbMeta = *pTableMeta; @@ -646,7 +729,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_IS_INF_DB(flag)) { + if (CTG_FLAG_IS_INF_DB(flag)) { strcpy(dbFName, pTableName->dbname); } else { tNameGetFullDbName(pTableName, dbFName); @@ -1304,18 +1387,21 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (orig) { origType = orig->tableType; - origSuid = orig->suid; - if (origType == TSDB_SUPER_TABLE && ((!isStb) || origSuid != meta->suid)) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + if (origType == TSDB_SUPER_TABLE) { + if ((!isStb) || orig->suid != meta->suid) { + CTG_LOCK(CTG_WRITE, &tbCache->stbLock); + if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + } + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); + } + + origSuid = orig->suid; } } @@ -1334,13 +1420,14 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDbgShowTableMeta(pCtg, tbName, meta); if (!isStb) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - if (isStb && origSuid == meta->suid) { + if (origType == TSDB_SUPER_TABLE && origSuid == meta->suid) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; @@ -1506,7 +1593,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - if (!CTG_IS_INF_DB(flag)) { + if (!CTG_FLAG_IS_INF_DB(flag)) { CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); } @@ -1518,11 +1605,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (CTG_IS_INF_DB(flag)) { + if (CTG_FLAG_IS_INF_DB(flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); - } else if (CTG_IS_STB(flag)) { + } else if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode @@ -1538,14 +1625,17 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); tfree(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); + } + if (0 == exist) { CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); @@ -1606,35 +1696,40 @@ _return: CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t flag) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; int32_t code = 0; + uint64_t dbId = 0; + uint64_t suid = 0; + STableMetaOutput *output = NULL; if (CTG_IS_INF_DBNAME(pTableName->dbname)) { - CTG_SET_INF_DB(flag); + CTG_FLAG_SET_INF_DB(flag); } - if ((!forceUpdate) || (CTG_IS_INF_DB(flag))) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag)); + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId)); - if (exist && CTG_TBTYPE_MATCH(flag, (*pTableMeta)->tableType)) { - return TSDB_CODE_SUCCESS; + int32_t tbType = 0; + + if (exist) { + if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) { + goto _return; } - tfree(*pTableMeta); - } else if (CTG_IS_UNKNOWN_STB(flag)) { - int32_t tbType = 0; - - CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType, flag)); + tbType = (*pTableMeta)->tableType; + suid = (*pTableMeta)->suid; - CTG_SET_STB(flag, tbType); + tfree(*pTableMeta); + } + + if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { + CTG_FLAG_SET_STB(flag, tbType); } - STableMetaOutput *output = NULL; while (true) { CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output)); @@ -1662,7 +1757,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL)); if (0 == exist) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; @@ -1675,10 +1770,26 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons _return: + if (CTG_TABLE_NOT_EXIST(code) && exist) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + if (CTG_FLAG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } + + if (TSDB_SUPER_TABLE == tbType) { + ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid); + } else { + ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname); + } + } + tfree(output); if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); + ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta); } CTG_RET(code); @@ -1694,7 +1805,7 @@ int32_t ctgActUpdateVg(SCtgMetaAction *action) { _return: - tfree(msg->dbInfo); + ctgFreeVgInfo(msg->dbInfo); tfree(msg); CTG_RET(code); @@ -1780,7 +1891,6 @@ _return: int32_t ctgActRemoveStb(SCtgMetaAction *action) { int32_t code = 0; SCtgRemoveStbMsg *msg = action->data; - bool removed = false; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; @@ -1826,7 +1936,36 @@ _return: } int32_t ctgActRemoveTbl(SCtgMetaAction *action) { + int32_t code = 0; + SCtgRemoveTblMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId != msg->dbId) { + ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + +_return: + + tfree(msg); + + CTG_RET(code); } @@ -1846,12 +1985,15 @@ void* ctgUpdateThreadFunc(void* param) { SCtgMetaAction *action = NULL; ctgPopAction(&action); + SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; - qDebug("process %s action", gCtgAction[action->act].name); + ctgDebug("process [%s] action", gCtgAction[action->act].name); (*gCtgAction[action->act].func)(action); CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); + + ctgDbgShowClusterCache(pCtg); } CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); @@ -2121,22 +2263,20 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->dbId = dbId; msg->dbInfo = dbInfo; - dbInfo = NULL; action.data = msg; CTG_ERR_JRET(ctgPushAction(&action)); + dbInfo = NULL; + ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); CTG_API_LEAVE(code); _return: - if (dbInfo) { - taosHashCleanup(dbInfo->vgHash); - tfree(dbInfo); - } + ctgFreeVgInfo(dbInfo); tfree(msg); @@ -2179,31 +2319,12 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; - SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); - CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); - } - - msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->stbName, stbName, sizeof(msg->stbName)); - msg->dbId = dbId; - msg->suid = suid; - - action.data = msg; - - CTG_ERR_JRET(ctgPushAction(&action)); - - ctgDebug("action [%s] added into queue", gCtgAction[action.act].name); + CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return: - tfree(action.data); - CTG_API_LEAVE(code); } @@ -2211,13 +2332,13 @@ _return: int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_STB)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); } int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { @@ -2279,13 +2400,13 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_GEN_STB_FLAG(isSTable), NULL)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL)); } int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, true, pTableMeta, CTG_GEN_STB_FLAG(isSTable))); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); } int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { @@ -2309,7 +2430,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -2441,7 +2562,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, false, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b7432429f4..b417a645be 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist, int32_t flag); + int32_t *exist, int32_t flag, uint64_t *dbId); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgDbgEnableDebug(char *option); @@ -57,12 +57,14 @@ enum { CTGT_RSP_CTBMETA, CTGT_RSP_STBMETA, CTGT_RSP_MSTBMETA, + CTGT_RSP_TBMETA_NOT_EXIST, }; bool ctgTestStop = false; bool ctgTestEnableSleep = false; +bool ctgTestEnableLog = true; bool ctgTestDeadLoop = false; -int32_t ctgTestPrintNum = 200000; +int32_t ctgTestPrintNum = 10000; int32_t ctgTestMTRunSec = 5; int32_t ctgTestCurrentVgVersion = 0; @@ -74,14 +76,18 @@ int32_t ctgTestSVersion = 1; int32_t ctgTestTVersion = 1; int32_t ctgTestSuid = 2; uint64_t ctgTestDbId = 33; +uint64_t ctgTestNormalTblUid = 1; uint64_t ctgTestClusterId = 0x1; char *ctgTestDbname = "1.db1"; char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; char *ctgTestSTablename = "stable1"; +char *ctgTestCurrentCTableName = NULL; +char *ctgTestCurrentTableName = NULL; +char *ctgTestCurrentSTableName = NULL; -int32_t ctgTestRspFunc[10] = {0}; +int32_t ctgTestRspFunc[100] = {0}; int32_t ctgTestRspIdx = 0; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { @@ -123,6 +129,10 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { } void ctgTestInitLogFile() { + if (!ctgTestEnableLog) { + return; + } + const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; @@ -131,6 +141,8 @@ void ctgTestInitLogFile() { strcpy(tsLogDir, "/var/log/taos"); ctgDbgEnableDebug("api"); + ctgDbgEnableDebug("meta"); + ctgDbgEnableDebug("cache"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -321,7 +333,7 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = 0; - metaRsp.tuid = 0x0000000000000001; + metaRsp.tuid = ctgTestNormalTblUid++; metaRsp.vgId = 8; metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); @@ -349,10 +361,15 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * tFreeSTableMetaRsp(&metaRsp); } +void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST; +} + + void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestCTablename); + strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename); strcpy(metaRsp.stbName, ctgTestSTablename); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; @@ -399,7 +416,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestSTablename); + strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename); strcpy(metaRsp.stbName, ctgTestSTablename); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; @@ -409,7 +426,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = ctgTestSuid; - metaRsp.tuid = ctgTestSuid; + metaRsp.tuid = ctgTestSuid++; metaRsp.vgId = 0; metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); @@ -511,6 +528,9 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp case CTGT_RSP_MSTBMETA: ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp); break; + case CTGT_RSP_TBMETA_NOT_EXIST: + ctgTestRspTableMetaNotExist(shandle, pEpSet, pMsg, pRsp); + break; default: break; } @@ -773,7 +793,7 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0); + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL); if (code || 0 == exist) { assert(0); } @@ -828,7 +848,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } -#if 0 +#if 1 TEST(tableMeta, normalTable) { @@ -860,7 +880,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.epset.numOfEps, 3); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { - usleep(10000); + usleep(50000); } ctgTestSetRspTableMeta(); @@ -870,6 +890,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 8); ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); @@ -880,7 +901,7 @@ TEST(tableMeta, normalTable) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -975,7 +996,7 @@ TEST(tableMeta, childTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -994,7 +1015,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tableMeta = NULL; + tfree(tableMeta); strcpy(n.tname, ctgTestSTablename); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -1074,8 +1095,8 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); - ASSERT_EQ(tableMeta->uid, ctgTestSuid); - ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); @@ -1084,7 +1105,7 @@ TEST(tableMeta, superTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -1111,7 +1132,7 @@ TEST(tableMeta, superTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { - usleep(10000); + usleep(50000); } else { break; } @@ -1199,8 +1220,8 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); - ASSERT_EQ(tableMeta->uid, ctgTestSuid); - ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); @@ -1209,21 +1230,21 @@ TEST(tableMeta, rmStbMeta) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } } - code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid); + code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid - 1); ASSERT_EQ(code, 0); while (true) { int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); if (n || m) { - usleep(10000); + usleep(50000); } else { break; } @@ -1269,8 +1290,8 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); - ASSERT_EQ(tableMeta->uid, ctgTestSuid); - ASSERT_EQ(tableMeta->suid, ctgTestSuid); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); @@ -1279,7 +1300,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(10000); + usleep(50000); } else { break; } @@ -1299,7 +1320,7 @@ TEST(tableMeta, updateStbMeta) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(10000); + usleep(50000); } else { break; } @@ -1330,6 +1351,499 @@ TEST(tableMeta, updateStbMeta) { memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); } +TEST(refreshGetMeta, normal2normal) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_TBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); +} + +TEST(refreshGetMeta, normal2notexist) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_TBMETA_NOT_EXIST; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, CTG_ERR_CODE_TABLE_NOT_EXIST); + ASSERT_TRUE(tableMeta == NULL); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); +} + + +TEST(refreshGetMeta, normal2child) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_TBMETA; + ctgTestRspFunc[2] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[3] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentCTableName = ctgTestTablename; + ctgTestCurrentSTableName = ctgTestSTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 8); + ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); + ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + +TEST(refreshGetMeta, stable2child) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[4] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentSTableName = ctgTestTablename; + ctgTestCurrentCTableName = ctgTestTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + ctgTestCurrentSTableName = ctgTestSTablename; + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + +TEST(refreshGetMeta, stable2stable) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_STBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_STBMETA; + ctgTestRspFunc[4] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentSTableName = ctgTestTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + + +TEST(refreshGetMeta, child2stable) { + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + ctgTestRspIdx = 0; + ctgTestRspFunc[0] = CTGT_RSP_VGINFO; + ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; + ctgTestRspFunc[2] = CTGT_RSP_STBMETA; + ctgTestRspFunc[3] = CTGT_RSP_STBMETA; + ctgTestRspFunc[4] = CTGT_RSP_STBMETA; + + ctgTestSetRspByIdx(); + + initQueryModuleMsgHandle(); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + ctgTestCurrentCTableName = ctgTestTablename; + ctgTestCurrentSTableName = ctgTestSTablename; + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); + + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); + } + + STableMeta *tableMeta = NULL; + code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 9); + ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { + usleep(50000); + } + + ctgTestCurrentSTableName = ctgTestTablename; + code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + ASSERT_EQ(code, 0); + ASSERT_EQ(tableMeta->vgId, 0); + ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); + ASSERT_EQ(tableMeta->sversion, ctgTestSVersion); + ASSERT_EQ(tableMeta->tversion, ctgTestTVersion); + ASSERT_EQ(tableMeta->uid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->suid, ctgTestSuid - 1); + ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum); + ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); + ASSERT_EQ(tableMeta->tableInfo.precision, 1); + ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); + tfree(tableMeta); + + catalogDestroy(); + memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + ctgTestCurrentCTableName = NULL; + ctgTestCurrentSTableName = NULL; +} + + TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1499,11 +2013,15 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); - while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM)) { - usleep(10000); + while (true) { + uint64_t n = 0; + ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); + if (n > 0) { + break; + } + usleep(50000); } - code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); @@ -1525,7 +2043,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(10000); + usleep(50000); } else { break; } @@ -1749,7 +2267,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { - usleep(10000); + usleep(50000); } code = catalogGetExpiredDBs(pCtg, &dbs, &num); From 76c4fce8d03c0201ea71bfb5aa42c774b6ea86f9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 11:37:19 +0800 Subject: [PATCH 07/13] ping test --- source/libs/sync/src/syncIO.c | 14 ++++-------- source/libs/sync/src/syncMain.c | 10 --------- source/libs/sync/test/syncPingTest.cpp | 31 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 32 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 8d04b5bb26..3ba145a96b 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -211,23 +211,17 @@ static void *syncIOConsumerFunc(void *param) { if (pRpcMsg->msgType == SYNC_PING) { if (io->FpOnSyncPing != NULL) { SyncPing *pSyncMsg; - - SRpcMsg tmpRpcMsg; - memcpy(&tmpRpcMsg, pRpcMsg, sizeof(SRpcMsg)); - pSyncMsg = syncPingBuild(tmpRpcMsg.contLen); - + pSyncMsg = syncPingBuild(pRpcMsg->contLen); syncPingFromRpcMsg(pRpcMsg, pSyncMsg); - // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); - io->FpOnSyncPing(io->pSyncNode, pSyncMsg); } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); - syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); - if (io->FpOnSyncPingReply != NULL) { + SyncPingReply *pSyncMsg; + pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen); + syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg); io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); } } else { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 49fac038da..9cb3a61fff 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -171,16 +171,6 @@ static int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, Syn SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - - /* - SRpcMsg rpcMsg; - rpcMsg.contLen = 64; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf((char*)rpcMsg.pCont, rpcMsg.contLen, "%s", "xxxxxxxxxxxxxx"); - rpcMsg.handle = NULL; - rpcMsg.msgType = 1; - */ - syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); { diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 24d9ead5e3..8268128347 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -13,7 +13,9 @@ void logTest() { sFatal("--- sync log test: fatal"); } -SSyncNode* doSync() { +uint16_t ports[3] = {7010, 7110, 7210}; + +SSyncNode* doSync(int myIndex) { SSyncFSM* pFsm; SSyncInfo syncInfo; @@ -24,18 +26,18 @@ SSyncNode* doSync() { snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = 0; - pCfg->replicaNum = 2; + pCfg->myIndex = myIndex; + pCfg->replicaNum = 3; - pCfg->nodeInfo[0].nodePort = 7010; + pCfg->nodeInfo[0].nodePort = ports[0]; snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - pCfg->nodeInfo[1].nodePort = 7110; + pCfg->nodeInfo[1].nodePort = ports[1]; snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - pCfg->nodeInfo[2].nodePort = 7210; + pCfg->nodeInfo[2].nodePort = ports[2]; snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); @@ -53,20 +55,25 @@ void timerPingAll(void* param, void* tmrId) { syncNodePingAll(pSyncNode); } -int main() { +int main(int argc, char** argv) { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; logTest(); - int32_t ret = syncIOStart((char*)"127.0.0.1", 7010); + int myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); ret = syncEnvStart(); assert(ret == 0); - SSyncNode* pSyncNode = doSync(); + SSyncNode* pSyncNode = doSync(myIndex); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; @@ -74,9 +81,9 @@ int main() { assert(ret == 0); /* - taosMsleep(10000); - ret = syncNodeStopPingTimer(pSyncNode); - assert(ret == 0); + taosMsleep(10000); + ret = syncNodeStopPingTimer(pSyncNode); + assert(ret == 0); */ while (1) { From aec7bef25b05afb69696533bafe00e7b53210c0c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 3 Mar 2022 12:29:54 +0800 Subject: [PATCH 08/13] add UT --- source/libs/index/src/index_tfile.c | 3 +++ source/libs/index/test/fstTest.cc | 23 +++++++++++++---------- source/libs/index/test/jsonUT.cc | 22 ++++++++++++++++++---- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a2089e8eee..0947c796b2 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -275,6 +275,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { __compar_fn_t fn; int8_t colType = tw->header.colType; + colType = INDEX_TYPE_GET_TYPE(colType); if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { fn = tfileStrCompare; } else { @@ -572,6 +573,8 @@ static int tfileWriteHeader(TFileWriter* writer) { static int tfileWriteData(TFileWriter* write, TFileValue* tval) { TFileHeader* header = &write->header; uint8_t colType = header->colType; + + colType = INDEX_TYPE_GET_TYPE(colType); if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); if (fstBuilderInsert(write->fb, key, tval->offset)) { diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 65118a2bce..74dcec4490 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -312,15 +312,18 @@ void validateTFile(char* arg) { tfCleanup(); } -void iterTFileReader(char* path, char* ver) { +void iterTFileReader(char* path, char* uid, char* colName, char* ver) { tfInit(); - int version = atoi(ver); - TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); - Iterate* iter = tfileIteratorCreate(reader); - bool tn = iter ? iter->next(iter) : false; - int count = 0; - int termCount = 0; + uint64_t suid = atoi(uid); + int version = atoi(ver); + + TFileReader* reader = tfileReaderOpen(path, suid, version, colName); + + Iterate* iter = tfileIteratorCreate(reader); + bool tn = iter ? iter->next(iter) : false; + int count = 0; + int termCount = 0; while (tn == true) { count++; IterateValue* cv = iter->getValue(iter); @@ -337,9 +340,9 @@ void iterTFileReader(char* path, char* ver) { int main(int argc, char* argv[]) { // tool to check all kind of fst test // if (argc > 1) { validateTFile(argv[1]); } - if (argc > 2) { - // opt - iterTFileReader(argv[1], argv[2]); + if (argc > 4) { + // path suid colName ver + iterTFileReader(argv[1], argv[2], argv[3], argv[4]); } // checkFstCheckIterator(); // checkFstLongTerm(); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e184bc49ae..e5c79d137f 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -22,7 +22,7 @@ class JsonEnv : public ::testing::Test { virtual void SetUp() { taosRemoveDir(dir.c_str()); taosMkDir(dir.c_str()); - + printf("set up\n"); opts = indexOptsCreate(); int ret = tIndexJsonOpen(opts, dir.c_str(), &index); assert(ret == 0); @@ -30,6 +30,7 @@ class JsonEnv : public ::testing::Test { virtual void TearDown() { tIndexJsonClose(index); indexOptsDestroy(opts); + printf("destory\n"); } SIndexJsonOpts* opts; SIndexJson* index; @@ -37,7 +38,7 @@ class JsonEnv : public ::testing::Test { TEST_F(JsonEnv, testWrite) { { - std::string colName("voltage"); + std::string colName("test"); std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -76,7 +77,7 @@ TEST_F(JsonEnv, testWrite) { indexMultiTermDestroy(terms); } { - std::string colName("voltage"); + std::string colName("test"); std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); @@ -91,6 +92,19 @@ TEST_F(JsonEnv, testWrite) { } } TEST_F(JsonEnv, testWriteMillonData) { + { + std::string colName("test"); + std::string colVal("ab"); + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 100; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); @@ -105,7 +119,7 @@ TEST_F(JsonEnv, testWriteMillonData) { indexMultiTermDestroy(terms); } { - std::string colName("voltage"); + std::string colName("test"); std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); From f263a623dd7186525c28bb79668426158404a71e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 14:52:30 +0800 Subject: [PATCH 09/13] sync encode/decode --- source/libs/sync/inc/syncMessage.h | 162 ++++++++------- source/libs/sync/src/syncMessage.c | 245 ++++++++++++++++------- source/libs/sync/test/syncEncodeTest.cpp | 129 +++++++++--- 3 files changed, 368 insertions(+), 168 deletions(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index be53559e8a..a51567d1dd 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -28,30 +28,25 @@ extern "C" { #include "syncRaftEntry.h" #include "taosdef.h" -// encode as uint64 +// encode as uint32 typedef enum ESyncMessageType { SYNC_PING = 101, SYNC_PING_REPLY = 103, - SYNC_CLIENT_REQUEST, - SYNC_CLIENT_REQUEST_REPLY, - SYNC_REQUEST_VOTE, - SYNC_REQUEST_VOTE_REPLY, - SYNC_APPEND_ENTRIES, - SYNC_APPEND_ENTRIES_REPLY, + SYNC_CLIENT_REQUEST = 105, + SYNC_CLIENT_REQUEST_REPLY = 107, + SYNC_REQUEST_VOTE = 109, + SYNC_REQUEST_VOTE_REPLY = 111, + SYNC_APPEND_ENTRIES = 113, + SYNC_APPEND_ENTRIES_REPLY = 115, } ESyncMessageType; -/* -typedef struct SRaftId { - SyncNodeId addr; // typedef uint64_t SyncNodeId; - SyncGroupId vgId; // typedef int32_t SyncGroupId; -} SRaftId; -*/ - +// --------------------------------------------- typedef struct SyncPing { uint32_t bytes; uint32_t msgType; SRaftId srcId; SRaftId destId; + // private data uint32_t dataLen; char data[]; } SyncPing; @@ -59,28 +54,22 @@ typedef struct SyncPing { #define SYNC_PING_FIX_LEN (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) SyncPing* syncPingBuild(uint32_t dataLen); - -void syncPingDestroy(SyncPing* pMsg); - -void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); - -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); - -void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); - -void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); - -cJSON* syncPing2Json(const SyncPing* pMsg); - +void syncPingDestroy(SyncPing* pMsg); +void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen); +void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg); +void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg); +void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg); +cJSON* syncPing2Json(const SyncPing* pMsg); SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); - SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId); +// --------------------------------------------- typedef struct SyncPingReply { uint32_t bytes; uint32_t msgType; SRaftId srcId; SRaftId destId; + // private data uint32_t dataLen; char data[]; } SyncPingReply; @@ -89,72 +78,95 @@ typedef struct SyncPingReply { (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(uint32_t)) SyncPingReply* syncPingReplyBuild(uint32_t dataLen); - -void syncPingReplyDestroy(SyncPingReply* pMsg); - -void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); - -void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); - -void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); - -void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); - -cJSON* syncPingReply2Json(const SyncPingReply* pMsg); - +void syncPingReplyDestroy(SyncPingReply* pMsg); +void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen); +void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg); +void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg); +void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg); +cJSON* syncPingReply2Json(const SyncPingReply* pMsg); SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str); - SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId); +// --------------------------------------------- typedef struct SyncClientRequest { - ESyncMessageType msgType; - char* data; - uint32_t dataLen; - int64_t seqNum; - bool isWeak; + uint32_t bytes; + uint32_t msgType; + int64_t seqNum; + bool isWeak; + uint32_t dataLen; + char data[]; } SyncClientRequest; +// --------------------------------------------- typedef struct SyncClientRequestReply { - ESyncMessageType msgType; - int32_t errCode; - SSyncBuffer* pErrMsg; - SSyncBuffer* pLeaderHint; + uint32_t bytes; + uint32_t msgType; + int32_t errCode; + SRaftId leaderHint; } SyncClientRequestReply; +// --------------------------------------------- typedef struct SyncRequestVote { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - SyncGroupId vgId; - SyncIndex lastLogIndex; - SyncTerm lastLogTerm; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm currentTerm; + SyncIndex lastLogIndex; + SyncTerm lastLogTerm; } SyncRequestVote; +SyncRequestVote* syncRequestVoteBuild(); +void syncRequestVoteDestroy(SyncRequestVote* pMsg); +void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen); +void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg); +void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg); +void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg); +cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg); + +// --------------------------------------------- typedef struct SyncRequestVoteReply { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - SyncGroupId vgId; - bool voteGranted; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + bool voteGranted; } SyncRequestVoteReply; +SyncRequestVoteReply* SyncRequestVoteReplyBuild(); +void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg); +void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen); +void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg); +void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg); +void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg); +cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg); + +// --------------------------------------------- typedef struct SyncAppendEntries { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - SyncIndex prevLogIndex; - SyncTerm prevLogTerm; - int32_t entryCount; - SSyncRaftEntry* logEntries; - SyncIndex commitIndex; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncIndex prevLogIndex; + SyncTerm prevLogTerm; + SyncIndex commitIndex; + uint32_t dataLen; + char data[]; } SyncAppendEntries; +// --------------------------------------------- typedef struct SyncAppendEntriesReply { - ESyncMessageType msgType; - SyncTerm currentTerm; - SyncNodeId nodeId; - bool success; - SyncIndex matchIndex; + uint32_t bytes; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + bool success; + SyncIndex matchIndex; } SyncAppendEntriesReply; #ifdef __cplusplus diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index a26c8401b2..ac3f550e3e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -60,12 +60,15 @@ void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg) { } cJSON* syncPing2Json(const SyncPing* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; cJSON* pTmp = pSrcId; @@ -79,7 +82,8 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; cJSON* pTmp = pDestId; @@ -154,12 +158,15 @@ void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg) { } cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pSrcId, "addr", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; cJSON* pTmp = pSrcId; @@ -173,7 +180,8 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; cJSON* pTmp = pDestId; @@ -208,72 +216,169 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) return pMsg; } -#if 0 -void syncPingSerialize(const SyncPing* pMsg, char** ppBuf, uint32_t* bufLen) { - *bufLen = sizeof(SyncPing) + pMsg->dataLen; - *ppBuf = (char*)malloc(*bufLen); - void* pStart = *ppBuf; - uint32_t allBytes = *bufLen; - - int len = 0; - len = taosEncodeFixedU32(&pStart, pMsg->msgType); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedU64(&pStart, pMsg->srcId.addr); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedI32(&pStart, pMsg->srcId.vgId); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedU64(&pStart, pMsg->destId.addr); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedI32(&pStart, pMsg->destId.vgId); - allBytes -= len; - assert(len > 0); - pStart += len; - - len = taosEncodeFixedU32(&pStart, pMsg->dataLen); - allBytes -= len; - assert(len > 0); - pStart += len; - - memcpy(pStart, pMsg->data, pMsg->dataLen); - allBytes -= pMsg->dataLen; - assert(allBytes == 0); +// ---- message process SyncRequestVote---- +SyncRequestVote* syncRequestVoteBuild() { + uint32_t bytes = sizeof(SyncRequestVote); + SyncRequestVote* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_REQUEST_VOTE; } - -void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { - void* pStart = (void*)buf; - uint64_t u64; - int32_t i32; - uint32_t u32; - - pStart = taosDecodeFixedU64(pStart, &u64); - pMsg->msgType = u64; - - pStart = taosDecodeFixedU64(pStart, &u64); - pMsg->srcId.addr = u64; - - pStart = taosDecodeFixedI32(pStart, &i32); - pMsg->srcId.vgId = i32; - - pStart = taosDecodeFixedU64(pStart, &u64); - pMsg->destId.addr = u64; - - pStart = taosDecodeFixedI32(pStart, &i32); - pMsg->destId.vgId = i32; - - pStart = taosDecodeFixedU32(pStart, &u32); - pMsg->dataLen = u32; +void syncRequestVoteDestroy(SyncRequestVote* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } } -#endif \ No newline at end of file + +void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncRequestVoteSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg) { + syncRequestVoteDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->currentTerm); + cJSON_AddStringToObject(pRoot, "currentTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); + cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); + cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncRequestVote", pRoot); + return pJson; +} + +// ---- message process SyncRequestVoteReply---- +SyncRequestVoteReply* SyncRequestVoteReplyBuild() { + uint32_t bytes = sizeof(SyncRequestVoteReply); + SyncRequestVoteReply* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; +} + +void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncRequestVoteReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg) { + syncRequestVoteReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + cJSON_AddNumberToObject(pDestId, "addr", pMsg->destId.addr); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "vote_granted", pMsg->voteGranted); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncRequestVoteReply", pRoot); + return pJson; +} \ No newline at end of file diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 4deceb603e..e0179b2c8b 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -3,6 +3,7 @@ #include "syncIO.h" #include "syncInt.h" #include "syncMessage.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -21,16 +22,16 @@ void test1() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "test ping"); SyncPing* pMsg = syncPingBuild(PING_MSG_LEN); - pMsg->srcId.addr = 1; - pMsg->srcId.vgId = 2; - pMsg->destId.addr = 3; - pMsg->destId.vgId = 4; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPing2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPing: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -45,7 +46,7 @@ void test1() { { cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPing2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -61,16 +62,16 @@ void test2() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "hello raft"); SyncPing* pMsg = syncPingBuild(PING_MSG_LEN); - pMsg->srcId.addr = 100; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 3333); pMsg->srcId.vgId = 200; - pMsg->destId.addr = 300; - pMsg->destId.vgId = 400; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 4444); + pMsg->destId.vgId = 200; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPing2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPing: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -84,7 +85,7 @@ void test2() { { cJSON* pJson = syncPing2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPing2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -99,16 +100,16 @@ void test3() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "test ping"); SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN); - pMsg->srcId.addr = 19; - pMsg->srcId.vgId = 29; - pMsg->destId.addr = 39; - pMsg->destId.vgId = 49; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 5555); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 6666); + pMsg->destId.vgId = 100; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPingReply2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -123,7 +124,7 @@ void test3() { { cJSON* pJson = syncPingReply2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -139,16 +140,16 @@ void test4() { char msg[PING_MSG_LEN]; snprintf(msg, sizeof(msg), "%s", "hello raft"); SyncPingReply* pMsg = syncPingReplyBuild(PING_MSG_LEN); - pMsg->srcId.addr = 66; - pMsg->srcId.vgId = 77; - pMsg->destId.addr = 88; - pMsg->destId.vgId = 99; + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 7777); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 8888); + pMsg->destId.vgId = 100; memcpy(pMsg->data, msg, PING_MSG_LEN); { cJSON* pJson = syncPingReply2Json(pMsg); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -162,7 +163,7 @@ void test4() { { cJSON* pJson = syncPingReply2Json(pMsg2); char* serialized = cJSON_Print(pJson); - printf("SyncPingReply2: \n%s\n\n", serialized); + printf("\n%s\n\n", serialized); free(serialized); cJSON_Delete(pJson); } @@ -170,6 +171,86 @@ void test4() { syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } + +void test5() { + sTrace("test5: ---- syncRequestVoteSerialize, syncRequestVoteDeserialize"); + + SyncRequestVote* pMsg = syncRequestVoteBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->currentTerm = 20; + pMsg->lastLogIndex = 21; + pMsg->lastLogTerm = 22; + + { + cJSON* pJson = syncRequestVote2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncRequestVoteSerialize(pMsg, buf, bufLen); + + SyncRequestVote* pMsg2 = (SyncRequestVote*)malloc(pMsg->bytes); + syncRequestVoteDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncRequestVote2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteDestroy(pMsg); + syncRequestVoteDestroy(pMsg2); + free(buf); +} + +void test6() { + sTrace("test6: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize"); + + SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 20; + pMsg->voteGranted = 1; + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncRequestVoteReplySerialize(pMsg, buf, bufLen); + + SyncRequestVoteReply* pMsg2 = (SyncRequestVoteReply*)malloc(pMsg->bytes); + syncRequestVoteReplyDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteReplyDestroy(pMsg); + syncRequestVoteReplyDestroy(pMsg2); + free(buf); +} + int main() { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; @@ -179,6 +260,8 @@ int main() { test2(); test3(); test4(); + test5(); + test6(); return 0; } From c572b9cad254265f4cf4905f809905323c41bd2c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 3 Mar 2022 15:07:33 +0800 Subject: [PATCH 10/13] feature/qnode --- include/libs/scheduler/scheduler.h | 10 +- source/client/inc/clientInt.h | 2 +- source/client/src/clientImpl.c | 12 +- source/libs/scheduler/inc/schedulerInt.h | 15 +- source/libs/scheduler/src/scheduler.c | 283 ++++++++++-------- source/libs/scheduler/test/schedulerTests.cpp | 120 +++++--- 6 files changed, 244 insertions(+), 198 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index e856adaf31..b2080cb655 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -23,8 +23,6 @@ extern "C" { #include "catalog.h" #include "planner.h" -struct SSchJob; - typedef struct SSchedulerCfg { uint32_t maxJobNum; } SSchedulerCfg; @@ -72,7 +70,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -80,7 +78,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, int64_t *pJob); /** * Fetch query result from the remote query executor @@ -88,7 +86,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDa * @param data * @return */ -int32_t schedulerFetchRows(struct SSchJob *pJob, void **data); +int32_t schedulerFetchRows(int64_t job, void **data); /** @@ -102,7 +100,7 @@ int32_t schedulerFetchRows(struct SSchJob *pJob, void **data); * Free the query job * @param pJob */ -void schedulerFreeJob(void *pJob); +void schedulerFreeJob(int64_t job); void schedulerDestroy(void); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 79909a5696..c93ea1aabe 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -171,7 +171,7 @@ typedef struct SRequestSendRecvBody { void* fp; SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + int64_t queryJob; // query job, created according to sql query DAG. struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3c6a49c3db..a0ba668f8a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -227,10 +227,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, &res); if (code != TSDB_CODE_SUCCESS) { - if (pRequest->body.pQueryJob != NULL) { - schedulerFreeJob(pRequest->body.pQueryJob); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); } pRequest->code = code; @@ -240,8 +240,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { pRequest->body.resInfo.numOfRows = res.numOfRows; - if (pRequest->body.pQueryJob != NULL) { - schedulerFreeJob(pRequest->body.pQueryJob); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); } } @@ -494,7 +494,7 @@ void* doFetchRow(SRequestObj* pRequest) { } SReqResultInfo* pResInfo = &pRequest->body.resInfo; - int32_t code = schedulerFetchRows(pRequest->body.pQueryJob, (void**)&pResInfo->pData); + int32_t code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; return NULL; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 42270cd645..50c274ad48 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -36,6 +36,11 @@ enum { SCH_WRITE, }; +typedef struct SSchTrans { + void *transInst; + void *transHandle; +} SSchTrans; + typedef struct SSchApiStat { } SSchApiStat; @@ -59,12 +64,13 @@ typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId SSchedulerCfg cfg; - SHashObj *jobs; // key: queryId, value: SQueryJob* + int32_t jobRef; SSchedulerStat stat; } SSchedulerMgmt; typedef struct SSchCallbackParam { uint64_t queryId; + int64_t refId; uint64_t taskId; } SSchCallbackParam; @@ -75,7 +81,8 @@ typedef struct SSchLevel { int32_t taskFailed; int32_t taskSucceed; int32_t taskNum; - SArray *subTasks; // Element is SQueryTask + int32_t taskLaunchIdx; // launch startup index + SArray *subTasks; // Element is SQueryTask } SSchLevel; typedef struct SSchTask { @@ -105,6 +112,7 @@ typedef struct SSchJobAttr { } SSchJobAttr; typedef struct SSchJob { + int64_t refId; uint64_t queryId; SSchJobAttr attr; int32_t levelNum; @@ -119,7 +127,6 @@ typedef struct SSchJob { SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* - int32_t ref; int8_t status; SQueryNodeAddr resNode; tsem_t rspSem; @@ -168,6 +175,8 @@ typedef struct SSchJob { static int32_t schLaunchTask(SSchJob *job, SSchTask *task); static int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); +SSchJob *schAcquireJob(int64_t refId); +int32_t schReleaseJob(int64_t refId); #ifdef __cplusplus } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f14b95873a..f1ed0cef7d 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -17,12 +17,17 @@ #include "tmsg.h" #include "query.h" #include "catalog.h" +#include "tref.h" -typedef struct SSchTrans { - void *transInst; - void *transHandle; -}SSchTrans; -static SSchedulerMgmt schMgmt = {0}; +SSchedulerMgmt schMgmt = {0}; + +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { + return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); +} + +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { + return taosReleaseRef(schMgmt.jobRef, refId); +} uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); @@ -886,7 +891,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } case TDMT_VND_DROP_TASK_RSP: { // SHOULD NEVER REACH HERE - SCH_TASK_ELOG("invalid status to handle drop task rsp, ref:%d", atomic_load_32(&pJob->ref)); + SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } @@ -908,28 +913,23 @@ _return: int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchJob *pJob = NULL; SSchTask *pTask = NULL; - SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); - if (NULL == job || NULL == (*job)) { - qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, pParam->refId); + if (NULL == pJob) { + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } - pJob = *job; - - atomic_add_fetch_32(&pJob->ref, 1); - int32_t s = taosHashGetSize(pJob->execTasks); if (s <= 0) { - qError("QID:%"PRIx64",TID:%"PRId64" no task in execTask list", pParam->queryId, pParam->taskId); + SCH_JOB_ELOG("empty execTask list, refId:%" PRIx64 ", taskId:%" PRIx64, pParam->refId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { - qError("QID:%"PRIx64",TID:%"PRId64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); + SCH_JOB_ELOG("task not found in execTask list, refId:%" PRIx64 ", taskId:%" PRIx64, pParam->refId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -942,7 +942,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in _return: if (pJob) { - atomic_sub_fetch_32(&pJob->ref, 1); + taosReleaseRef(schMgmt.jobRef, pParam->refId); } tfree(param); @@ -1003,28 +1003,29 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { } -int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, uint32_t msgSize) { +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SMsgSendInfo)); + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); if (NULL == param) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SSchCallbackParam)); + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchCallbackParam)); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); - param->queryId = qId; - param->taskId = tId; + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = pTask->taskId; pMsgSendInfo->param = param; @@ -1040,7 +1041,7 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t SCH_ERR_JRET(code); } - qDebug("QID:0x%"PRIx64 ",TID:0x%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); + SCH_TASK_DLOG("req msg sent, refId:%" PRIx64 ", type:%d, %s", pJob->refId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: @@ -1160,7 +1161,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, atomic_store_32(&pTask->lastMsgType, msgType); SSchTrans trans = {.transInst = pJob->transport, .transHandle = pTask->handle}; - SCH_ERR_JRET(schAsyncSendMsg(&trans, &epSet, pJob->queryId, pTask->taskId, msgType, msg, msgSize)); + SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize)); if (isCandidateAddr) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); @@ -1283,7 +1284,60 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, const char* sql, bool syncSchedule) { + +int32_t schCancelJob(SSchJob *pJob) { + //TODO + + //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST + +} + +void schFreeJobImpl(void *job) { + if (NULL == job) { + return; + } + + SSchJob *pJob = job; + uint64_t queryId = pJob->queryId; + int64_t refId = pJob->refId; + + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { + schCancelJob(pJob); + } + + schDropJobAllTasks(pJob); + + pJob->subPlans = NULL; // it is a reference to pDag->pSubplans + + int32_t numOfLevels = taosArrayGetSize(pJob->levels); + for(int32_t i = 0; i < numOfLevels; ++i) { + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + + int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); + for(int32_t j = 0; j < numOfTasks; ++j) { + SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); + schFreeTask(pTask); + } + + taosArrayDestroy(pLevel->subTasks); + } + + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); + + taosArrayDestroy(pJob->levels); + taosArrayDestroy(pJob->nodeList); + + tfree(pJob->res); + + tfree(pJob); + + qDebug("QID:0x%"PRIx64" job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); +} + + +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, int64_t *job, const char* sql, bool syncSchedule) { qDebug("QID:0x%"PRIx64" job started", pDag->queryId); if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { @@ -1327,21 +1381,20 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDa tsem_init(&pJob->rspSem, 0, 0); - code = taosHashPut(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId), &pJob, POINTER_BYTES); - if (0 != code) { - if (HASH_NODE_EXIST(code)) { - SCH_JOB_ELOG("job already exist, isQueryJob:%d", pJob->attr.queryJob); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } else { - SCH_JOB_ELOG("taosHashPut job failed, errno:%d", errno); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } + pJob->refId = taosAddRef(schMgmt.jobRef, pJob); + if (pJob->refId < 0) { + SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno)); + SCH_ERR_JRET(terrno); } + SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + pJob->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(pJob)); - *(SSchJob **)job = pJob; + taosAcquireRef(schMgmt.jobRef, pJob->refId); + + *job = pJob->refId; if (syncSchedule) { SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob)); @@ -1349,25 +1402,20 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDa } SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); + + taosReleaseRef(schMgmt.jobRef, pJob->refId); + return TSDB_CODE_SUCCESS; _return: - *(SSchJob **)job = NULL; - schedulerFreeJob(pJob); + schFreeJobImpl(pJob); SCH_RET(code); } -int32_t schCancelJob(SSchJob *pJob) { - //TODO - - //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST - -} - int32_t schedulerInit(SSchedulerCfg *cfg) { - if (schMgmt.jobs) { + if (schMgmt.jobRef) { qError("scheduler already initialized"); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -1381,9 +1429,9 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } else { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } - - schMgmt.jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == schMgmt.jobs) { + + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); + if (schMgmt.jobRef < 0) { qError("init schduler jobs failed, num:%u", schMgmt.cfg.maxJobNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1398,24 +1446,28 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); - pRes->code = atomic_load_32(&(*pJob)->errCode); - pRes->numOfRows = (*pJob)->resNumOfRows; + + SSchJob *job = taosAcquireRef(schMgmt.jobRef, *pJob); + pRes->code = atomic_load_32(&job->errCode); + pRes->numOfRows = job->resNumOfRows; + taosReleaseRef(schMgmt.jobRef, *pJob); return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, int64_t *pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); + return TSDB_CODE_SUCCESS; } @@ -1541,28 +1593,35 @@ _return: } -int32_t schedulerFetchRows(SSchJob *pJob, void** pData) { - if (NULL == pJob || NULL == pData) { +int32_t schedulerFetchRows(int64_t job, void** pData) { + if (NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; - atomic_add_fetch_32(&pJob->ref, 1); + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, job); + if (NULL == pJob) { + qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } int8_t status = SCH_GET_JOB_STATUS(pJob); if (status == JOB_TASK_STATUS_DROPPING) { SCH_JOB_ELOG("job is dropping, status:%d", status); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + taosReleaseRef(schMgmt.jobRef, job); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + taosReleaseRef(schMgmt.jobRef, job); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + taosReleaseRef(schMgmt.jobRef, job); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { @@ -1588,7 +1647,6 @@ int32_t schedulerFetchRows(SSchJob *pJob, void** pData) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } -_return: while (true) { *pData = atomic_load_ptr(&pJob->res); @@ -1609,96 +1667,47 @@ _return: SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - SCH_JOB_DLOG("fetch done, totalRows:%d, code:%s", pJob->resNumOfRows, tstrerror(code)); - atomic_sub_fetch_32(&pJob->ref, 1); + +_return: + + atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); + + taosReleaseRef(schMgmt.jobRef, job); SCH_RET(code); } -int32_t scheduleCancelJob(void *job) { - SSchJob *pJob = (SSchJob *)job; - - atomic_add_fetch_32(&pJob->ref, 1); +int32_t scheduleCancelJob(int64_t job) { + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, job); + if (NULL == pJob) { + qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } int32_t code = schCancelJob(pJob); - atomic_sub_fetch_32(&pJob->ref, 1); + taosReleaseRef(schMgmt.jobRef, job); SCH_RET(code); } -void schedulerFreeJob(void *job) { - if (NULL == job) { +void schedulerFreeJob(int64_t job) { + SSchJob *pJob = taosAcquireRef(schMgmt.jobRef, job); + if (NULL == pJob) { + qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); return; } - SSchJob *pJob = job; - uint64_t queryId = pJob->queryId; - bool setJobFree = false; - - if (SCH_GET_JOB_STATUS(pJob) > 0) { - if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { - SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); - return; - } - - SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); - - while (true) { - int32_t ref = atomic_load_32(&pJob->ref); - if (0 == ref) { - break; - } else if (ref > 0) { - if (1 == ref && atomic_load_8(&pJob->userFetch) > 0 && !setJobFree) { - schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); - setJobFree = true; - } - - usleep(1); - } else { - SCH_JOB_ELOG("invalid job ref number, ref:%d", ref); - break; - } - } - - SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); - - if (pJob->status == JOB_TASK_STATUS_EXECUTING) { - schCancelJob(pJob); - } - - schDropJobAllTasks(pJob); + if (atomic_load_8(&pJob->userFetch) > 0) { + schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); } - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans - - int32_t numOfLevels = taosArrayGetSize(pJob->levels); - for(int32_t i = 0; i < numOfLevels; ++i) { - SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + SCH_JOB_DLOG("start to remove job from jobRef list, refId:%" PRIx64, job); - int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); - for(int32_t j = 0; j < numOfTasks; ++j) { - SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); - schFreeTask(pTask); - } - - taosArrayDestroy(pLevel->subTasks); + if (taosRemoveRef(schMgmt.jobRef, job)) { + SCH_JOB_ELOG("remove job from job list failed, refId:%" PRIx64, job); } - - taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); - - taosArrayDestroy(pJob->levels); - taosArrayDestroy(pJob->nodeList); - - tfree(pJob->res); - - tfree(pJob); - - qDebug("QID:0x%"PRIx64" job freed", queryId); } void schedulerFreeTaskList(SArray *taskList) { @@ -1716,9 +1725,17 @@ void schedulerFreeTaskList(SArray *taskList) { } void schedulerDestroy(void) { - if (schMgmt.jobs) { - taosHashCleanup(schMgmt.jobs); //TODO - schMgmt.jobs = NULL; + if (schMgmt.jobRef) { + SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); + + while (pJob) { + taosRemoveRef(schMgmt.jobRef, pJob->refId); + + pJob = taosIterateRef(schMgmt.jobRef, pJob->refId); + } + + taosCloseRef(schMgmt.jobRef); + schMgmt.jobRef = 0; } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 89d365a7e7..11ed3335e6 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -38,15 +38,15 @@ #include "schedulerInt.h" #include "stub.h" #include "addr_any.h" - +#include "tref.h" namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); extern "C" int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode); -struct SSchJob *pInsertJob = NULL; -struct SSchJob *pQueryJob = NULL; +int64_t insertJobRefId = 0; +int64_t queryJobRefId = 0; uint64_t schtMergeTemplateId = 0x4; uint64_t schtFetchTaskId = 0; @@ -65,6 +65,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -255,34 +256,40 @@ void schtSetAsyncSendMsgToServer() { void *schtSendRsp(void *param) { - SSchJob *job = NULL; + SSchJob *pJob = NULL; + int64_t job = 0; int32_t code = 0; while (true) { - job = *(SSchJob **)param; + job = *(int64_t *)param; if (job) { break; } usleep(1000); } + + pJob = schAcquireJob(job); - void *pIter = taosHashIterate(job->execTasks, NULL); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SSubmitRsp rsp = {0}; rsp.affectedRows = 10; - schHandleResponseMsg(job, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); + schHandleResponseMsg(pJob, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } + schReleaseJob(job); + return NULL; } void *schtCreateFetchRspThread(void *param) { - struct SSchJob* job = (struct SSchJob*)param; + int64_t job = *(int64_t *)param; + SSchJob* pJob = schAcquireJob(job); sleep(1); @@ -291,8 +298,10 @@ void *schtCreateFetchRspThread(void *param) { rsp->completed = 1; rsp->numOfRows = 10; - code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); - + code = schHandleResponseMsg(pJob, pJob->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); + + schReleaseJob(job); + assert(code == 0); } @@ -329,9 +338,9 @@ void *schtFetchRspThread(void *aa) { void schtFreeQueryJob(int32_t freeThread) { static uint32_t freeNum = 0; - SSchJob *job = atomic_load_ptr(&pQueryJob); + int64_t job = queryJobRefId; - if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) { + if (job && atomic_val_compare_exchange_64(&queryJobRefId, job, 0)) { schedulerFreeJob(job); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { @@ -360,7 +369,7 @@ void* schtRunJobThread(void *aa) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - SSchJob *job = NULL; + SSchJob *pJob = NULL; SSchCallbackParam *param = NULL; SHashObj *execTasks = NULL; SDataBuf dataBuf = {0}; @@ -376,24 +385,29 @@ void* schtRunJobThread(void *aa) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &queryJobRefId); assert(code == 0); + pJob = schAcquireJob(queryJobRefId); + if (NULL == pJob) { + taosArrayDestroy(qnodeList); + schtFreeQueryDag(&dag); + continue; + } + execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - void *pIter = taosHashIterate(job->execTasks, NULL); + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; schtFetchTaskId = task->taskId - 1; taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; - - pQueryJob = job; - + param->refId = queryJobRefId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -412,8 +426,9 @@ void* schtRunJobThread(void *aa) { param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; - + param->refId = queryJobRefId; + param->queryId = pJob->queryId; + pIter = taosHashIterate(execTasks, NULL); while (pIter) { SSchTask *task = (SSchTask *)pIter; @@ -431,7 +446,8 @@ void* schtRunJobThread(void *aa) { param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; + param->refId = queryJobRefId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -450,7 +466,8 @@ void* schtRunJobThread(void *aa) { param = (SSchCallbackParam *)calloc(1, sizeof(*param)); - param->queryId = schtQueryId; + param->refId = queryJobRefId; + param->queryId = pJob->queryId; pIter = taosHashIterate(execTasks, NULL); while (pIter) { @@ -470,7 +487,7 @@ void* schtRunJobThread(void *aa) { atomic_store_32(&schtStartFetch, 1); void *data = NULL; - code = schedulerFetchRows(pQueryJob, &data); + code = schedulerFetchRows(queryJobRefId, &data); assert(code == 0 || code); if (0 == code) { @@ -480,12 +497,13 @@ void* schtRunJobThread(void *aa) { } data = NULL; - code = schedulerFetchRows(pQueryJob, &data); + code = schedulerFetchRows(queryJobRefId, &data); assert(code == 0 || code); schtFreeQueryJob(0); taosHashCleanup(execTasks); + taosArrayDestroy(qnodeList); schtFreeQueryDag(&dag); @@ -516,7 +534,7 @@ TEST(queryTest, normalCase) { char *dbname = "1.db1"; char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - SSchJob *pJob = NULL; + int64_t job = 0; SQueryDag dag = {0}; schtInitLogFile(); @@ -537,59 +555,61 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); ASSERT_EQ(code, 0); - SSchJob *job = (SSchJob *)pJob; - void *pIter = taosHashIterate(job->execTasks, NULL); + + SSchJob *pJob = schAcquireJob(job); + + void *pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } - pIter = taosHashIterate(job->execTasks, NULL); + pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); printf("code:%d", code); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } - pIter = taosHashIterate(job->execTasks, NULL); + pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } - pIter = taosHashIterate(job->execTasks, NULL); + pIter = taosHashIterate(pJob->execTasks, NULL); while (pIter) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); - pIter = taosHashIterate(job->execTasks, pIter); + pIter = taosHashIterate(pJob->execTasks, pIter); } pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_t thread1; - pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job); + pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, &job); void *data = NULL; code = schedulerFetchRows(job, &data); @@ -603,9 +623,11 @@ TEST(queryTest, normalCase) { data = NULL; code = schedulerFetchRows(job, &data); ASSERT_EQ(code, 0); - ASSERT_TRUE(data); + ASSERT_TRUE(data == NULL); - schedulerFreeJob(pJob); + schReleaseJob(job); + + schedulerFreeJob(job); schtFreeQueryDag(&dag); @@ -644,14 +666,14 @@ TEST(insertTest, normalCase) { pthread_attr_init(&thattr); pthread_t thread1; - pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); + pthread_create(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, "insert into tb values(now,1)", &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); - schedulerFreeJob(pInsertJob); + schedulerFreeJob(insertJobRefId); schedulerDestroy(); } @@ -684,4 +706,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop From aeb94af5e0b382e2ca29db907de5c6c78c649bf3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 16:15:18 +0800 Subject: [PATCH 11/13] sync encode/decode --- source/libs/sync/inc/syncMessage.h | 20 ++ source/libs/sync/src/syncMessage.c | 176 ++++++++++++++++ source/libs/sync/test/syncEncodeTest.cpp | 244 ++++++++++++++++++++++- 3 files changed, 439 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index a51567d1dd..3057e23bc2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -158,6 +158,18 @@ typedef struct SyncAppendEntries { char data[]; } SyncAppendEntries; +#define SYNC_APPEND_ENTRIES_FIX_LEN \ + (sizeof(uint32_t) + sizeof(uint32_t) + sizeof(SRaftId) + sizeof(SRaftId) + sizeof(SyncIndex) + sizeof(SyncTerm) + \ + sizeof(SyncIndex) + sizeof(uint32_t)) + +SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen); +void syncAppendEntriesDestroy(SyncAppendEntries* pMsg); +void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen); +void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg); +void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg); +void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg); +cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg); + // --------------------------------------------- typedef struct SyncAppendEntriesReply { uint32_t bytes; @@ -169,6 +181,14 @@ typedef struct SyncAppendEntriesReply { SyncIndex matchIndex; } SyncAppendEntriesReply; +SyncAppendEntriesReply* syncAppendEntriesReplyBuild(); +void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen); +void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg); +void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg); +void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg); +cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index ac3f550e3e..4c44b4691c 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -381,4 +381,180 @@ cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { cJSON* pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "SyncRequestVoteReply", pRoot); return pJson; +} + +// ---- message process SyncAppendEntries---- +SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { + uint32_t bytes = SYNC_APPEND_ENTRIES_FIX_LEN + dataLen; + SyncAppendEntries* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_APPEND_ENTRIES; + pMsg->dataLen = dataLen; +} + +void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); + assert(pMsg->bytes == SYNC_APPEND_ENTRIES_FIX_LEN + pMsg->dataLen); +} + +void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncAppendEntriesSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg) { + syncAppendEntriesDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); + cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commit_index", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", pMsg->data); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntries", pRoot); + return pJson; +} + +// ---- message process SyncAppendEntriesReply---- +SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { + uint32_t bytes = sizeof(SyncAppendEntriesReply); + SyncAppendEntriesReply* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; +} + +void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncAppendEntriesReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg) { + syncAppendEntriesReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + cJSON_AddNumberToObject(pRoot, "success", pMsg->success); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); + cJSON_AddStringToObject(pRoot, "match_index", u64buf); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot); + return pJson; } \ No newline at end of file diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index e0179b2c8b..6197621051 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -15,6 +15,7 @@ void logTest() { } #define PING_MSG_LEN 20 +#define APPEND_ENTRIES_VALUE_LEN 32 void test1() { sTrace("test1: ---- syncPingSerialize, syncPingDeserialize"); @@ -213,7 +214,45 @@ void test5() { } void test6() { - sTrace("test6: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize"); + sTrace("test6: ---- syncRequestVote2RpcMsg, syncRequestVoteFromRpcMsg"); + + SyncRequestVote* pMsg = syncRequestVoteBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->currentTerm = 20; + pMsg->lastLogIndex = 21; + pMsg->lastLogTerm = 22; + + { + cJSON* pJson = syncRequestVote2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncRequestVote2RpcMsg(pMsg, &rpcMsg); + SyncRequestVote* pMsg2 = (SyncRequestVote*)malloc(pMsg->bytes); + syncRequestVoteFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncRequestVote2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteDestroy(pMsg); + syncRequestVoteDestroy(pMsg2); +} + +void test7() { + sTrace("test7: ---- syncRequestVoteReplySerialize, syncRequestVoteReplyDeserialize"); SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild(); pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); @@ -251,6 +290,203 @@ void test6() { free(buf); } +void test8() { + sTrace("test8: ---- syncRequestVoteReply2RpcMsg, syncRequestVoteReplyFromRpcMsg"); + + SyncRequestVoteReply* pMsg = SyncRequestVoteReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("8.8.8.8", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 20; + pMsg->voteGranted = 1; + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg); + SyncRequestVoteReply* pMsg2 = (SyncRequestVoteReply*)malloc(pMsg->bytes); + syncRequestVoteReplyFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncRequestVoteReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncRequestVoteReplyDestroy(pMsg); + syncRequestVoteReplyDestroy(pMsg2); +} + +void test9() { + sTrace("test9: ---- syncAppendEntriesSerialize, syncAppendEntriesDeserialize"); + + char msg[APPEND_ENTRIES_VALUE_LEN]; + snprintf(msg, sizeof(msg), "%s", "test value"); + SyncAppendEntries* pMsg = syncAppendEntriesBuild(APPEND_ENTRIES_VALUE_LEN); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->prevLogIndex = 55; + pMsg->prevLogTerm = 66; + pMsg->commitIndex = 77; + memcpy(pMsg->data, msg, APPEND_ENTRIES_VALUE_LEN); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncAppendEntriesSerialize(pMsg, buf, bufLen); + + SyncAppendEntries* pMsg2 = (SyncAppendEntries*)malloc(pMsg->bytes); + syncAppendEntriesDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesDestroy(pMsg); + syncAppendEntriesDestroy(pMsg2); + free(buf); +} + +void test10() { + sTrace("test10: ---- syncAppendEntries2RpcMsg, syncAppendEntriesFromRpcMsg"); + + char msg[APPEND_ENTRIES_VALUE_LEN]; + snprintf(msg, sizeof(msg), "%s", "test value"); + SyncAppendEntries* pMsg = syncAppendEntriesBuild(APPEND_ENTRIES_VALUE_LEN); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->prevLogIndex = 55; + pMsg->prevLogTerm = 66; + pMsg->commitIndex = 77; + memcpy(pMsg->data, msg, APPEND_ENTRIES_VALUE_LEN); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncAppendEntries2RpcMsg(pMsg, &rpcMsg); + SyncAppendEntries* pMsg2 = (SyncAppendEntries*)malloc(pMsg->bytes); + syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncAppendEntries2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesDestroy(pMsg); + syncAppendEntriesDestroy(pMsg2); +} + +void test11() { + sTrace("test11: ---- syncAppendEntriesReplySerialize, syncAppendEntriesReplyDeserialize"); + + SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->success = 1; + pMsg->matchIndex = 23; + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + uint32_t bufLen = pMsg->bytes; + char* buf = (char*)malloc(bufLen); + syncAppendEntriesReplySerialize(pMsg, buf, bufLen); + + SyncAppendEntriesReply* pMsg2 = (SyncAppendEntriesReply*)malloc(pMsg->bytes); + syncAppendEntriesReplyDeserialize(buf, bufLen, pMsg2); + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesReplyDestroy(pMsg); + syncAppendEntriesReplyDestroy(pMsg2); + free(buf); +} + +void test12() { + sTrace("test12: ---- syncAppendEntriesReply2RpcMsg, syncAppendEntriesReplyFromRpcMsg"); + + SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyBuild(); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1111); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 2222); + pMsg->destId.vgId = 100; + pMsg->success = 1; + pMsg->matchIndex = 23; + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg); + SyncAppendEntriesReply* pMsg2 = (SyncAppendEntriesReply*)malloc(pMsg->bytes); + syncAppendEntriesReplyFromRpcMsg(&rpcMsg, pMsg2); + rpcFreeCont(rpcMsg.pCont); + + { + cJSON* pJson = syncAppendEntriesReply2Json(pMsg2); + char* serialized = cJSON_Print(pJson); + printf("\n%s\n\n", serialized); + free(serialized); + cJSON_Delete(pJson); + } + + syncAppendEntriesReplyDestroy(pMsg); + syncAppendEntriesReplyDestroy(pMsg2); +} + int main() { // taosInitLog((char*)"syncPingTest.log", 100000, 10); tsAsyncLog = 0; @@ -262,6 +498,12 @@ int main() { test4(); test5(); test6(); + test7(); + test8(); + test9(); + test10(); + test11(); + test12(); return 0; } From 64d224a0d2b5450e221b0136b8a80b8d57184025 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 3 Mar 2022 17:28:00 +0800 Subject: [PATCH 12/13] syncInt --- include/libs/sync/sync.h | 4 +- source/libs/sync/inc/syncInt.h | 72 +++++++++++++++++++----------- source/libs/sync/inc/syncVoteMgr.h | 6 +++ source/libs/sync/src/syncMain.c | 2 +- 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index cd04783dbc..53fad4607a 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -34,9 +34,7 @@ typedef enum { TAOS_SYNC_STATE_FOLLOWER = 0, TAOS_SYNC_STATE_CANDIDATE = 1, TAOS_SYNC_STATE_LEADER = 2, -} ESyncRole; - -typedef ESyncRole ESyncState; +} ESyncState; typedef struct SSyncBuffer { void* data; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0901330488..aedb9662b1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -25,6 +25,7 @@ extern "C" { #include #include "sync.h" #include "taosdef.h" +#include "tglobal.h" #include "tlog.h" #include "ttimer.h" @@ -91,31 +92,61 @@ typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; struct SSyncEnv; typedef struct SSyncEnv SSyncEnv; +struct SRaftStore; +typedef struct SRaftStore SRaftStore; + +struct SVotesGranted; +typedef struct SVotesGranted SVotesGranted; + +struct SVotesResponded; +typedef struct SVotesResponded SVotesResponded; + typedef struct SRaftId { SyncNodeId addr; // typedef uint64_t SyncNodeId; SyncGroupId vgId; // typedef int32_t SyncGroupId; } SRaftId; typedef struct SSyncNode { + // init by SSyncInfo SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - SSyncFSM* pFsm; - - // passed from outside - void* rpcClient; + void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); + // init internal + SNodeInfo me; + int32_t peersNum; + SNodeInfo peers[TSDB_MAX_REPLICA]; + + // raft algorithm + SSyncFSM* pFsm; + SRaftId raftId; + SRaftId peersId[TSDB_MAX_REPLICA]; + int32_t replicaNum; + int32_t quorum; + + // life cycle int32_t refCount; int64_t rid; - SNodeInfo me; - SNodeInfo peers[TSDB_MAX_REPLICA]; - int32_t peersNum; + // tla+ server vars + ESyncState state; + SRaftStore* pRaftStore; - ESyncRole role; - SRaftId raftId; + // tla+ candidate vars + SVotesGranted* pVotesGranted; + SVotesResponded* pVotesResponded; + // tla+ leader vars + SHashObj* pNextIndex; + SHashObj* pMatchIndex; + + // tla+ log vars + SSyncLogStore* pLogStore; + SyncIndex commitIndex; + + // timer tmr_h pPingTimer; int32_t pingTimerMS; uint8_t pingTimerStart; @@ -136,32 +167,21 @@ typedef struct SSyncNode { // callback int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); - int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); - int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); - int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); - int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); - int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); } SSyncNode; SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); - -void syncNodeClose(SSyncNode* pSyncNode); - -void syncNodePingAll(SSyncNode* pSyncNode); - -void syncNodePingPeers(SSyncNode* pSyncNode); - -void syncNodePingSelf(SSyncNode* pSyncNode); - -int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); - -int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); +void syncNodeClose(SSyncNode* pSyncNode); +void syncNodePingAll(SSyncNode* pSyncNode); +void syncNodePingPeers(SSyncNode* pSyncNode); +void syncNodePingSelf(SSyncNode* pSyncNode); +int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode); +int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index cfcf58bee2..b841f2e316 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -26,6 +26,12 @@ extern "C" { #include "syncInt.h" #include "taosdef.h" +typedef struct SVotesGranted { +} SVotesGranted; + +typedef struct SVotesResponded { +} SVotesResponded; + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9cb3a61fff..7e01e7e81c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -88,7 +88,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { } } - pSyncNode->role = TAOS_SYNC_STATE_FOLLOWER; + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId); pSyncNode->pPingTimer = NULL; From 35b3edaea562f5c01d7f63d17634bd5479d534d0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 4 Mar 2022 00:04:57 +0800 Subject: [PATCH 13/13] [TD-13756]: file system stat access func. --- include/os/osDir.h | 2 +- include/os/osFile.h | 24 +++-- source/common/test/commonTests.cpp | 4 +- source/common/test/tmsgTest.cpp | 2 +- source/dnode/vnode/src/tq/tqMetaStore.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 15 ++- source/libs/CMakeLists.txt | 1 - source/libs/catalog/test/catalogTests.cpp | 6 +- source/libs/executor/src/executorimpl.c | 8 +- source/libs/executor/test/executorTests.cpp | 6 +- source/libs/executor/test/sortTests.cpp | 2 +- .../index/src/index_fst_counting_writer.c | 18 ++-- source/libs/parser/test/mockCatalog.cpp | 6 +- source/libs/parser/test/parserTests.cpp | 4 +- source/libs/parser/test/plannerTest.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 4 +- .../libs/scalar/test/filter/filterTests.cpp | 4 +- .../libs/scalar/test/scalar/scalarTests.cpp | 4 +- source/libs/scheduler/test/schedulerTests.cpp | 24 ++--- .../sync/test/syncIOSendMsgClientTest.cpp | 2 +- .../sync/test/syncIOSendMsgServerTest.cpp | 2 +- source/libs/sync/test/syncIOSendMsgTest.cpp | 2 +- source/libs/sync/test/syncIOTickPingTest.cpp | 2 +- source/libs/sync/test/syncIOTickQTest.cpp | 2 +- source/libs/sync/test/syncRaftStoreTest.cpp | 4 +- source/libs/sync/test/syncTest.cpp | 2 +- source/libs/tdb/src/db/tdb.c | 2 +- source/libs/tdb/src/db/tdbUtil.c | 43 ++++---- source/libs/tdb/src/inc/tdbUtil.h | 8 +- source/libs/tdb/test/tdbTest.cpp | 2 +- source/libs/tfs/src/tfs.c | 11 +-- source/libs/tfs/test/tfsTest.cpp | 98 +++++++++---------- source/libs/wal/src/walMeta.c | 22 ++--- source/os/src/osDir.c | 2 +- source/os/src/osFile.c | 39 ++++++++ source/util/test/cacheTest.cpp | 2 +- source/util/test/encodeTest.cpp | 2 +- source/util/test/freelistTest.cpp | 2 +- source/util/test/hashTest.cpp | 4 +- tools/shell/src/backup/shellImport.c | 8 +- 42 files changed, 221 insertions(+), 184 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index eda83ea0ea..223c603352 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -21,7 +21,7 @@ extern "C" { #endif void taosRemoveDir(const char *dirname); -int32_t taosDirExist(char *dirname); +bool taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); diff --git a/include/os/osFile.h b/include/os/osFile.h index cbd5d693c5..7e3a5277c8 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -25,8 +25,12 @@ extern "C" { #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID - // #define close CLOSE_FUNC_TAOS_FORBID - // #define fclose FCLOSE_FUNC_TAOS_FORBID + #define access ACCESS_FUNC_TAOS_FORBID + #define stat STAT_FUNC_TAOS_FORBID + #define lstat LSTAT_FUNC_TAOS_FORBID + #define fstat FSTAT_FUNC_TAOS_FORBID + #define close CLOSE_FUNC_TAOS_FORBID + #define fclose FCLOSE_FUNC_TAOS_FORBID #endif #ifndef PATH_MAX @@ -44,6 +48,12 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_EXCL 0x0080 #define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +#define TD_FILE_ACCESS_EXIST_OK 0x1 +#define TD_FILE_ACCESS_READ_OK 0x2 +#define TD_FILE_ACCESS_WRITE_OK 0x4 +bool taosCheckAccessFile(const char *pathname, int mode); int32_t taosLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile); @@ -51,9 +61,9 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); +int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); - -TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); +bool taosCheckExistFile(const char *pathname); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); @@ -62,7 +72,7 @@ int32_t taosFsyncFile(TdFilePtr pFile); int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); -void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); @@ -71,7 +81,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile); int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); -void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); +void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); @@ -79,7 +89,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); bool taosValidFile(TdFilePtr pFile); -int taosGetErrorFile(TdFilePtr pFile); +int32_t taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 4821d60875..563efe4d54 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,6 +1,4 @@ -#include "tcommon.h" #include -#include #include #pragma GCC diagnostic push @@ -10,6 +8,8 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "tep.h" +#include "tcommon.h" #include "taos.h" #include "tvariant.h" #include "tdef.h" diff --git a/source/common/test/tmsgTest.cpp b/source/common/test/tmsgTest.cpp index ca33d24a8c..e8c3284ea5 100644 --- a/source/common/test/tmsgTest.cpp +++ b/source/common/test/tmsgTest.cpp @@ -1,6 +1,6 @@ #include -#include "gtest/gtest.h" +#include #include "tmsg.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 6d5085ee74..ce00c98ff9 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -90,7 +90,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F char name[pathLen + 10]; strcpy(name, path); - if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { + if (!taosDirExist(name) && taosMkDir(name) != 0) { terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; tqError("failed to create dir:%s since %s ", name, terrstr()); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2e7116969e..7a95820115 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -580,7 +580,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh)); SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith); - if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) { + if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmadF))) { tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD); @@ -614,7 +614,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh)); SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith); - if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) { + if ((pCommith->isLFileSame) && taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmalF))) { tsdbInitDFileEx(pWSmalF, pRSmalF); if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index dcc9700d83..411a166caa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -314,7 +314,7 @@ int tsdbOpenFS(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); tsdbGetRtnSnap(pRepo, &pRepo->rtn); - if (access(current, F_OK) == 0) { + if (taosCheckExistFile(current)) { if (tsdbOpenFSFromCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4d70c429ff..74fb8c1c1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -443,25 +443,24 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { - struct stat dfstat; SDFile df; tsdbInitDFileEx(&df, pDFile); - if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) { + if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) { tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); return 0; } - - if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) { + int64_t file_size = 0; + if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (pDFile->info.size < dfstat.st_size) { + if (pDFile->info.size < file_size) { // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; @@ -480,10 +479,10 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { tsdbCloseDFile(&df); tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), - dfstat.st_size, pDFile->info.size); - } else if (pDFile->info.size > dfstat.st_size) { + file_size, pDFile->info.size); + } else if (pDFile->info.size > file_size) { tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", - REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size); + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 78625d1eed..d766f3cbf1 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,4 +1,3 @@ -add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b417a645be..b05fc7812a 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -14,9 +14,7 @@ */ #include -#include #include -#include "os.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -24,8 +22,10 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wformat" +#include -#include "addr_any.h" +#include "os.h" +#include "tglobal.h" #include "catalog.h" #include "stub.h" #include "taos.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c400ded1c1..d449d7fac4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2215,10 +2215,10 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); if (pColInfoData) { - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - if (f) { - fclose(f); - *(FILE **)pColInfoData->pData = NULL; + TdFilePtr pFile = *(TdFilePtr *)pColInfoData->pData; // TODO refactor + if (pFile != NULL) { + taosCloseFile(&pFile); + *(TdFilePtr *)pColInfoData->pData = NULL; } } } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 262232ceb1..7d3665a8cc 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include #include -#include #include #pragma GCC diagnostic push @@ -26,6 +23,9 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "tglobal.h" +#include "executorimpl.h" +#include "function.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index fc366e4cc8..612b0705ea 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include #include @@ -26,6 +25,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "executorimpl.h" #include "executor.h" #include "stub.h" #include "taos.h" diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 16d9893778..8cb2ff9246 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -63,9 +63,9 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxGetSize(WriterCtx* ctx) { if (ctx->type == TFile) { - struct stat fstat; - stat(ctx->file.buf, &fstat); - return fstat.st_size; + int64_t file_size = 0; + taosStatFile(ctx->file.buf, &file_size, NULL); + return (int)file_size; } return 0; } @@ -99,9 +99,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + int64_t file_size = 0; + taosFStatFile(ctx->file.pFile, &file_size, NULL); + ctx->file.size = (int)file_size; #ifdef USE_MMAP ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif @@ -142,8 +142,10 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { #endif } if (ctx->file.readOnly == false) { - struct stat fstat; - stat(ctx->file.buf, &fstat); + int64_t file_size = 0; + taosStatFile(ctx->file.buf, &file_size, NULL); + // struct stat fstat; + // stat(ctx->file.buf, &fstat); // indexError("write file size: %d", (int)(fstat.st_size)); } if (remove) { unlink(ctx->file.buf); } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 457d586ea4..e626fc68ae 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -13,19 +13,17 @@ * along with this program. If not, see . */ -#include "mockCatalog.h" - #include - #include "stub.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat" -#include "addr_any.h" +#include #pragma GCC diagnostic pop +#include "mockCatalog.h" namespace { void generateTestT1(MockCatalogService* mcs) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index b971760132..b80efe09ed 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include #include #include -#include "tglobal.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -25,6 +23,8 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "function.h" +#include "tglobal.h" #include "astGenerator.h" #include "parserInt.h" #include "taos.h" diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 43daff9fec..095e2f4f97 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include #include @@ -25,6 +24,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "function.h" #include "astGenerator.h" #include "parserInt.h" #include "taos.h" diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8ad5a76388..94d4260696 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,9 +25,11 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" @@ -37,7 +38,6 @@ #include "planner.h" #include "qworker.h" #include "stub.h" -#include "addr_any.h" #include "executor.h" #include "dataSinkMgt.h" diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index fafc1ea42e..13829618a2 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,15 +25,16 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" #include "tep.h" #include "stub.h" -#include "addr_any.h" #include "scalar.h" #include "nodes.h" #include "tlog.h" diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index b9aef99088..8eef1836a5 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,15 +25,16 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" #include "tep.h" #include "stub.h" -#include "addr_any.h" #include "scalar.h" #include "nodes.h" #include "tlog.h" diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 11ed3335e6..1a34c20ba0 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -14,19 +14,8 @@ */ #include -#include #include -#include "os.h" - -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "catalog.h" -#include "scheduler.h" -#include "tep.h" -#include "trpc.h" - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" @@ -34,10 +23,21 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wreturn-type" #pragma GCC diagnostic ignored "-Wformat" +#include + +#include "os.h" + +#include "tglobal.h" +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "scheduler.h" +#include "tep.h" +#include "trpc.h" #include "schedulerInt.h" #include "stub.h" -#include "addr_any.h" #include "tref.h" namespace { diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp index 9e8c7c8b65..0d06c4f811 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOSendMsgServerTest.cpp index 8af9344ed6..1582e097d3 100644 --- a/source/libs/sync/test/syncIOSendMsgServerTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgServerTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index a297981ee5..9b2cfcf1d8 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 1559b57585..42b9a73432 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 90304079e3..3d31c596bf 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index e533b89a92..9cfbe5a4ea 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,6 +1,6 @@ -#include "syncRaftStore.h" #include -#include "gtest/gtest.h" +#include +#include "syncRaftStore.h" #include "syncIO.h" #include "syncInt.h" diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 0b397ef921..c1c5658aba 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/tdb/src/db/tdb.c b/source/libs/tdb/src/db/tdb.c index cc3b7fa6b9..65d4cf80cc 100644 --- a/source/libs/tdb/src/db/tdb.c +++ b/source/libs/tdb/src/db/tdb.c @@ -90,7 +90,7 @@ int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) { // get page file from the env, if not opened yet, open it pPgFile = NULL; snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname); - fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0); + fileExist = taosCheckExistFile(fname); if (fileExist) { tdbGnrtFileID(dbfname, fileid, false); pPgFile = tdbEnvGetPageFile(pEnv, fileid); diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index fa9a3297da..fe0f3befd6 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -16,16 +16,16 @@ #include "tdbInt.h" int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { - struct stat statbuf; + int64_t stDev = 0, stIno = 0; - if (stat(fname, &statbuf) < 0) { + if (taosDevInoFile(fname, &stDev, &stIno) < 0) { return -1; } memset(fileid, 0, TDB_FILE_ID_LEN); - ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; - ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[0] = stDev; + ((uint64_t *)fileid)[1] = stIno; if (unique) { ((uint64_t *)fileid)[2] = rand(); } @@ -33,35 +33,34 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { return 0; } -int tdbCheckFileAccess(const char *pathname, int mode) { - int flags = 0; +// int tdbCheckFileAccess(const char *pathname, int mode) { +// int flags = 0; - if (mode & TDB_F_OK) { - flags |= F_OK; - } +// if (mode & TDB_F_OK) { +// flags |= F_OK; +// } - if (mode & TDB_R_OK) { - flags |= R_OK; - } +// if (mode & TDB_R_OK) { +// flags |= R_OK; +// } - if (mode & TDB_W_OK) { - flags |= W_OK; - } +// if (mode & TDB_W_OK) { +// flags |= W_OK; +// } - return access(pathname, flags); -} +// return access(pathname, flags); +// } int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { - struct stat st; int ret; - - ret = stat(fname, &st); + int64_t file_size = 0; + ret = taosStatFile(fname, &file_size, NULL); if (ret != 0) { return -1; } - ASSERT(st.st_size % pgSize == 0); + ASSERT(file_size % pgSize == 0); - *pSize = st.st_size / pgSize; + *pSize = file_size / pgSize; return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 8108e5aba6..ca05790f77 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -30,10 +30,10 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); -#define TDB_F_OK 0x1 -#define TDB_R_OK 0x2 -#define TDB_W_OK 0x4 -int tdbCheckFileAccess(const char *pathname, int mode); +// #define TDB_F_OK 0x1 +// #define TDB_R_OK 0x2 +// #define TDB_W_OK 0x4 +// int tdbCheckFileAccess(const char *pathname, int mode); int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5ab0b4c0f1..ad550c7804 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,4 +1,4 @@ -#include "gtest/gtest.h" +#include #include "tdb.h" diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 944e67c863..26fa90bdef 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -389,7 +389,6 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { char dirName[TSDB_FILENAME_LEN] = "\0"; - struct stat pstat; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); @@ -422,19 +421,13 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { return -1; } - if (access(dirName, W_OK | R_OK | F_OK) != 0) { + if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; } - if (stat(dirName, &pstat) < 0) { - fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!S_ISDIR(pstat.st_mode)) { + if (!taosIsDir(dirName)) { fError("failed to mount %s to FS since not a directory", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index af66304f84..9cb914a670 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -119,29 +119,29 @@ TEST_F(TfsTest, 03_Dir) { char p1[] = "p1"; char ap1[128] = {0}; snprintf(ap1, 128, "%s%s%s", root, TD_DIRSEP, p1); - EXPECT_NE(taosDirExist(ap1), 0); + EXPECT_NE(taosDirExist(ap1), 1); EXPECT_EQ(tfsMkdir(pTfs, p1), 0); - EXPECT_EQ(taosDirExist(ap1), 0); + EXPECT_EQ(taosDirExist(ap1), 1); char p2[] = "p2"; char ap2[128] = {0}; snprintf(ap2, 128, "%s%s%s", root, TD_DIRSEP, p2); SDiskID did = {0}; - EXPECT_NE(taosDirExist(ap2), 0); + EXPECT_NE(taosDirExist(ap2), 1); EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); - EXPECT_EQ(taosDirExist(ap2), 0); + EXPECT_EQ(taosDirExist(ap2), 1); char p3[] = "p3/p2/p1/p0"; char ap3[128] = {0}; snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3); - EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(taosDirExist(ap3), 1); EXPECT_NE(tfsMkdir(pTfs, p3), 0); EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); - EXPECT_EQ(taosDirExist(ap3), 0); + EXPECT_EQ(taosDirExist(ap3), 1); EXPECT_EQ(tfsRmdir(pTfs, p3), 0); - EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(taosDirExist(ap3), 1); char p45[] = "p5"; char p44[] = "p4"; @@ -149,12 +149,12 @@ TEST_F(TfsTest, 03_Dir) { char ap4[128] = {0}; snprintf(ap4, 128, "%s%s%s", root, TD_DIRSEP, p4); - EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_NE(taosDirExist(ap4), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); - EXPECT_EQ(taosDirExist(ap4), 0); + EXPECT_EQ(taosDirExist(ap4), 1); EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); - EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_NE(taosDirExist(ap4), 1); tfsClose(pTfs); } @@ -251,9 +251,9 @@ TEST_F(TfsTest, 04_File) { char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); - EXPECT_EQ(taosDirExist(af2), 0); + EXPECT_EQ(taosDirExist(af2), 1); tfsRemoveFile(&f2); - EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(taosDirExist(af2), 1); { STfsDir *pDir = tfsOpendir(pTfs, "t3"); @@ -529,35 +529,35 @@ TEST_F(TfsTest, 05_MultiDisk) { snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1); char ap23[128] = {0}; snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1); - EXPECT_NE(taosDirExist(ap00), 0); - EXPECT_NE(taosDirExist(ap01), 0); - EXPECT_NE(taosDirExist(ap10), 0); - EXPECT_NE(taosDirExist(ap11), 0); - EXPECT_NE(taosDirExist(ap12), 0); - EXPECT_NE(taosDirExist(ap20), 0); - EXPECT_NE(taosDirExist(ap21), 0); - EXPECT_NE(taosDirExist(ap22), 0); - EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_NE(taosDirExist(ap00), 1); + EXPECT_NE(taosDirExist(ap01), 1); + EXPECT_NE(taosDirExist(ap10), 1); + EXPECT_NE(taosDirExist(ap11), 1); + EXPECT_NE(taosDirExist(ap12), 1); + EXPECT_NE(taosDirExist(ap20), 1); + EXPECT_NE(taosDirExist(ap21), 1); + EXPECT_NE(taosDirExist(ap22), 1); + EXPECT_NE(taosDirExist(ap23), 1); EXPECT_EQ(tfsMkdir(pTfs, p1), 0); - EXPECT_EQ(taosDirExist(ap00), 0); - EXPECT_EQ(taosDirExist(ap01), 0); - EXPECT_EQ(taosDirExist(ap10), 0); - EXPECT_EQ(taosDirExist(ap11), 0); - EXPECT_EQ(taosDirExist(ap12), 0); - EXPECT_EQ(taosDirExist(ap20), 0); - EXPECT_EQ(taosDirExist(ap21), 0); - EXPECT_EQ(taosDirExist(ap22), 0); - EXPECT_EQ(taosDirExist(ap23), 0); + EXPECT_EQ(taosDirExist(ap00), 1); + EXPECT_EQ(taosDirExist(ap01), 1); + EXPECT_EQ(taosDirExist(ap10), 1); + EXPECT_EQ(taosDirExist(ap11), 1); + EXPECT_EQ(taosDirExist(ap12), 1); + EXPECT_EQ(taosDirExist(ap20), 1); + EXPECT_EQ(taosDirExist(ap21), 1); + EXPECT_EQ(taosDirExist(ap22), 1); + EXPECT_EQ(taosDirExist(ap23), 1); EXPECT_EQ(tfsRmdir(pTfs, p1), 0); - EXPECT_NE(taosDirExist(ap00), 0); - EXPECT_NE(taosDirExist(ap01), 0); - EXPECT_NE(taosDirExist(ap10), 0); - EXPECT_NE(taosDirExist(ap11), 0); - EXPECT_NE(taosDirExist(ap12), 0); - EXPECT_NE(taosDirExist(ap20), 0); - EXPECT_NE(taosDirExist(ap21), 0); - EXPECT_NE(taosDirExist(ap22), 0); - EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_NE(taosDirExist(ap00), 1); + EXPECT_NE(taosDirExist(ap01), 1); + EXPECT_NE(taosDirExist(ap10), 1); + EXPECT_NE(taosDirExist(ap11), 1); + EXPECT_NE(taosDirExist(ap12), 1); + EXPECT_NE(taosDirExist(ap20), 1); + EXPECT_NE(taosDirExist(ap21), 1); + EXPECT_NE(taosDirExist(ap22), 1); + EXPECT_NE(taosDirExist(ap23), 1); char p2[] = "p2"; char _ap21[128] = {0}; @@ -565,22 +565,22 @@ TEST_F(TfsTest, 05_MultiDisk) { SDiskID did = {0}; did.level = 2; did.id = 1; - EXPECT_NE(taosDirExist(_ap21), 0); + EXPECT_NE(taosDirExist(_ap21), 1); EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); - EXPECT_EQ(taosDirExist(_ap21), 0); + EXPECT_EQ(taosDirExist(_ap21), 1); char p3[] = "p3/p2/p1/p0"; char _ap12[128] = {0}; snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3); did.level = 1; did.id = 2; - EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(taosDirExist(_ap12), 1); EXPECT_NE(tfsMkdir(pTfs, p3), 0); EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); - EXPECT_EQ(taosDirExist(_ap12), 0); + EXPECT_EQ(taosDirExist(_ap12), 1); EXPECT_EQ(tfsRmdir(pTfs, p3), 0); - EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(taosDirExist(_ap12), 1); char p45[] = "p5"; char p44[] = "p4"; @@ -590,12 +590,12 @@ TEST_F(TfsTest, 05_MultiDisk) { did.level = 2; did.id = 2; - EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_NE(taosDirExist(_ap22), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); - EXPECT_EQ(taosDirExist(_ap22), 0); + EXPECT_EQ(taosDirExist(_ap22), 1); EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); - EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_NE(taosDirExist(_ap22), 1); } //------------- File -----------------// @@ -660,7 +660,7 @@ TEST_F(TfsTest, 05_MultiDisk) { char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2); - EXPECT_EQ(taosDirExist(af2), 0); + EXPECT_EQ(taosDirExist(af2), 1); tfsRemoveFile(&f2); { @@ -678,7 +678,7 @@ TEST_F(TfsTest, 05_MultiDisk) { tfsClosedir(pDir); } - EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(taosDirExist(af2), 1); EXPECT_GT(tfsCopyFile(&f1, &f2), 0); { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index ae0b0bd849..e64260c541 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -64,10 +64,10 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - struct stat statbuf; - stat(fnameStr, &statbuf); - int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); - pLastFileInfo->fileSize = statbuf.st_size; + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); + int readSize = TMIN(WAL_MAX_SIZE + 2, file_size); + pLastFileInfo->fileSize = file_size; TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pFile == NULL) { @@ -177,11 +177,11 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - struct stat statbuf; - stat(fnameStr, &statbuf); + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); - if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) { - pLastFileInfo->fileSize = statbuf.st_size; + if (oldSz != newSz || pLastFileInfo->fileSize != file_size) { + pLastFileInfo->fileSize = file_size; pWal->vers.lastVer = walScanLogGetLastVer(pWal); ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; ASSERT(pWal->vers.lastVer != -1); @@ -395,9 +395,9 @@ int walLoadMeta(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); // read metafile - struct stat statbuf; - stat(fnameStr, &statbuf); - int size = statbuf.st_size; + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); + int size = (int)file_size; char* buf = malloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 91ef97e66b..e0f5f80dab 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -60,7 +60,7 @@ void taosRemoveDir(const char *dirname) { //printf("dir:%s is removed\n", dirname); } -int32_t taosDirExist(char *dirname) { return access(dirname, F_OK); } +bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c2e9b52f88..37004e9d70 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -186,6 +186,27 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; #endif } +int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (stDev != NULL) { + *stDev = fileStat.st_dev; + } + + if (stIno != NULL) { + *stIno = fileStat.st_ino; + } + + return 0; +#endif +} void autoDelFileListAdd(const char *path) { return; } @@ -733,3 +754,21 @@ int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } +bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { + int flags = 0; + + if (tdFileAccessOptions & TD_FILE_ACCESS_EXIST_OK) { + flags |= F_OK; + } + + if (tdFileAccessOptions & TD_FILE_ACCESS_READ_OK) { + flags |= R_OK; + } + + if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { + flags |= W_OK; + } + + return access(pathname, flags) == 0; +} +bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; \ No newline at end of file diff --git a/source/util/test/cacheTest.cpp b/source/util/test/cacheTest.cpp index 970f1c23a9..4dde374bdd 100644 --- a/source/util/test/cacheTest.cpp +++ b/source/util/test/cacheTest.cpp @@ -1,7 +1,7 @@ -#include "os.h" #include #include +#include "os.h" #include "taos.h" #include "tcache.h" diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 5505a6207f..7b993ebf35 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -1,6 +1,6 @@ #include -#include "gtest/gtest.h" +#include #include "tencode.h" diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index cf11d6b5bf..7fd3d693fb 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -1,4 +1,4 @@ -#include "gtest/gtest.h" +#include #include "tfreelist.h" diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index ac1bae2434..4c6ee8d10c 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -1,9 +1,9 @@ -#include "os.h" #include #include -#include #include +#include "os.h" +#include "taosdef.h" #include "thash.h" #include "taos.h" diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index c0ab7ef461..36489a448b 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -93,8 +93,7 @@ static void shellCheckTablesSQLFile(const char *directoryName) { sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName); - struct stat fstat; - if (stat(shellTablesSQLFile, &fstat) < 0) { + if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) { shellTablesSQLFile[0] = 0; } } @@ -109,13 +108,12 @@ static void shellMallocSQLFiles() static void shellGetDirectoryFileList(char *inputDir) { - struct stat fileStat; - if (stat(inputDir, &fileStat) < 0) { + if (!taosDirExist(inputDir)) { fprintf(stderr, "ERROR: %s not exist\n", inputDir); exit(0); } - if (fileStat.st_mode & S_IFDIR) { + if (taosIsDir(inputDir)) { shellCheckTablesSQLFile(inputDir); shellSQLFileNum = shellGetFilesNum(inputDir, "sql"); int totalSQLFileNum = shellSQLFileNum;