From 007552afb2ab7199407d51ca829c993d4c4db896 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 03:23:31 +0000 Subject: [PATCH 001/118] more --- source/libs/tdb/inc/tdb.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index eee8f8ed33..02b4422882 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -26,26 +26,23 @@ extern "C" { #define TDB_PUBLIC #define TDB_STATIC static -typedef enum { - TDB_BTREE_T = 0, - TDB_HASH_T, - TDB_HEAP_T, -} tdb_db_t; +typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; -// Forward declaration +// Forward declarations typedef struct TDB TDB; typedef struct TDB_CURSOR TDB_CURSOR; -// SKey typedef struct { void* bdata; uint32_t size; } TDB_KEY, TDB_VALUE; // TDB Operations -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type); -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); +int tdbCreateDB(TDB** dbpp, tdb_db_t type); +int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); +int tdbCloseDB(TDB* dbp, uint32_t flags); +int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); +int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); #ifdef __cplusplus } From 1e84bce89fac773651535066a54239a56572e4ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 07:53:02 +0000 Subject: [PATCH 002/118] more tkv --- source/libs/tdb/inc/tdb.h | 14 +++++ source/libs/tdb/src/inc/tdbDB.h | 4 -- source/libs/tdb/src/inc/tdbDef.h | 1 + .../src/inc/{tdbBufPool.h => tdb_mpfile.h} | 22 ++++---- source/libs/tdb/src/inc/tdb_mpool.h | 35 ++++++++++++ source/libs/tdb/src/mpool/tdbBufPool.c | 54 ------------------- source/libs/tdb/src/mpool/tdb_mpool.c | 14 +++++ 7 files changed, 74 insertions(+), 70 deletions(-) rename source/libs/tdb/src/inc/{tdbBufPool.h => tdb_mpfile.h} (57%) create mode 100644 source/libs/tdb/src/inc/tdb_mpool.h delete mode 100644 source/libs/tdb/src/mpool/tdbBufPool.c create mode 100644 source/libs/tdb/src/mpool/tdb_mpool.c diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 02b4422882..3a1523e6c6 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -30,6 +30,8 @@ typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; // Forward declarations typedef struct TDB TDB; +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; typedef struct TDB_CURSOR TDB_CURSOR; typedef struct { @@ -44,6 +46,18 @@ int tdbCloseDB(TDB* dbp, uint32_t flags); int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); +// TDB_MPOOL +int tdbOpenMPool(TDB_MPOOL** mp); +int tdbCloseMPool(TDB_MPOOL* mp); + +// TDB_MPFILE +int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); +int tdbCloseMPFile(TDB_MPFILE** mpf); + +// TDB_CURSOR +int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); +int tdbCloseCurosr(TDB_CURSOR* tdbcp); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index d0ef9e22d0..261fb587b4 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -25,10 +25,6 @@ extern "C" { #endif -typedef struct { - // TODO -} TDB_MPOOL; - typedef struct { int fd; } TDB_FH; diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdbDef.h index 4b5e54368b..48a6831f86 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdbDef.h @@ -17,6 +17,7 @@ #define _TD_TDB_DEF_H_ #include "os.h" +#include "tlist.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdb_mpfile.h similarity index 57% rename from source/libs/tdb/src/inc/tdbBufPool.h rename to source/libs/tdb/src/inc/tdb_mpfile.h index 5200d22faa..8eaafafd41 100644 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ b/source/libs/tdb/src/inc/tdb_mpfile.h @@ -13,27 +13,25 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_BUF_POOL_H_ -#define _TD_TDB_BUF_POOL_H_ +#ifndef _TD_TDB_MPFILE_H_ +#define _TD_TDB_MPFILE_H_ -#include "tdbPage.h" +#include "tdbDef.h" +#include "tdb_mpool.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STdbBufPool STdbBufPool; +struct TDB_MPFILE { + TDB_MPOOL *mp; // memory pool used to get/put pages in this file -int tbpOpen(STdbBufPool **ppTkvBufPool); -int tbpClose(STdbBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); -int tbpDelPage(STdbBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STdbBufPool *pTkvBufPool); + char *fname; + int fd; +}; #ifdef __cplusplus } #endif -#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_MPFILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h new file mode 100644 index 0000000000..9e34362c40 --- /dev/null +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_MPOOL_H_ +#define _TD_TDB_MPOOL_H_ + +#include "tdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB_MPOOL { + pthread_mutex_t mutex; + int64_t cachesize; + pgsize_t pgsize; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/mpool/tdbBufPool.c b/source/libs/tdb/src/mpool/tdbBufPool.c deleted file mode 100644 index bc3c386b0f..0000000000 --- a/source/libs/tdb/src/mpool/tdbBufPool.c +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "thash.h" -#include "tlist.h" - -#include "tdbBufPool.h" -#include "tdbDiskMgr.h" -#include "tdbPage.h" - -struct SFrameIdWrapper { - TD_SLIST_NODE(SFrameIdWrapper); - frame_id_t id; -}; - -struct STdbBufPool { - STdbPage* pages; - STkvDiskMgr* pDiskMgr; - SHashObj* pgTb; // page_id_t --> frame_id_t - TD_SLIST(SFrameIdWrapper) freeList; - pthread_mutex_t mutex; -}; - -typedef struct STkvLRUReplacer { -} STkvLRUReplacer; - -typedef struct STkvLFUReplacer { -} STkvLFUReplacer; - -typedef struct STkvCLKReplacer { -} STkvCLKReplacer; - -typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; - -typedef struct STkvReplacer { - tkv_replacer_t type; - union { - STkvLRUReplacer lruRep; - STkvLFUReplacer lfuRep; - STkvCLKReplacer clkRep; - }; -} STkvReplacer; \ No newline at end of file diff --git a/source/libs/tdb/src/mpool/tdb_mpool.c b/source/libs/tdb/src/mpool/tdb_mpool.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/mpool/tdb_mpool.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From 73f802c3f5a16217ee9ff51fd586b0b7698da4ad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:17:40 +0000 Subject: [PATCH 003/118] more tkv --- source/libs/tdb/inc/tdb.h | 26 ++++---- .../src/{inc/tdb_mpool.h => devinc/tdb_inc.h} | 14 +--- .../libs/tdb/src/{inc => devinc}/tdb_mpfile.h | 18 +++-- source/libs/tdb/src/devinc/tdb_mpool.h | 65 +++++++++++++++++++ source/libs/tdb/src/inc/tdbDB.h | 4 +- 5 files changed, 97 insertions(+), 30 deletions(-) rename source/libs/tdb/src/{inc/tdb_mpool.h => devinc/tdb_inc.h} (77%) rename source/libs/tdb/src/{inc => devinc}/tdb_mpfile.h (64%) create mode 100644 source/libs/tdb/src/devinc/tdb_mpool.h diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 3a1523e6c6..5264707eb8 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -29,10 +29,10 @@ extern "C" { typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; // Forward declarations -typedef struct TDB TDB; -typedef struct TDB_MPOOL TDB_MPOOL; -typedef struct TDB_MPFILE TDB_MPFILE; -typedef struct TDB_CURSOR TDB_CURSOR; +typedef struct TDB TDB; +// typedef struct TDB_MPOOL TDB_MPOOL; +// typedef struct TDB_MPFILE TDB_MPFILE; +// typedef struct TDB_CURSOR TDB_CURSOR; typedef struct { void* bdata; @@ -46,17 +46,17 @@ int tdbCloseDB(TDB* dbp, uint32_t flags); int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); -// TDB_MPOOL -int tdbOpenMPool(TDB_MPOOL** mp); -int tdbCloseMPool(TDB_MPOOL* mp); +// // TDB_MPOOL +// int tdbOpenMPool(TDB_MPOOL** mp); +// int tdbCloseMPool(TDB_MPOOL* mp); -// TDB_MPFILE -int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); -int tdbCloseMPFile(TDB_MPFILE** mpf); +// // TDB_MPFILE +// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); +// int tdbCloseMPFile(TDB_MPFILE** mpf); -// TDB_CURSOR -int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); -int tdbCloseCurosr(TDB_CURSOR* tdbcp); +// // TDB_CURSOR +// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); +// int tdbCloseCurosr(TDB_CURSOR* tdbcp); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/devinc/tdb_inc.h similarity index 77% rename from source/libs/tdb/src/inc/tdb_mpool.h rename to source/libs/tdb/src/devinc/tdb_inc.h index 9e34362c40..c05247c44a 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/devinc/tdb_inc.h @@ -13,23 +13,15 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_MPOOL_H_ -#define _TD_TDB_MPOOL_H_ - -#include "tdbDef.h" +#ifndef _TD_TDB_INC_H_ +#define _TD_TDB_INC_H_ #ifdef __cplusplus extern "C" { #endif -struct TDB_MPOOL { - pthread_mutex_t mutex; - int64_t cachesize; - pgsize_t pgsize; -}; - #ifdef __cplusplus } #endif -#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/inc/tdb_mpfile.h b/source/libs/tdb/src/devinc/tdb_mpfile.h similarity index 64% rename from source/libs/tdb/src/inc/tdb_mpfile.h rename to source/libs/tdb/src/devinc/tdb_mpfile.h index 8eaafafd41..74593af2f6 100644 --- a/source/libs/tdb/src/inc/tdb_mpfile.h +++ b/source/libs/tdb/src/devinc/tdb_mpfile.h @@ -23,11 +23,21 @@ extern "C" { #endif -struct TDB_MPFILE { - TDB_MPOOL *mp; // memory pool used to get/put pages in this file +// Exposed handle +typedef struct TDB_MPFILE TDB_MPFILE; - char *fname; - int fd; +// Exposed apis +int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPFClose(TDB_MPFILE *mpf); +int tdbMPFGet(TDB_MPFILE *mpf, pgid_t pgid, void *addr); +int tdbMPFPut(TDB_MPOOL *mpf, pgid_t pgid, void *addr); + +// Hidden structures +struct TDB_MPFILE { + uint8_t fuid[20]; // file unique ID + TDB_MPOOL *mp; // underlying memory pool + char * fname; // file name + int fd; // fd }; #ifdef __cplusplus diff --git a/source/libs/tdb/src/devinc/tdb_mpool.h b/source/libs/tdb/src/devinc/tdb_mpool.h new file mode 100644 index 0000000000..4acf8bef8c --- /dev/null +++ b/source/libs/tdb/src/devinc/tdb_mpool.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_MPOOL_H_ +#define _TD_TDB_MPOOL_H_ + +#include "tdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Exposed handle +typedef struct TDB_MPOOL TDB_MPOOL; + +// Exposed apis +int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbCloseMP(TDB_MPOOL *mp); +int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); + +// Hidden impls +#define TDB_FILE_UID_LEN 20 + +typedef struct { + uint8_t fuid[TDB_FILE_UID_LEN]; + pgid_t pgid; +} mp_pgid_t; + +typedef struct MP_PAGE { + SRWLatch rwLatch; + mp_pgid_t mpgid; + uint8_t dirty; + int32_t pinRef; + // TD_DLIST_NODE(MP_PAGE); // The free list handle + char *page[]; +} MP_PAGE; + +struct TDB_MPOOL { + pthread_mutex_t mutex; + int64_t cachesize; + pgsize_t pgsize; + MP_PAGE * pages; + // TD_DBLIST(MP_PAGE) freeList; + // TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool + // Hash hash; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 261fb587b4..c246c18775 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -40,8 +40,8 @@ struct TDB { TDB_HEAP * heap; } dbam; // db access method - TDB_FH * fhp; // The backup file handle - TDB_MPOOL *mph; // The memory pool handle + // TDB_FH * fhp; // The backup file handle + // TDB_MPOOL *mph; // The memory pool handle }; #ifdef __cplusplus From 1793f616cafa5b8e28931df2b86626d4f15e7cd7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:31:35 +0000 Subject: [PATCH 004/118] refact --- source/libs/tdb/CMakeLists.txt | 4 +- source/libs/tdb/inc/tdb.h | 38 +++++----- source/libs/tdb/src/db/tdbDB.c | 4 +- source/libs/tdb/src/devinc/tdb_inc.h | 27 ------- source/libs/tdb/src/dmgr/tdbDiskMgr.c | 74 ------------------- source/libs/tdb/src/inc/tdbBtree.h | 35 --------- source/libs/tdb/src/inc/tdbDB.h | 12 ++- source/libs/tdb/src/inc/tdbDiskMgr.h | 40 ---------- source/libs/tdb/src/inc/tdbHash.h | 35 --------- source/libs/tdb/src/inc/tdbHeap.h | 35 --------- source/libs/tdb/src/inc/tdbPage.h | 43 ----------- .../libs/tdb/src/inc/{tdbDef.h => tdb_inc.h} | 7 +- .../libs/tdb/src/{devinc => inc}/tdb_mpfile.h | 0 .../libs/tdb/src/{devinc => inc}/tdb_mpool.h | 2 +- source/libs/tdb/src/mpool/tdb_mpool.c | 14 ---- 15 files changed, 33 insertions(+), 337 deletions(-) delete mode 100644 source/libs/tdb/src/devinc/tdb_inc.h delete mode 100644 source/libs/tdb/src/dmgr/tdbDiskMgr.c delete mode 100644 source/libs/tdb/src/inc/tdbBtree.h delete mode 100644 source/libs/tdb/src/inc/tdbDiskMgr.h delete mode 100644 source/libs/tdb/src/inc/tdbHash.h delete mode 100644 source/libs/tdb/src/inc/tdbHeap.h delete mode 100644 source/libs/tdb/src/inc/tdbPage.h rename source/libs/tdb/src/inc/{tdbDef.h => tdb_inc.h} (92%) rename source/libs/tdb/src/{devinc => inc}/tdb_mpfile.h (100%) rename source/libs/tdb/src/{devinc => inc}/tdb_mpool.h (98%) delete mode 100644 source/libs/tdb/src/mpool/tdb_mpool.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index eb63f2b144..bb8904117c 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ -set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr") +set(TDB_SUBDIRS "db") foreach(TDB_SUBDIR ${TDB_SUBDIRS}) aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) endforeach() @@ -18,5 +18,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - add_subdirectory(test) + # add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 5264707eb8..faf9208f8f 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,29 +22,29 @@ extern "C" { #endif -#define TDB_EXTERN -#define TDB_PUBLIC -#define TDB_STATIC static +// #define TDB_EXTERN +// #define TDB_PUBLIC +// #define TDB_STATIC static -typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; +// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; -// Forward declarations -typedef struct TDB TDB; -// typedef struct TDB_MPOOL TDB_MPOOL; -// typedef struct TDB_MPFILE TDB_MPFILE; -// typedef struct TDB_CURSOR TDB_CURSOR; +// // Forward declarations +// typedef struct TDB TDB; +// // typedef struct TDB_MPOOL TDB_MPOOL; +// // typedef struct TDB_MPFILE TDB_MPFILE; +// // typedef struct TDB_CURSOR TDB_CURSOR; -typedef struct { - void* bdata; - uint32_t size; -} TDB_KEY, TDB_VALUE; +// typedef struct { +// void* bdata; +// uint32_t size; +// } TDB_KEY, TDB_VALUE; -// TDB Operations -int tdbCreateDB(TDB** dbpp, tdb_db_t type); -int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -int tdbCloseDB(TDB* dbp, uint32_t flags); -int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); -int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); +// // TDB Operations +// int tdbCreateDB(TDB** dbpp, tdb_db_t type); +// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); +// int tdbCloseDB(TDB* dbp, uint32_t flags); +// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); +// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); // // TDB_MPOOL // int tdbOpenMPool(TDB_MPOOL** mp); diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c index eaf85ea4a1..9fac9887d0 100644 --- a/source/libs/tdb/src/db/tdbDB.c +++ b/source/libs/tdb/src/db/tdbDB.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#if 0 #include "tdbDB.h" #include "tdb.h" @@ -83,4 +84,5 @@ TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32 TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { // TODO return 0; -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/devinc/tdb_inc.h b/source/libs/tdb/src/devinc/tdb_inc.h deleted file mode 100644 index c05247c44a..0000000000 --- a/source/libs/tdb/src/devinc/tdb_inc.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_INC_H_ -#define _TD_TDB_INC_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/dmgr/tdbDiskMgr.c b/source/libs/tdb/src/dmgr/tdbDiskMgr.c deleted file mode 100644 index 71ab5f2589..0000000000 --- a/source/libs/tdb/src/dmgr/tdbDiskMgr.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbDiskMgr.h" - -struct STkvDiskMgr { - char * fname; - uint16_t pgsize; - FileFd fd; - pgid_t npgid; -}; - -#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - STkvDiskMgr *pDiskMgr; - - pDiskMgr = malloc(sizeof(*pDiskMgr)); - if (pDiskMgr == NULL) { - return -1; - } - - pDiskMgr->fname = strdup(fname); - if (pDiskMgr->fname == NULL) { - free(pDiskMgr); - return -1; - } - pDiskMgr->pgsize = pgsize; - pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pDiskMgr->fd < 0) { - free(pDiskMgr->fname); - free(pDiskMgr); - return -1; - } - - *ppDiskMgr = pDiskMgr; - - return 0; -} - -int tdmClose(STkvDiskMgr *pDiskMgr) { - close(pDiskMgr->fd); - free(pDiskMgr->fname); - free(pDiskMgr); - return 0; -} - -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } - -int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h deleted file mode 100644 index 28258b8e60..0000000000 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_BTREE_H_ -#define _TD_TDB_BTREE_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t root; // root page number -} TDB_BTREE; - -TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index c246c18775..7607580d3e 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -18,8 +18,6 @@ #include "tdb.h" #include "tdbBtree.h" -#include "tdbHash.h" -#include "tdbHeap.h" #ifdef __cplusplus extern "C" { @@ -34,11 +32,11 @@ struct TDB { tdb_db_t type; char * fname; char * dbname; - union { - TDB_BTREE *btree; - TDB_HASH * hash; - TDB_HEAP * heap; - } dbam; // db access method + // union { + // TDB_BTREE *btree; + // TDB_HASH * hash; + // TDB_HEAP * heap; + // } dbam; // db access method // TDB_FH * fhp; // The backup file handle // TDB_MPOOL *mph; // The memory pool handle diff --git a/source/libs/tdb/src/inc/tdbDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h deleted file mode 100644 index b83a147437..0000000000 --- a/source/libs/tdb/src/inc/tdbDiskMgr.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDISK_MGR_H_ -#define _TD_TDISK_MGR_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tdbDef.h" - -typedef struct STkvDiskMgr STkvDiskMgr; - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(STkvDiskMgr *pDiskMgr); -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(STkvDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDISK_MGR_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHash.h b/source/libs/tdb/src/inc/tdbHash.h deleted file mode 100644 index 8219bda2f8..0000000000 --- a/source/libs/tdb/src/inc/tdbHash.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_HASH_H_ -#define _TD_TDB_HASH_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - // TODO -} TDB_HASH; - -TDB_PUBLIC int tdbInitHashDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_HASH_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHeap.h b/source/libs/tdb/src/inc/tdbHeap.h deleted file mode 100644 index 25a812fa5f..0000000000 --- a/source/libs/tdb/src/inc/tdbHeap.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_HEAP_H_ -#define _TD_TDB_HEAP_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - // TODO -} TDB_HEAP; - -TDB_PUBLIC int tdbInitHeapDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_HEAP_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h deleted file mode 100644 index e7245b6c39..0000000000 --- a/source/libs/tdb/src/inc/tdbPage.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_PAGE_H_ -#define _TD_TKV_PAGE_H_ - -#include "os.h" -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t pgid; - int32_t pinCount; - bool idDirty; - char* pData; -} STdbPage; - -typedef struct { - uint16_t dbver; - uint16_t pgsize; - uint32_t cksm; -} STdbPgHdr; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TKV_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdb_inc.h similarity index 92% rename from source/libs/tdb/src/inc/tdbDef.h rename to source/libs/tdb/src/inc/tdb_inc.h index 48a6831f86..1e9c3dd8cc 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -13,11 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_DEF_H_ -#define _TD_TDB_DEF_H_ +#ifndef _TD_TDB_INC_H_ +#define _TD_TDB_INC_H_ #include "os.h" -#include "tlist.h" #ifdef __cplusplus extern "C" { @@ -41,4 +40,4 @@ typedef int32_t pgsize_t; } #endif -#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/devinc/tdb_mpfile.h b/source/libs/tdb/src/inc/tdb_mpfile.h similarity index 100% rename from source/libs/tdb/src/devinc/tdb_mpfile.h rename to source/libs/tdb/src/inc/tdb_mpfile.h diff --git a/source/libs/tdb/src/devinc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h similarity index 98% rename from source/libs/tdb/src/devinc/tdb_mpool.h rename to source/libs/tdb/src/inc/tdb_mpool.h index 4acf8bef8c..1023b14a50 100644 --- a/source/libs/tdb/src/devinc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -16,7 +16,7 @@ #ifndef _TD_TDB_MPOOL_H_ #define _TD_TDB_MPOOL_H_ -#include "tdbDef.h" +#include "tdb_inc.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/mpool/tdb_mpool.c b/source/libs/tdb/src/mpool/tdb_mpool.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/libs/tdb/src/mpool/tdb_mpool.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file From 131b3f16e8b7ac5e6c6cad606518890cc709f0a3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:38:03 +0000 Subject: [PATCH 005/118] refact --- source/libs/tdb/src/db/{tdbDB.c => tdb_db.c} | 0 source/libs/tdb/src/inc/{tdbDB.h => tdb_db.h} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename source/libs/tdb/src/db/{tdbDB.c => tdb_db.c} (100%) rename source/libs/tdb/src/inc/{tdbDB.h => tdb_db.h} (100%) diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdb_db.c similarity index 100% rename from source/libs/tdb/src/db/tdbDB.c rename to source/libs/tdb/src/db/tdb_db.c diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdb_db.h similarity index 100% rename from source/libs/tdb/src/inc/tdbDB.h rename to source/libs/tdb/src/inc/tdb_db.h From a243752b420119bf5a7cf2542915ff94b1dc9b32 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:44:38 +0000 Subject: [PATCH 006/118] refact --- source/libs/tdb/src/db/tdb_mpfile.c | 14 ++++++++++++++ source/libs/tdb/src/db/tdb_mpool.c | 16 ++++++++++++++++ source/libs/tdb/src/inc/tdb_inc.h | 1 + source/libs/tdb/src/inc/tdb_mpfile.h | 1 - source/libs/tdb/src/inc/tdb_mpool.h | 14 +++++++------- 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 source/libs/tdb/src/db/tdb_mpfile.c create mode 100644 source/libs/tdb/src/db/tdb_mpool.c diff --git a/source/libs/tdb/src/db/tdb_mpfile.c b/source/libs/tdb/src/db/tdb_mpfile.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/db/tdb_mpfile.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c new file mode 100644 index 0000000000..ac4018d348 --- /dev/null +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdb_mpool.h" \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 1e9c3dd8cc..cce6bc3bda 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -17,6 +17,7 @@ #define _TD_TDB_INC_H_ #include "os.h" +#include "tlist.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdb_mpfile.h b/source/libs/tdb/src/inc/tdb_mpfile.h index 74593af2f6..cd4823081a 100644 --- a/source/libs/tdb/src/inc/tdb_mpfile.h +++ b/source/libs/tdb/src/inc/tdb_mpfile.h @@ -16,7 +16,6 @@ #ifndef _TD_TDB_MPFILE_H_ #define _TD_TDB_MPFILE_H_ -#include "tdbDef.h" #include "tdb_mpool.h" #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 1023b14a50..8d113e5edc 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -25,6 +25,12 @@ extern "C" { // Exposed handle typedef struct TDB_MPOOL TDB_MPOOL; +#define TDB_FILE_UID_LEN 20 +typedef struct { + uint8_t fuid[TDB_FILE_UID_LEN]; + pgid_t pgid; +} mp_pgid_t; + // Exposed apis int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbCloseMP(TDB_MPOOL *mp); @@ -32,15 +38,9 @@ int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); // Hidden impls -#define TDB_FILE_UID_LEN 20 - -typedef struct { - uint8_t fuid[TDB_FILE_UID_LEN]; - pgid_t pgid; -} mp_pgid_t; typedef struct MP_PAGE { - SRWLatch rwLatch; + // SRWLatch rwLatch; mp_pgid_t mpgid; uint8_t dirty; int32_t pinRef; From a751c97f2061484876f73cc913405a60675a904b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 09:50:34 +0000 Subject: [PATCH 007/118] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 57 ++++++++++++++++++- source/libs/tdb/src/inc/tdb_inc.h | 8 ++- .../tdb/src/inc/{tdb_mpool.h => tdb_mp.h} | 39 +++++++------ 3 files changed, 85 insertions(+), 19 deletions(-) rename source/libs/tdb/src/inc/{tdb_mpool.h => tdb_mp.h} (68%) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index ac4018d348..6d4f7c4bdc 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -13,4 +13,59 @@ * along with this program. If not, see . */ -#include "tdb_mpool.h" \ No newline at end of file +#include "tdb_mp.h" + +int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { + TDB_MPOOL *mp; + size_t tsize; + MP_PAGE * pagep; + + // check parameters + if (!TDB_IS_PGSIZE_VLD(pgsize)) { + tdbError("invalid page size"); + return -1; + } + + // allocate handle + mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); + if (mp == NULL) { + tdbError("failed to malloc memory pool handle"); + return -1; + } + + // initialize the handle + mp->cachesize = cachesize; + mp->pgsize = pgsize; + mp->npages = cachesize / pgsize; + mp->pages = (MP_PAGE *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); + if (mp->pages == NULL) { + tdbError("failed to malloc memory pool pages"); + free(mp); + return -1; + } + + TD_DLIST_INIT(&(mp->freeList)); + + mp->nbucket = mp->npages; + mp->hashtab = (MP_PAGE_LIST *)calloc(mp->nbucket, sizeof(MP_PAGE_LIST)); + if (mp->hashtab == NULL) { + tdbError("failed to malloc memory pool hash table"); + free(mp->pages); + free(mp); + return -1; + } + + for (int i = 0; i < mp->npages; i++) { + pagep = (MP_PAGE *)MP_PAGE_AT(mp, i); + TD_DLIST_APPEND(&mp->freeList, pagep); + } + + // return + *mpp = mp; + return 0; +} + +int tdbCloseMP(TDB_MPOOL *mp) { + // TODO + return 0; +} diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index cce6bc3bda..2eed9a8dc7 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -35,7 +35,13 @@ typedef int32_t pgsize_t; #define TDB_MIN_PGSIZE 512 #define TDB_MAX_PGSIZE 16384 #define TDB_DEFAULT_PGSIZE 4096 -#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) +#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) + +// fileid +#define TDB_FILE_UID_LEN 20 + +// tdb_log +#define tdbError(var) #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mp.h similarity index 68% rename from source/libs/tdb/src/inc/tdb_mpool.h rename to source/libs/tdb/src/inc/tdb_mp.h index 8d113e5edc..d4e7457b05 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mp.h @@ -25,39 +25,44 @@ extern "C" { // Exposed handle typedef struct TDB_MPOOL TDB_MPOOL; -#define TDB_FILE_UID_LEN 20 typedef struct { uint8_t fuid[TDB_FILE_UID_LEN]; pgid_t pgid; } mp_pgid_t; -// Exposed apis -int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); -int tdbCloseMP(TDB_MPOOL *mp); -int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); -int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); - -// Hidden impls - typedef struct MP_PAGE { // SRWLatch rwLatch; mp_pgid_t mpgid; uint8_t dirty; int32_t pinRef; - // TD_DLIST_NODE(MP_PAGE); // The free list handle + TD_DLIST_NODE(MP_PAGE); char *page[]; } MP_PAGE; +#define MP_PAGE_SIZE(pgsize) (sizeof(MP_PAGE) + (pgsize)) + +typedef TD_DLIST(MP_PAGE) MP_PAGE_LIST; struct TDB_MPOOL { - pthread_mutex_t mutex; - int64_t cachesize; - pgsize_t pgsize; - MP_PAGE * pages; - // TD_DBLIST(MP_PAGE) freeList; - // TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool - // Hash hash; + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + MP_PAGE * pages; + MP_PAGE_LIST freeList; + // Hash + int32_t nbucket; + MP_PAGE_LIST *hashtab; + // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool }; +#define MP_PAGE_AT(mp, idx) ((char *)((mp)->pages) + MP_PAGE_SIZE((mp)->pgsize) * (idx)) + +// Exposed apis ===================================================================================================== + +int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbCloseMP(TDB_MPOOL *mp); +int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); + #ifdef __cplusplus } #endif From f207fbc0afed1e934c8620c0d7659f4a44f6f262 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 01:58:19 +0000 Subject: [PATCH 008/118] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 11 +++++++++++ source/libs/tdb/src/inc/tdb_mp.h | 1 + 2 files changed, 12 insertions(+) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 6d4f7c4bdc..f9a939b5e9 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -69,3 +69,14 @@ int tdbCloseMP(TDB_MPOOL *mp) { // TODO return 0; } + +int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { + // Search the hash + // TODO + return 0; +} + +int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mp.h b/source/libs/tdb/src/inc/tdb_mp.h index d4e7457b05..4a53c7b8d7 100644 --- a/source/libs/tdb/src/inc/tdb_mp.h +++ b/source/libs/tdb/src/inc/tdb_mp.h @@ -34,6 +34,7 @@ typedef struct MP_PAGE { // SRWLatch rwLatch; mp_pgid_t mpgid; uint8_t dirty; + uint8_t fileid[TDB_FILE_UID_LEN]; int32_t pinRef; TD_DLIST_NODE(MP_PAGE); char *page[]; From 0230a529f74d9dfefe12de2229fcb6f00922e523 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:12:50 +0000 Subject: [PATCH 009/118] refact tdb --- source/libs/tdb/src/db/tdb_mpfile.c | 14 ------ source/libs/tdb/src/db/tdb_mpool.c | 2 +- source/libs/tdb/src/inc/tdb_inc.h | 6 +-- source/libs/tdb/src/inc/tdb_mpfile.h | 46 ------------------- .../tdb/src/inc/{tdb_mp.h => tdb_mpool.h} | 22 +++++++-- 5 files changed, 22 insertions(+), 68 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdb_mpfile.c delete mode 100644 source/libs/tdb/src/inc/tdb_mpfile.h rename source/libs/tdb/src/inc/{tdb_mp.h => tdb_mpool.h} (73%) diff --git a/source/libs/tdb/src/db/tdb_mpfile.c b/source/libs/tdb/src/db/tdb_mpfile.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/libs/tdb/src/db/tdb_mpfile.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index f9a939b5e9..d61c0de2c3 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tdb_mp.h" +#include "tdb_mpool.h" int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp; diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 2eed9a8dc7..3e16036722 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -23,9 +23,9 @@ extern "C" { #endif -// pgid_t -typedef int32_t pgid_t; -#define TDB_IVLD_PGID ((pgid_t)-1) +// pgno_t +typedef int32_t pgno_t; +#define TDB_IVLD_PGID ((pgno_t)-1) // framd_id_t typedef int32_t frame_id_t; diff --git a/source/libs/tdb/src/inc/tdb_mpfile.h b/source/libs/tdb/src/inc/tdb_mpfile.h deleted file mode 100644 index cd4823081a..0000000000 --- a/source/libs/tdb/src/inc/tdb_mpfile.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_MPFILE_H_ -#define _TD_TDB_MPFILE_H_ - -#include "tdb_mpool.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Exposed handle -typedef struct TDB_MPFILE TDB_MPFILE; - -// Exposed apis -int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPFClose(TDB_MPFILE *mpf); -int tdbMPFGet(TDB_MPFILE *mpf, pgid_t pgid, void *addr); -int tdbMPFPut(TDB_MPOOL *mpf, pgid_t pgid, void *addr); - -// Hidden structures -struct TDB_MPFILE { - uint8_t fuid[20]; // file unique ID - TDB_MPOOL *mp; // underlying memory pool - char * fname; // file name - int fd; // fd -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_MPFILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mp.h b/source/libs/tdb/src/inc/tdb_mpool.h similarity index 73% rename from source/libs/tdb/src/inc/tdb_mp.h rename to source/libs/tdb/src/inc/tdb_mpool.h index 4a53c7b8d7..d84c2ffaeb 100644 --- a/source/libs/tdb/src/inc/tdb_mp.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -23,11 +23,12 @@ extern "C" { #endif // Exposed handle -typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; typedef struct { uint8_t fuid[TDB_FILE_UID_LEN]; - pgid_t pgid; + pgno_t pgid; } mp_pgid_t; typedef struct MP_PAGE { @@ -55,15 +56,28 @@ struct TDB_MPOOL { // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool }; +struct TDB_MPFILE { + uint8_t fuid[20]; // file unique ID + TDB_MPOOL *mp; // underlying memory pool + char * fname; // file name + int fd; // fd +}; + #define MP_PAGE_AT(mp, idx) ((char *)((mp)->pages) + MP_PAGE_SIZE((mp)->pgsize) * (idx)) -// Exposed apis ===================================================================================================== - +/*=================================================== Exposed apis ==================================================*/ +// TDB_MPOOL int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbCloseMP(TDB_MPOOL *mp); int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +// TDB_MPFILE +int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPFClose(TDB_MPFILE *mpf); +int tdbMPFGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); +int tdbMPFPut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); + #ifdef __cplusplus } #endif From 632bb1b4d2c63e24b6fdf9f61ebb03a05102aa54 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:13:32 +0000 Subject: [PATCH 010/118] refact --- source/libs/tdb/src/db/tdb_mpool.c | 4 ++-- source/libs/tdb/src/inc/tdb_mpool.h | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index d61c0de2c3..c387b6adcc 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -70,13 +70,13 @@ int tdbCloseMP(TDB_MPOOL *mp) { return 0; } -int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { +int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { // Search the hash // TODO return 0; } -int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { +int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d84c2ffaeb..2e9e9be73e 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -29,11 +29,11 @@ typedef struct TDB_MPFILE TDB_MPFILE; typedef struct { uint8_t fuid[TDB_FILE_UID_LEN]; pgno_t pgid; -} mp_pgid_t; +} pgid_t; typedef struct MP_PAGE { // SRWLatch rwLatch; - mp_pgid_t mpgid; + pgid_t mpgid; uint8_t dirty; uint8_t fileid[TDB_FILE_UID_LEN]; int32_t pinRef; @@ -50,7 +50,7 @@ struct TDB_MPOOL { int32_t npages; MP_PAGE * pages; MP_PAGE_LIST freeList; - // Hash + // Hash int32_t nbucket; MP_PAGE_LIST *hashtab; // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool @@ -69,8 +69,8 @@ struct TDB_MPFILE { // TDB_MPOOL int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbCloseMP(TDB_MPOOL *mp); -int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); -int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); +int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); // TDB_MPFILE int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); From 8bbc5603216d0f79668bb503b64771820187b1e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:20:06 +0000 Subject: [PATCH 011/118] refact TDB --- source/libs/tdb/src/db/tdb_mpool.c | 15 ++------------- source/libs/tdb/src/inc/tdb_inc.h | 12 +++++++++--- source/libs/tdb/src/inc/tdb_mpool.h | 19 ++++++------------- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index c387b6adcc..0d9ddd9d46 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,7 +15,7 @@ #include "tdb_mpool.h" -int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp; size_t tsize; MP_PAGE * pagep; @@ -65,18 +65,7 @@ int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { return 0; } -int tdbCloseMP(TDB_MPOOL *mp) { - // TODO - return 0; -} - -int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { - // Search the hash - // TODO - return 0; -} - -int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { +int tdbMPoolClose(TDB_MPOOL *mp) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 3e16036722..17dd5254d6 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -27,6 +27,15 @@ extern "C" { typedef int32_t pgno_t; #define TDB_IVLD_PGID ((pgno_t)-1) +// fileid +#define TDB_FILE_UID_LEN 20 + +// pgid_t +typedef struct { + uint8_t fuid[TDB_FILE_UID_LEN]; + pgno_t pgid; +} pgid_t; + // framd_id_t typedef int32_t frame_id_t; @@ -37,9 +46,6 @@ typedef int32_t pgsize_t; #define TDB_DEFAULT_PGSIZE 4096 #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) -// fileid -#define TDB_FILE_UID_LEN 20 - // tdb_log #define tdbError(var) diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 2e9e9be73e..73596d91dd 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,17 +26,12 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; -typedef struct { - uint8_t fuid[TDB_FILE_UID_LEN]; - pgno_t pgid; -} pgid_t; - typedef struct MP_PAGE { // SRWLatch rwLatch; - pgid_t mpgid; - uint8_t dirty; - uint8_t fileid[TDB_FILE_UID_LEN]; - int32_t pinRef; + pgid_t mpgid; + uint8_t dirty; + uint8_t fileid[TDB_FILE_UID_LEN]; + int32_t pinRef; TD_DLIST_NODE(MP_PAGE); char *page[]; } MP_PAGE; @@ -67,10 +62,8 @@ struct TDB_MPFILE { /*=================================================== Exposed apis ==================================================*/ // TDB_MPOOL -int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); -int tdbCloseMP(TDB_MPOOL *mp); -int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); -int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); From a00243ca404c4c28d406b40070f952dde9ebb04c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:25:54 +0000 Subject: [PATCH 012/118] refact --- source/libs/tdb/src/inc/tdb_inc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 17dd5254d6..82947e2965 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -32,7 +32,7 @@ typedef int32_t pgno_t; // pgid_t typedef struct { - uint8_t fuid[TDB_FILE_UID_LEN]; + uint8_t fileid[TDB_FILE_UID_LEN]; pgno_t pgid; } pgid_t; From ed06d82230e4ffd687b26fb3c2384ff53c954e04 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:58:38 +0000 Subject: [PATCH 013/118] refact --- source/libs/tdb/src/db/tdb_mpool.c | 8 +++---- source/libs/tdb/src/inc/tdb_inc.h | 1 + source/libs/tdb/src/inc/tdb_mpool.h | 33 ++++++++++++++--------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 0d9ddd9d46..4e005b9e03 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -18,7 +18,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp; size_t tsize; - MP_PAGE * pagep; + pg_t * pagep; // check parameters if (!TDB_IS_PGSIZE_VLD(pgsize)) { @@ -37,7 +37,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->cachesize = cachesize; mp->pgsize = pgsize; mp->npages = cachesize / pgsize; - mp->pages = (MP_PAGE *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); + mp->pages = (pg_t *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); if (mp->pages == NULL) { tdbError("failed to malloc memory pool pages"); free(mp); @@ -47,7 +47,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TD_DLIST_INIT(&(mp->freeList)); mp->nbucket = mp->npages; - mp->hashtab = (MP_PAGE_LIST *)calloc(mp->nbucket, sizeof(MP_PAGE_LIST)); + mp->hashtab = (pg_list_t *)calloc(mp->nbucket, sizeof(pg_list_t)); if (mp->hashtab == NULL) { tdbError("failed to malloc memory pool hash table"); free(mp->pages); @@ -56,7 +56,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { } for (int i = 0; i < mp->npages; i++) { - pagep = (MP_PAGE *)MP_PAGE_AT(mp, i); + pagep = (pg_t *)MP_PAGE_AT(mp, i); TD_DLIST_APPEND(&mp->freeList, pagep); } diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 82947e2965..ba9d6a560c 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -18,6 +18,7 @@ #include "os.h" #include "tlist.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 73596d91dd..6b8cb8f844 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,28 +26,27 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; -typedef struct MP_PAGE { - // SRWLatch rwLatch; - pgid_t mpgid; - uint8_t dirty; - uint8_t fileid[TDB_FILE_UID_LEN]; - int32_t pinRef; - TD_DLIST_NODE(MP_PAGE); +typedef struct pg_t { + SRWLatch rwLatch; + pgid_t mpgid; + uint8_t dirty; + int32_t pinRef; + TD_DLIST_NODE(pg_t); char *page[]; -} MP_PAGE; +} pg_t; -#define MP_PAGE_SIZE(pgsize) (sizeof(MP_PAGE) + (pgsize)) +#define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) -typedef TD_DLIST(MP_PAGE) MP_PAGE_LIST; +typedef TD_DLIST(pg_t) pg_list_t; struct TDB_MPOOL { - int64_t cachesize; - pgsize_t pgsize; - int32_t npages; - MP_PAGE * pages; - MP_PAGE_LIST freeList; + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; // Hash - int32_t nbucket; - MP_PAGE_LIST *hashtab; + int32_t nbucket; + pg_list_t *hashtab; // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool }; From bfd427ae842ef06af29c12821f4c85bdf37bf427 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 07:03:06 +0000 Subject: [PATCH 014/118] more TDB --- source/libs/tdb/src/db/tdb_mpool.c | 96 ++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdb_inc.h | 7 ++- source/libs/tdb/src/inc/tdb_mpool.h | 45 +++++++------- 3 files changed, 107 insertions(+), 41 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 4e005b9e03..60829ede94 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,8 +15,10 @@ #include "tdb_mpool.h" +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); + int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { - TDB_MPOOL *mp; + TDB_MPOOL *mp = NULL; size_t tsize; pg_t * pagep; @@ -30,42 +32,102 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); if (mp == NULL) { tdbError("failed to malloc memory pool handle"); - return -1; + goto _err; } // initialize the handle mp->cachesize = cachesize; mp->pgsize = pgsize; mp->npages = cachesize / pgsize; - mp->pages = (pg_t *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); + + TD_DLIST_INIT(&mp->freeList); + + mp->pages = (pg_t **)calloc(mp->npages, sizeof(pg_t *)); if (mp->pages == NULL) { tdbError("failed to malloc memory pool pages"); - free(mp); - return -1; + goto _err; } - TD_DLIST_INIT(&(mp->freeList)); + for (frame_id_t i = 0; i < mp->npages; i++) { + mp->pages[i] = (pg_t *)calloc(1, MP_PAGE_SIZE(pgsize)); + if (mp->pages[i] == NULL) { + goto _err; + } - mp->nbucket = mp->npages; - mp->hashtab = (pg_list_t *)calloc(mp->nbucket, sizeof(pg_list_t)); - if (mp->hashtab == NULL) { + taosInitRWLatch(&mp->pages[i]->rwLatch); + mp->pages[i]->frameid = i; + mp->pages[i]->pgid = TDB_IVLD_PGID; + + // TODO: add the new page to the free list + // TD_DLIST_APPEND(&mp->freeList, mp->pages[i]); + } + +#define PGTAB_FACTOR 1.0 + mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; + mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); + if (mp->pgtab.hashtab == NULL) { tdbError("failed to malloc memory pool hash table"); - free(mp->pages); - free(mp); - return -1; - } - - for (int i = 0; i < mp->npages; i++) { - pagep = (pg_t *)MP_PAGE_AT(mp, i); - TD_DLIST_APPEND(&mp->freeList, pagep); + goto _err; } // return *mpp = mp; return 0; + +_err: + tdbMPoolClose(mp); + *mpp = NULL; + return -1; } int tdbMPoolClose(TDB_MPOOL *mp) { + if (mp) { + tfree(mp->pgtab.hashtab); + if (mp->pages) { + for (int i = 0; i < mp->npages; i++) { + tfree(mp->pages[i]); + } + + free(mp->pages); + } + + free(mp); + } + return 0; +} + +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { + TDB_MPFILE *mpf; + + if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { + return -1; + } + + mpf->fd = -1; + + if ((mpf->fname = strdup(fname)) == NULL) { + goto _err; + } + + if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { + goto _err; + } + + *mpfp = mpf; + return 0; + +_err: + tdbMPoolFileClose(mpf); + *mpfp = NULL; + return -1; +} + +int tdbMPoolFileClose(TDB_MPFILE *mpf) { + // TODO + return 0; +} + +static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index ba9d6a560c..d4c9796e54 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -26,16 +26,17 @@ extern "C" { // pgno_t typedef int32_t pgno_t; -#define TDB_IVLD_PGID ((pgno_t)-1) +#define TDB_IVLD_PGNO ((pgno_t)-1) // fileid -#define TDB_FILE_UID_LEN 20 +#define TDB_FILE_ID_LEN 24 // pgid_t typedef struct { - uint8_t fileid[TDB_FILE_UID_LEN]; + uint8_t fileid[TDB_FILE_ID_LEN]; pgno_t pgid; } pgid_t; +#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; // framd_id_t typedef int32_t frame_id_t; diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 6b8cb8f844..4bd4d12108 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,13 +26,16 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; +typedef TD_DLIST_NODE(pg_t) pg_free_list_node_t, pg_hash_list_node_t; typedef struct pg_t { - SRWLatch rwLatch; - pgid_t mpgid; - uint8_t dirty; - int32_t pinRef; - TD_DLIST_NODE(pg_t); - char *page[]; + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + int32_t pinRef; + pg_free_list_node_t free; + pg_hash_list_node_t hash; + uint8_t data[]; } pg_t; #define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) @@ -42,33 +45,33 @@ struct TDB_MPOOL { int64_t cachesize; pgsize_t pgsize; int32_t npages; - pg_t * pages; + pg_t ** pages; pg_list_t freeList; - // Hash - int32_t nbucket; - pg_list_t *hashtab; - // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool + struct { + int32_t nbucket; + pg_list_t *hashtab; + } pgtab; // page table, hash }; +#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] + struct TDB_MPFILE { - uint8_t fuid[20]; // file unique ID - TDB_MPOOL *mp; // underlying memory pool - char * fname; // file name - int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL *mp; // underlying memory pool + char * fname; // file name + int fd; // fd }; -#define MP_PAGE_AT(mp, idx) ((char *)((mp)->pages) + MP_PAGE_SIZE((mp)->pgsize) * (idx)) - /*=================================================== Exposed apis ==================================================*/ // TDB_MPOOL int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE -int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPFClose(TDB_MPFILE *mpf); -int tdbMPFGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); -int tdbMPFPut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPoolFileClose(TDB_MPFILE *mpf); +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); #ifdef __cplusplus } From 2394e0e0ffb168a1c10cc1a0b11eb711fa65a4e9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 07:28:58 +0000 Subject: [PATCH 015/118] more --- source/libs/tdb/src/db/tdb_mpool.c | 26 ++++++++++++++++++++++++-- source/libs/tdb/src/inc/tdb_mpool.h | 4 ++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 60829ede94..6191d8d4d1 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -104,6 +104,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { } mpf->fd = -1; + mpf->mp = mp; if ((mpf->fname = strdup(fname)) == NULL) { goto _err; @@ -113,6 +114,10 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { goto _err; } + if (tdbGnrtFileID(fname, mpf->fileid) < 0) { + goto _err; + } + *mpfp = mpf; return 0; @@ -123,11 +128,28 @@ _err: } int tdbMPoolFileClose(TDB_MPFILE *mpf) { - // TODO + if (mpf) { + if (mpf->fd > 0) { + close(mpf->fd); + } + tfree(mpf->fname); + free(mpf); + } return 0; } static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { - // TODO + struct stat statbuf; + + if (stat(fname, &statbuf) < 0) { + return -1; + } + + memset(fileid, 0, TDB_FILE_ID_LEN); + + ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; + ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[2] = rand(); + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 4bd4d12108..dd1d802fa5 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -56,10 +56,10 @@ struct TDB_MPOOL { #define MP_PAGE_AT(mp, idx) (mp)->pages[idx] struct TDB_MPFILE { - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL *mp; // underlying memory pool char * fname; // file name int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL *mp; // underlying memory pool }; /*=================================================== Exposed apis ==================================================*/ From 9f3d9db127895cdecd0abf8001c65676fb94d1a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 08:03:33 +0000 Subject: [PATCH 016/118] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 45 ++++++++++++++++++++++++++++- source/libs/tdb/src/inc/tdb_mpool.h | 5 ++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 6191d8d4d1..aea6211493 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,7 +15,8 @@ #include "tdb_mpool.h" -static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -118,6 +119,9 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { goto _err; } + // Register current MPF to MP + tdbMpoolRegFile(mp, mpf); + *mpfp = mpf; return 0; @@ -138,6 +142,41 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf) { return 0; } +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr) { + pg_t * pagep; + TDB_MPOOL *mp; + + mp = mpf->mp; + + // get page in the cache + if (pagep) { + // page is found in the page table + // todo: pin the page and return + *(void **)addr = pagep->data; + return 0; + } else { + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // TD_DLIST_POP(&(mp->freeList), pagep); + } else { + // no page found in the freelist, need to evict + // pagep = tdbMpoolEvict(mp); + if (pagep) { + } else { + // TODO: Cannot find a page to evict + } + } + } + + return 0; +} + +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr) { + // TODO + return 0; +} + static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { struct stat statbuf; @@ -152,4 +191,8 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { ((uint64_t *)fileid)[2] = rand(); return 0; +} + +static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + // TODO } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index dd1d802fa5..e300cd2240 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -51,6 +51,10 @@ struct TDB_MPOOL { int32_t nbucket; pg_list_t *hashtab; } pgtab; // page table, hash + struct { + int32_t nbucket; + TD_DLIST(TDB_MPFILE) hashtab[8]; + } mpftab; }; #define MP_PAGE_AT(mp, idx) (mp)->pages[idx] @@ -60,6 +64,7 @@ struct TDB_MPFILE { int fd; // fd uint8_t fileid[TDB_FILE_ID_LEN]; // file ID TDB_MPOOL *mp; // underlying memory pool + TD_DLIST_NODE(TDB_MPFILE); }; /*=================================================== Exposed apis ==================================================*/ From b6ddd323564fbb0827e51116aad29172e75ba523 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 08:23:38 +0000 Subject: [PATCH 017/118] more tdv --- include/util/tlist.h | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/include/util/tlist.h b/include/util/tlist.h index 06d0abc797..f318c3c58f 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,6 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->(feild).sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -48,12 +49,25 @@ extern "C" { TD_SLIST_NELES(sl) += 1; \ } while (0) +#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, feild) \ + do { \ + TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + #define TD_SLIST_POP(sl) \ do { \ TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) +#define TD_SLIST_POP_WITH_FIELD(sl, feild) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), feild); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + // Double linked list ================ #define TD_DLIST_NODE(TYPE) \ struct { \ @@ -70,6 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->(feild).dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->(feild).dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) @@ -94,6 +110,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_APPEND_WITH_FEILD(dl, dln, feild) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), feild) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_PREPEND(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ @@ -108,6 +138,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, feild) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), feild) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_POP(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ @@ -126,6 +170,26 @@ extern "C" { TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ } while (0) +#define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + } \ + if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) != NULL) { \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild), feild) = \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + } \ + if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) != NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild), feild) = \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + } while (0) + // General double linked list typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; From c50b787fdaf138dbc93a1c64e2735bcd0c2743c0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 08:37:46 +0000 Subject: [PATCH 018/118] more tdb --- include/util/tlist.h | 6 +++--- source/libs/tdb/src/db/tdb_mpool.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/util/tlist.h b/include/util/tlist.h index f318c3c58f..4311356cd8 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,7 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->(feild).sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->feild.sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -84,8 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) -#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->(feild).dl_prev_) -#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->(feild).dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->feild.dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->feild.dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index aea6211493..9f2b8bb15b 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -59,8 +59,8 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->pages[i]->frameid = i; mp->pages[i]->pgid = TDB_IVLD_PGID; - // TODO: add the new page to the free list - // TD_DLIST_APPEND(&mp->freeList, mp->pages[i]); + // add new page to the free list + TD_DLIST_APPEND_WITH_FEILD(&(mp->freeList), mp->pages[i], free); } #define PGTAB_FACTOR 1.0 From a9eeafe319af9d22d64033305c4c68feb56d99fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 09:41:28 +0000 Subject: [PATCH 019/118] more tdb --- include/util/tlist.h | 52 +++++++++--------- source/libs/tdb/src/db/tdb_mpool.c | 85 ++++++++++++++++++++++++++--- source/libs/tdb/src/inc/tdb_mpool.h | 39 +++++++------ 3 files changed, 126 insertions(+), 50 deletions(-) diff --git a/include/util/tlist.h b/include/util/tlist.h index 4311356cd8..134873a993 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,7 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->feild.sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -49,9 +49,9 @@ extern "C" { TD_SLIST_NELES(sl) += 1; \ } while (0) -#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, feild) \ +#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, field) \ do { \ - TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) = TD_SLIST_HEAD(sl); \ + TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) = TD_SLIST_HEAD(sl); \ TD_SLIST_HEAD(sl) = (sln); \ TD_SLIST_NELES(sl) += 1; \ } while (0) @@ -62,9 +62,9 @@ extern "C" { TD_SLIST_NELES(sl) -= 1; \ } while (0) -#define TD_SLIST_POP_WITH_FIELD(sl, feild) \ +#define TD_SLIST_POP_WITH_FIELD(sl, field) \ do { \ - TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), feild); \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), field); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) @@ -84,8 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) -#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->feild.dl_prev_) -#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->feild.dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) @@ -110,15 +110,15 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) -#define TD_DLIST_APPEND_WITH_FEILD(dl, dln, feild) \ +#define TD_DLIST_APPEND_WITH_FIELD(dl, dln, field) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ } else { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_TAIL(dl); \ - TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ - TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), feild) = (dln); \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), field) = (dln); \ TD_DLIST_TAIL(dl) = (dln); \ } \ TD_DLIST_NELES(dl) += 1; \ @@ -138,15 +138,15 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) -#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, feild) \ +#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, field) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ } else { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = NULL; \ - TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = TD_DLIST_HEAD(dl); \ - TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), feild) = (dln); \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), field) = (dln); \ TD_DLIST_HEAD(dl) = (dln); \ } \ TD_DLIST_NELES(dl) += 1; \ @@ -173,21 +173,21 @@ extern "C" { #define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ - TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ } \ if (TD_DLIST_TAIL(dl) == (dln)) { \ - TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ } \ - if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) != NULL) { \ - TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild), feild) = \ - TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ } \ - if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) != NULL) { \ - TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild), feild) = \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ } \ TD_DLIST_NELES(dl) -= 1; \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ } while (0) // General double linked list diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 9f2b8bb15b..d0092a44c9 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,8 +15,10 @@ #include "tdb_mpool.h" -static int tdbGnrtFileID(const char *fname, uint8_t *fileid); -static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -60,7 +62,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->pages[i]->pgid = TDB_IVLD_PGID; // add new page to the free list - TD_DLIST_APPEND_WITH_FEILD(&(mp->freeList), mp->pages[i], free); + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), mp->pages[i], free); } #define PGTAB_FACTOR 1.0 @@ -105,7 +107,6 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { } mpf->fd = -1; - mpf->mp = mp; if ((mpf->fname = strdup(fname)) == NULL) { goto _err; @@ -120,7 +121,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { } // Register current MPF to MP - tdbMpoolRegFile(mp, mpf); + tdbMPoolRegFile(mp, mpf); *mpfp = mpf; return 0; @@ -193,6 +194,76 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { return 0; } -static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - // TODO +#define MPF_GET_BUCKETID(fileid) \ + ({ \ + uint64_t *tmp = fileid; \ + (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ + }) + +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + int bucketid; + mpf_bucket_t *bktp; + + bucketid = MPF_GET_BUCKETID(mpf->fileid); + bktp = mp->mpfht.buckets + bucketid; + + taosWLockLatch(&(bktp->latch)); + + TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); + + taosWUnLockLatch(&(bktp->latch)); + + mpf->mp = mp; +} + +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { + int bucketid; + TDB_MPFILE * mpf = NULL; + mpf_bucket_t *bktp; + + bucketid = MPF_GET_BUCKETID(fileid); + bktp = mp->mpfht.buckets + bucketid; + + taosRLockLatch(&(bktp->latch)); + + mpf = TD_DLIST_HEAD(bktp); + while (mpf) { + if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { + break; + } + + mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); + } + + taosRUnLockLatch(&(bktp->latch)); + + return mpf; +} + +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + TDB_MPFILE * tmpf; + + if (mpf->mp == NULL) return; + + ASSERT(mpf->mp == mp); + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + tmpf = TD_DLIST_HEAD(bktp); + + while (tmpf) { + if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { + TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); + break; + } + + tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); + } + + taosWUnLockLatch(&(bktp->latch)); + + ASSERT(tmpf == mpf); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index e300cd2240..d77e14715d 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,21 +26,25 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; -typedef TD_DLIST_NODE(pg_t) pg_free_list_node_t, pg_hash_list_node_t; +typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; typedef struct pg_t { - SRWLatch rwLatch; - frame_id_t frameid; - pgid_t pgid; - uint8_t dirty; - int32_t pinRef; - pg_free_list_node_t free; - pg_hash_list_node_t hash; - uint8_t data[]; + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + int32_t pinRef; + pg_free_dlist_node_t free; + pg_hash_dlist_node_t hash; + uint8_t data[]; } pg_t; #define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) typedef TD_DLIST(pg_t) pg_list_t; +typedef struct { + SRWLatch latch; + TD_DLIST(TDB_MPFILE); +} mpf_bucket_t; struct TDB_MPOOL { int64_t cachesize; pgsize_t pgsize; @@ -52,19 +56,20 @@ struct TDB_MPOOL { pg_list_t *hashtab; } pgtab; // page table, hash struct { - int32_t nbucket; - TD_DLIST(TDB_MPFILE) hashtab[8]; - } mpftab; +#define MPF_HASH_BUCKETS 16 + mpf_bucket_t buckets[MPF_HASH_BUCKETS]; + } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table }; #define MP_PAGE_AT(mp, idx) (mp)->pages[idx] +typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; struct TDB_MPFILE { - char * fname; // file name - int fd; // fd - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL *mp; // underlying memory pool - TD_DLIST_NODE(TDB_MPFILE); + char * fname; // file name + int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL * mp; // underlying memory pool + td_mpf_dlist_node_t node; }; /*=================================================== Exposed apis ==================================================*/ From ca25ddc730d8bd2f27ef89adf758dbebd0f63c63 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 10:15:07 +0000 Subject: [PATCH 020/118] refact --- source/libs/tdb/src/db/tdb_mpool.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index d0092a44c9..2c536a8e90 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -196,16 +196,14 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { #define MPF_GET_BUCKETID(fileid) \ ({ \ - uint64_t *tmp = fileid; \ + uint64_t *tmp = (uint64_t *)fileid; \ (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ }) static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - int bucketid; mpf_bucket_t *bktp; - bucketid = MPF_GET_BUCKETID(mpf->fileid); - bktp = mp->mpfht.buckets + bucketid; + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); taosWLockLatch(&(bktp->latch)); @@ -217,12 +215,10 @@ static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { } static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { - int bucketid; TDB_MPFILE * mpf = NULL; mpf_bucket_t *bktp; - bucketid = MPF_GET_BUCKETID(fileid); - bktp = mp->mpfht.buckets + bucketid; + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); taosRLockLatch(&(bktp->latch)); From 9c984f9a906fa65767008f7657edb513a5110044 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 02:43:10 +0000 Subject: [PATCH 021/118] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 21 +++++++++++++++++++-- source/libs/tdb/src/inc/tdb_inc.h | 2 +- source/libs/tdb/src/inc/tdb_mpool.h | 4 ++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 2c536a8e90..3450f7c582 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -143,13 +143,30 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf) { return 0; } -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr) { +#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ + }) + +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { pg_t * pagep; TDB_MPOOL *mp; + pg_list_t *pglist; mp = mpf->mp; // get page in the cache + pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); + pagep = TD_DLIST_HEAD(pglist); + while (pagep) { + if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { + break; + } + + pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); + } + if (pagep) { // page is found in the page table // todo: pin the page and return @@ -173,7 +190,7 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr) { return 0; } -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr) { +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { // TODO return 0; } diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index d4c9796e54..885191477c 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -34,7 +34,7 @@ typedef int32_t pgno_t; // pgid_t typedef struct { uint8_t fileid[TDB_FILE_ID_LEN]; - pgno_t pgid; + pgno_t pgno; } pgid_t; #define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d77e14715d..d9ee0bfdcd 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -80,8 +80,8 @@ int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr); #ifdef __cplusplus } From a0d7e2a155afdb286af53b4ee6ea05e6742ef08b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 02:59:49 +0000 Subject: [PATCH 022/118] more TDB --- source/libs/tdb/src/db/tdb_mpool.c | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 3450f7c582..965c3e023b 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -156,7 +156,7 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { mp = mpf->mp; - // get page in the cache + // check if the page already in pool pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); pagep = TD_DLIST_HEAD(pglist); while (pagep) { @@ -168,25 +168,38 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { } if (pagep) { - // page is found in the page table + // page is found // todo: pin the page and return *(void **)addr = pagep->data; return 0; + } + + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // has free page + TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); + // todo: load the page from file and pin the page } else { - // page not found - pagep = TD_DLIST_HEAD(&mp->freeList); + // no free page available + // pagep = tdbMpoolEvict(mp); if (pagep) { - // TD_DLIST_POP(&(mp->freeList), pagep); } else { - // no page found in the freelist, need to evict - // pagep = tdbMpoolEvict(mp); - if (pagep) { - } else { - // TODO: Cannot find a page to evict - } + // TODO: Cannot find a page to evict } } + if (pagep == NULL) { + // no available container page + return -1; + } + + // load page from the disk if a container page is available + // TODO: load the page from the disk + + // add current page to page table + TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); + return 0; } From 7ef98d1af96df22985c67049ccf4629b3bc01aad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:20:18 +0000 Subject: [PATCH 023/118] more TDB --- source/libs/tdb/src/db/tdb_mpool.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 965c3e023b..abcce801bb 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -19,6 +19,7 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid); static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -196,6 +197,14 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { // load page from the disk if a container page is available // TODO: load the page from the disk + if (tdbMPoolFileReadPage(mpf, pgno, pagep->data) < 0) { + return -1; + } + + memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); + pagep->pgid.pgno = pgno; + pagep->dirty = 0; + pagep->pinRef = 1; // add current page to page table TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); @@ -292,4 +301,21 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { taosWUnLockLatch(&(bktp->latch)); ASSERT(tmpf == mpf); +} + +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + size_t rsize; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + // TODO: use loop to read all data + rsize = pread(mpf->fd, p, pgsize, offset); + // TODO: error handle + + return 0; } \ No newline at end of file From 8dfad06979b7b4de024b641548908788c3d20539 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:24:25 +0000 Subject: [PATCH 024/118] more --- source/libs/tdb/src/db/tdb_mpool.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index abcce801bb..5f3108a487 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -20,6 +20,7 @@ static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -317,5 +318,23 @@ static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { rsize = pread(mpf->fd, p, pgsize, offset); // TODO: error handle + return 0; +} + +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + lseek(mpf->fd, offset, SEEK_SET); + // TODO: handle error + + write(mpf->fd, p, pgsize); + // TODO: handle error + return 0; } \ No newline at end of file From 4824ec7d7ac802636a8d407ffa6969b9171c389c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:29:19 +0000 Subject: [PATCH 025/118] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 14 ++++++++++++-- source/libs/tdb/src/inc/tdb_mpool.h | 6 ++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 5f3108a487..e368801900 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -151,7 +151,17 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf) { (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ }) -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { pg_t * pagep; TDB_MPOOL *mp; pg_list_t *pglist; @@ -213,7 +223,7 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { return 0; } -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { +int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { // TODO return 0; } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d9ee0bfdcd..c92038dfc8 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -80,8 +80,10 @@ int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr); #ifdef __cplusplus } From 04487db683266e9ce967c9de61cc147cfe932799 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 13:38:08 +0800 Subject: [PATCH 026/118] [td-11818]Fix memory leak. --- source/client/src/clientEnv.c | 4 + source/client/src/clientImpl.c | 2 + source/client/src/clientMsgHandler.c | 20 +++-- source/client/test/clientTests.cpp | 112 ++++++++++++------------- source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/libs/parser/src/dCDAstProcess.c | 5 +- source/libs/parser/src/parser.c | 26 +++--- 7 files changed, 93 insertions(+), 78 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f747ccf3b6..d2696fb355 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -196,6 +196,10 @@ static void doDestroyRequest(void* p) { doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryDag(pRequest->body.pDag); + if (pRequest->body.showInfo.pArray != NULL) { + taosArrayDestroy(pRequest->body.showInfo.pArray); + } + deregisterRequest(pRequest); tfree(pRequest); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 159a92b0ab..42f9378a4e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -666,6 +666,8 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { if (pMsg->contLen > 0) { buf.pData = calloc(1, pMsg->contLen); + printf("create------------>%p\n", buf.pData); + if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 02e36043dc..ec088eb073 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -145,19 +145,23 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { } pSchema = pMetaMsg->pSchema; - TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); - for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { - tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); - pFields[i].type = pSchema[i].type; - pFields[i].bytes = pSchema[i].bytes; - } + tfree(pRequest->body.resInfo.pRspMsg); pRequest->body.resInfo.pRspMsg = pMsg->pData; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - pResInfo->fields = pFields; - pResInfo->numOfCols = pMetaMsg->numOfColumns; + if (pResInfo->fields == NULL) { + TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); + pFields[i].type = pSchema[i].type; + pFields[i].bytes = pSchema[i].bytes; + } + pResInfo->fields = pFields; + } + + pResInfo->numOfCols = pMetaMsg->numOfColumns; pRequest->body.showInfo.execId = pShow->showId; // todo diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 736a47273f..891a6cfbf4 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -451,39 +451,39 @@ TEST(testCase, driverInit_Test) { // // taos_close(pConn); //} -// -//TEST(testCase, show_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show tables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// } -// -// pRes = taos_query(pConn, "show abc1.tables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// int32_t count = 0; -// char str[512] = {0}; -// -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%d: %s\n", ++count, str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, show_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "show abc1.tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + int32_t count = 0; + char str[512] = {0}; + + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%d: %s\n", ++count, str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, drop_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -525,29 +525,29 @@ TEST(testCase, driverInit_Test) { // taosHashCleanup(phash); //} // -TEST(testCase, create_topic_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_topic_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == nullptr); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// char* sql = "select * from tu"; +// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +// taos_free_result(pRes); +// taos_close(pConn); +//} //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 1d4cbf37ce..ac66e7d88b 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -127,7 +127,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); - SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); pTopic->logicalPlan = calloc(len + 1, sizeof(char)); diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 5852678880..994875c0a3 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -62,9 +62,8 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out pEpSet->port[i] = info->epAddr[i].port; } - *outputLen = sizeof(SVShowTablesReq); - *output = pShowReq; - + *outputLen = sizeof(SVShowTablesReq); + *output = pShowReq; *pExtension = array; } else { if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 58e368aa0d..4271aae451 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -36,25 +36,29 @@ bool qIsDdlQuery(const SQueryNode* pQueryNode) { } int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { + int32_t code = TSDB_CODE_SUCCESS; + SSqlInfo info = doGenerateAST(pCxt->pSql); if (!info.valid) { strncpy(pCxt->pMsg, info.msg, pCxt->msgLen); - terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - return terrno; + code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + goto _end; } if (!isDqlSqlStatement(&info)) { if (info.type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pModifStmtInfo == NULL) { - return terrno; + code = terrno; + goto _end; } *pQuery = (SQueryNode*)pModifStmtInfo; } else { SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pDcl == NULL) { - return terrno; + code = terrno; + goto _end; } *pQuery = (SQueryNode*)pDcl; @@ -63,21 +67,22 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { } else { SQueryStmtInfo* pQueryInfo = createQueryInfo(); if (pQueryInfo == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code. - return terrno; + code = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code. + goto _end; } - int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); + code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } else { - terrno = code; - return code; + goto _end; } } + _end: destroySqlInfo(&info); - return TSDB_CODE_SUCCESS; + terrno = code; + return code; } int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQueryNode) { @@ -247,5 +252,6 @@ void qDestroyQuery(SQueryNode* pQueryNode) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); } + tfree(pQueryNode); } From 3af9da8f51751d1f74bf274bc6f6ff0b25f37eec Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 21 Jan 2022 13:49:12 +0800 Subject: [PATCH 027/118] feature/qnode --- source/libs/qworker/inc/qworkerInt.h | 2 +- source/libs/qworker/src/qworker.c | 123 ++++++++++++++------------- source/libs/qworker/src/qworkerMsg.c | 7 +- 3 files changed, 67 insertions(+), 65 deletions(-) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 5f9b33f7e3..9a33268472 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -83,7 +83,7 @@ typedef struct SQWMsg { } SQWMsg; typedef struct SQWPhaseInput { - int8_t status; + int8_t taskStatus; int8_t taskType; int32_t code; qTaskInfo_t taskHandle; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 566356e255..cedb3fa926 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -240,14 +240,14 @@ void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { } -int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWTaskCtx **ctx) { +int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - QW_LOCK(rwType, &mgmt->ctxLock); - *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); + //QW_LOCK(rwType, &mgmt->ctxLock); + *ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { - QW_UNLOCK(rwType, &mgmt->ctxLock); + //QW_UNLOCK(rwType, &mgmt->ctxLock); QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); } @@ -268,19 +268,19 @@ int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tI return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, int32_t status, SQWTaskCtx **ctx) { +int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool acquire, int32_t status, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx nctx = {0}; - QW_LOCK(QW_WRITE, &mgmt->ctxLock); + //QW_LOCK(QW_WRITE, &mgmt->ctxLock); int32_t code = taosHashPut(mgmt->ctxHash, id, sizeof(id), &nctx, sizeof(SQWTaskCtx)); if (0 != code) { - QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); if (HASH_NODE_EXIST(code)) { - if (rwType && ctx) { + if (acquire && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); @@ -293,9 +293,9 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } - QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); - if (rwType && ctx) { + if (acquire && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); @@ -305,22 +305,23 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ } int32_t qwAddTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { - QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, NULL)); + QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, NULL)); } -int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), rwType, 0, ctx); +int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { + return qwAddTaskCtxImpl(QW_FPARAMS(), true, 0, ctx); } int32_t qwAddGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, ctx); + return qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, ctx); } -void qwReleaseTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt) { - QW_UNLOCK(rwType, &mgmt->ctxLock); +void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { + //QW_UNLOCK(rwType, &mgmt->ctxLock); + taosHashRelease(mgmt->ctxHash, ctx); } void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { @@ -355,7 +356,7 @@ void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { // Note: NEED CTX HASH LOCKED BEFORE ENTRANCE -int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx octx; @@ -367,6 +368,13 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t octx = *ctx; + atomic_store_ptr(&ctx->taskHandle, NULL); + atomic_store_ptr(&ctx->sinkHandle, NULL); + + if (rwType) { + QW_UNLOCK(rwType, &ctx->lock); + } + if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); @@ -439,7 +447,7 @@ _return: QW_RET(code); } -int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType) { +int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType, bool execOnce) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; @@ -463,6 +471,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH if (TASK_TYPE_TEMP == taskType) { qwFreeTaskHandle(QW_FPARAMS(), taskHandle); } + break; } @@ -475,7 +484,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); - if (!qcontinue) { + if (execOnce || (!qcontinue)) { break; } } @@ -573,7 +582,6 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ int8_t status = 0; SQWTaskCtx *ctx = NULL; bool locked = false; - bool ctxAcquired = false; void *readyConnection = NULL; void *dropConnection = NULL; void *cancelConnection = NULL; @@ -582,7 +590,9 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ switch (phase) { case QW_PHASE_PRE_QUERY: { - QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); atomic_store_32(&ctx->phase, phase); atomic_store_8(&ctx->taskType, input->taskType); @@ -597,7 +607,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; @@ -621,7 +631,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ break; } case QW_PHASE_POST_QUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -641,8 +651,8 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { output->needStop = true; - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; dropConnection = ctx->dropConnection; @@ -665,13 +675,12 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } if (!output->needStop) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->status)); + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->taskStatus)); } break; } case QW_PHASE_PRE_FETCH: { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - ctxAcquired = true; QW_LOCK(QW_WRITE, &ctx->lock); @@ -686,6 +695,13 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropped, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + } + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); output->needStop = true; @@ -721,7 +737,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ break; } case QW_PHASE_POST_FETCH: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -743,7 +759,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ output->needStop = true; QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; dropConnection = ctx->dropConnection; @@ -772,7 +788,6 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } case QW_PHASE_PRE_CQUERY: { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - ctxAcquired = true; QW_LOCK(QW_WRITE, &ctx->lock); @@ -808,7 +823,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ break; } case QW_PHASE_POST_CQUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -829,8 +844,8 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ QW_TASK_WLOG("start to drop task, phase:%d", phase); output->needStop = true; - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; dropConnection = ctx->dropConnection; @@ -871,8 +886,8 @@ _return: QW_UNLOCK(QW_WRITE, &ctx->lock); } - if (ctxAcquired && ctx) { - qwReleaseTaskCtx(QW_READ, mgmt); + if (ctx) { + qwReleaseTaskCtx(mgmt, ctx); } if (readyConnection) { @@ -896,7 +911,7 @@ _return: } -int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg, int8_t taskType) { +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { int32_t code = 0; bool queryRsped = false; bool needStop = false; @@ -944,7 +959,7 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t queryRsped = true; if (pTaskInfo && sinkHandle) { - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &pTaskInfo, sinkHandle, taskType)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &pTaskInfo, sinkHandle, taskType, true)); } _return: @@ -954,14 +969,8 @@ _return: } if (!queryRsped) { - code = qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); - if (TSDB_CODE_SUCCESS == code) { - QW_TASK_DLOG("query msg rsped, code:%d", rspCode); - } - - if (TSDB_CODE_SUCCESS == rspCode && code) { - rspCode = code; - } + qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); + QW_TASK_DLOG("query msg rsped, code:%x", rspCode); } if (needStop) { @@ -971,12 +980,7 @@ _return: input.code = rspCode; input.taskHandle = pTaskInfo; input.sinkHandle = sinkHandle; - - if (TSDB_CODE_SUCCESS != rspCode) { - input.status = JOB_TASK_STATUS_FAILED; - } else { - input.status = JOB_TASK_STATUS_PARTIAL_SUCCEED; - } + input.taskStatus = rspCode ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_PARTIAL_SUCCEED; QW_ERR_RET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); @@ -989,8 +993,8 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int8_t phase = 0; bool needRsp = false; int32_t rspCode = 0; - - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -1020,6 +1024,7 @@ _return: if (ctx) { QW_UNLOCK(QW_WRITE, &ctx->lock); + qwReleaseTaskCtx(mgmt, ctx); } if (needRsp) { @@ -1059,7 +1064,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx->taskHandle, sinkHandle, ctx->taskType)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx->taskHandle, sinkHandle, ctx->taskType, QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH))); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; @@ -1191,7 +1196,7 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t SQWTaskCtx *ctx = NULL; bool locked = false; - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_WRITE, &ctx)); + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -1210,7 +1215,7 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t ctx->dropConnection = qwMsg->connection; } else if (ctx->phase > 0) { QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); locked = false; needRsp = true; @@ -1226,12 +1231,8 @@ _return: QW_SET_RSP_CODE(ctx, code); } - if (locked) { - QW_UNLOCK(QW_WRITE, &ctx->lock); - } - if (ctx) { - qwReleaseTaskCtx(QW_WRITE, mgmt); + qwReleaseTaskCtx(mgmt, ctx); } if (TSDB_CODE_SUCCESS != code || needRsp) { diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index feb8fd645e..0b1200745b 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -273,7 +273,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen); + QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -307,7 +307,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - QW_ELOG("invalid cquery msg, contLen:%d", pMsg->contLen); + QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -337,7 +337,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - qError("invalid task status msg"); + QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -398,6 +398,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { + QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } From 8fcbd64f807d36093d86a597d7b1443b160ee949 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:49:41 +0000 Subject: [PATCH 028/118] more --- cmake/cmake.options | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index faa45256fb..8e57165565 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -47,7 +47,7 @@ option( option( BUILD_WITH_UV "If build with libuv" - OFF + ON ) option( @@ -59,7 +59,7 @@ option( option( BUILD_WITH_TRAFT "If build with traft" - OFF + ON ) option( From ef53305b4721cf6b25bc3fde1128b1c0f09b618b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:50:10 +0000 Subject: [PATCH 029/118] more --- cmake/cmake.options | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 8e57165565..4883b6ee8e 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -59,7 +59,7 @@ option( option( BUILD_WITH_TRAFT "If build with traft" - ON + OFF ) option( From 72ed99e8674faa05e8af7177992b44adc884ad2c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 06:04:38 +0000 Subject: [PATCH 030/118] more --- cmake/traft_CMakeLists.txt.in | 2 +- contrib/test/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/traft_CMakeLists.txt.in b/cmake/traft_CMakeLists.txt.in index 9b571b3666..1dd1323603 100644 --- a/cmake/traft_CMakeLists.txt.in +++ b/cmake/traft_CMakeLists.txt.in @@ -7,7 +7,7 @@ ExternalProject_Add(traft BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft" #BUILD_IN_SOURCE TRUE # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ - CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure BUILD_COMMAND "$(MAKE)" INSTALL_COMMAND "" TEST_COMMAND "" diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 0c71113056..eacaeb9524 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -20,7 +20,7 @@ if(${BUILD_WITH_CRAFT}) endif(${BUILD_WITH_CRAFT}) if(${BUILD_WITH_TRAFT}) - add_subdirectory(traft) + # add_subdirectory(traft) endif(${BUILD_WITH_TRAFT}) add_subdirectory(tdev) From 4c4be3c575bdccd492afb3abc1603488786a377c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 14:21:13 +0800 Subject: [PATCH 031/118] put createStreamExecTaskInfo into right place --- include/libs/executor/executor.h | 4 +++- source/dnode/vnode/inc/tq.h | 15 --------------- source/dnode/vnode/src/tq/tq.c | 24 +++++++++--------------- source/libs/executor/src/executor.c | 10 ++++++---- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index a98bb8f51a..36cc0f2665 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -32,7 +32,9 @@ struct SSubplan; * @param pStreamBlockReadHandle * @return */ -qTaskInfo_t createStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle); +qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle); + +void qStreamExecTaskSetInput(qTaskInfo_t qHandle, void* input); /** * Create the exec task object according to task json diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 8c4effa221..f49542b5ec 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -82,27 +82,12 @@ typedef struct STqSubscribeReq { int64_t topic[]; } STqSubscribeReq; -typedef struct STqSubscribeRsp { - STqMsgHead head; - int64_t vgId; - char ep[TSDB_EP_LEN]; // TSDB_EP_LEN -} STqSubscribeRsp; - typedef struct STqHeartbeatReq { } STqHeartbeatReq; typedef struct STqHeartbeatRsp { } STqHeartbeatRsp; -typedef struct STqTopicVhandle { - int64_t topicId; - // executor for filter - void* filterExec; - // callback for mnode - // trigger when vnode list associated topic change - void* (*mCallback)(void*, void*); -} STqTopicVhandle; - #define TQ_BUFFER_SIZE 8 typedef struct STqExec { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 89d4af48fd..52c541dcfd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -633,12 +633,16 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { // read until find TDMT_VND_SUBMIT } SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; + void* task = pHandle->buffer.output[pos].task; - /*SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg;*/ + qStreamExecTaskSetInput(task, pCont); + SSDataBlock* pDataBlock; + uint64_t ts; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + } // TODO: launch query and get output data - void* outputData; - pHandle->buffer.output[pos].dst = outputData; + pHandle->buffer.output[pos].dst = pDataBlock; if (pHandle->buffer.firstOffset == -1 || pReq->offset < pHandle->buffer.firstOffset) { pHandle->buffer.firstOffset = pReq->offset; @@ -674,22 +678,12 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { strcpy(pTopic->sql, pReq->sql); strcpy(pTopic->logicalPlan, pReq->logicalPlan); strcpy(pTopic->physicalPlan, pReq->physicalPlan); - SArray *pArray; - //TODO: deserialize to SQueryDag - SQueryDag *pDag; - // convert to task - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - // TODO: handle error - } - STaskInfo *pInfo = taosArrayGet(pArray, 0); - SArray* pTasks; - schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); + pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); pTopic->buffer.output[i].status = 0; - pTopic->buffer.output[i].task = createStreamExecTaskInfo(pMsg, NULL); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, NULL); } pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); // write mq meta diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a933402296..49bf42f383 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -13,10 +13,12 @@ * along with this program. If not, see . */ -#include "planner.h" #include "executor.h" +#include "planner.h" -qTaskInfo_t createStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle) { +void qStreamExecTaskSetInput(qTaskInfo_t qHandle, void* input) {} + +qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* pStreamBlockReadHandle) { if (pMsg == NULL || pStreamBlockReadHandle == NULL) { return NULL; } @@ -27,8 +29,8 @@ qTaskInfo_t createStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadH pMsg->taskId = be64toh(pMsg->taskId); pMsg->contentLen = ntohl(pMsg->contentLen); - struct SSubplan *plan = NULL; - int32_t code = qStringToSubplan(pMsg->msg, &plan); + struct SSubplan* plan = NULL; + int32_t code = qStringToSubplan(pMsg->msg, &plan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; From 30ed49842a4af59e2b45d7abad47acbaffee6fef Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 20 Jan 2022 22:47:40 -0800 Subject: [PATCH 032/118] fix bug in ut --- source/libs/tfs/src/tfs.c | 8 +- source/libs/tfs/src/tfsTier.c | 2 +- source/libs/tfs/test/tfsTest.cpp | 449 +++++++++++++++++++++++++++++-- tests/script/sh/deploy.sh | 22 +- 4 files changed, 449 insertions(+), 32 deletions(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 623293f82b..9002879b10 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -118,7 +118,11 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { pDiskId->id = -1; if (pDiskId->level >= pTfs->nlevel) { - pDiskId->level--; + pDiskId->level = pTfs->nlevel - 1; + } + + if (pDiskId->level < 0) { + pDiskId->level = 0; } while (pDiskId->level >= 0) { @@ -289,7 +293,7 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { STfsDisk *pDisk = pTier->disks[id]; snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); - if (taosRenameFile(oaname, naname) != 0) { + if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { terrno = TAOS_SYSTEM_ERROR(errno); fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); return -1; diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 270fff9ff3..e4390d13d1 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -69,7 +69,7 @@ STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { pTier->disks[id] = pDisk; pTier->ndisk++; - fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); + fDebug("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); return pTier->disks[id]; } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index abc00c5cd9..178d115c59 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -246,29 +246,13 @@ TEST_F(TfsTest, 04_File) { snprintf(fulldir, 128, "%s%s%s", root, TD_DIRSEP, "t3"); EXPECT_STREQ(dir, fulldir); - EXPECT_NE(tfsCopyFile(&f1, &f2), 0); + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); EXPECT_EQ(taosDirExist(af2), 0); tfsRemoveFile(&f2); EXPECT_NE(taosDirExist(af2), 0); - EXPECT_NE(tfsCopyFile(&f1, &f2), 0); - - { - STfsDir *pDir = tfsOpendir(pTfs, ""); - - const STfsFile *pf1 = tfsReaddir(pDir); - EXPECT_STREQ(pf1->rname, "t3"); - EXPECT_EQ(pf1->did.id, 0); - EXPECT_EQ(pf1->did.level, 0); - EXPECT_EQ(pf1->pTfs, pTfs); - - const STfsFile *pf2 = tfsReaddir(pDir); - EXPECT_EQ(pf2, nullptr); - - tfsClosedir(pDir); - } { STfsDir *pDir = tfsOpendir(pTfs, "t3"); @@ -280,7 +264,26 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(pf1->pTfs, pTfs); const STfsFile *pf2 = tfsReaddir(pDir); - EXPECT_NE(pf2, nullptr); + EXPECT_EQ(pf2, nullptr); + + tfsClosedir(pDir); + } + + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_EQ(pf1->did.id, 0); + EXPECT_EQ(pf1->did.level, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_EQ(pf2->did.id, 0); + EXPECT_EQ(pf2->did.level, 0); + EXPECT_EQ(pf2->pTfs, pTfs); const STfsFile *pf3 = tfsReaddir(pDir); EXPECT_EQ(pf3, nullptr); @@ -289,5 +292,415 @@ TEST_F(TfsTest, 04_File) { } } + tfsClose(pTfs); +} + +TEST_F(TfsTest, 05_MultiDisk) { + int32_t code = 0; + + const char *root00 = "/tmp/tfsTest00"; + const char *root01 = "/tmp/tfsTest01"; + const char *root10 = "/tmp/tfsTest10"; + const char *root11 = "/tmp/tfsTest11"; + const char *root12 = "/tmp/tfsTest12"; + const char *root20 = "/tmp/tfsTest20"; + const char *root21 = "/tmp/tfsTest21"; + const char *root22 = "/tmp/tfsTest22"; + const char *root23 = "/tmp/tfsTest23"; + + SDiskCfg dCfg[9] = {0}; + tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN); + dCfg[0].level = 0; + dCfg[0].primary = 0; + tstrncpy(dCfg[1].dir, root00, TSDB_FILENAME_LEN); + dCfg[1].level = 0; + dCfg[1].primary = 0; + tstrncpy(dCfg[2].dir, root20, TSDB_FILENAME_LEN); + dCfg[2].level = 2; + dCfg[2].primary = 0; + tstrncpy(dCfg[3].dir, root21, TSDB_FILENAME_LEN); + dCfg[3].level = 2; + dCfg[3].primary = 0; + tstrncpy(dCfg[4].dir, root22, TSDB_FILENAME_LEN); + dCfg[4].level = 2; + dCfg[4].primary = 0; + tstrncpy(dCfg[5].dir, root23, TSDB_FILENAME_LEN); + dCfg[5].level = 2; + dCfg[5].primary = 0; + tstrncpy(dCfg[6].dir, root10, TSDB_FILENAME_LEN); + dCfg[6].level = 1; + dCfg[6].primary = 0; + tstrncpy(dCfg[7].dir, root11, TSDB_FILENAME_LEN); + dCfg[7].level = 1; + dCfg[7].primary = 0; + tstrncpy(dCfg[8].dir, root12, TSDB_FILENAME_LEN); + dCfg[8].level = 1; + dCfg[8].primary = 0; + + taosRemoveDir(root00); + taosRemoveDir(root01); + taosRemoveDir(root10); + taosRemoveDir(root11); + taosRemoveDir(root12); + taosRemoveDir(root20); + taosRemoveDir(root21); + taosRemoveDir(root22); + taosRemoveDir(root23); + taosMkDir(root00); + taosMkDir(root01); + taosMkDir(root10); + taosMkDir(root11); + taosMkDir(root12); + taosMkDir(root20); + taosMkDir(root21); + taosMkDir(root22); + taosMkDir(root23); + + STfs *pTfs = tfsOpen(dCfg, 9); + ASSERT_EQ(pTfs, nullptr); + + dCfg[0].primary = 1; + dCfg[1].primary = 1; + pTfs = tfsOpen(dCfg, 9); + ASSERT_EQ(pTfs, nullptr); + + dCfg[0].primary = 0; + dCfg[1].primary = 1; + pTfs = tfsOpen(dCfg, 9); + ASSERT_NE(pTfs, nullptr); + + tfsUpdateSize(pTfs); + SDiskSize size = tfsGetSize(pTfs); + + EXPECT_GT(size.avail, 0); + EXPECT_GT(size.used, 0); + EXPECT_GT(size.total, size.avail); + EXPECT_GT(size.total, size.used); + + //------------- AllocDisk -----------------// + { + const char *path = NULL; + SDiskID did; + did.id = 0; + did.level = 0; + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root00); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root01); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root00); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root01); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root00); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root01); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root10); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root11); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 2); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root12); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root10); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root20); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root21); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 2); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root22); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 3); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root23); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root20); + + code = tfsAllocDisk(pTfs, 3, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root21); + + code = tfsAllocDisk(pTfs, 4, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 2); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root22); + + const char *primary = tfsGetPrimaryPath(pTfs); + EXPECT_STREQ(primary, root00); + } + + //------------- Dir -----------------// + { + char p1[] = "p1"; + char ap00[128] = {0}; + snprintf(ap00, 128, "%s%s%s", root00, TD_DIRSEP, p1); + char ap01[128] = {0}; + snprintf(ap01, 128, "%s%s%s", root01, TD_DIRSEP, p1); + char ap10[128] = {0}; + snprintf(ap10, 128, "%s%s%s", root10, TD_DIRSEP, p1); + char ap11[128] = {0}; + snprintf(ap11, 128, "%s%s%s", root11, TD_DIRSEP, p1); + char ap12[128] = {0}; + snprintf(ap12, 128, "%s%s%s", root12, TD_DIRSEP, p1); + char ap20[128] = {0}; + snprintf(ap20, 128, "%s%s%s", root20, TD_DIRSEP, p1); + char ap21[128] = {0}; + snprintf(ap21, 128, "%s%s%s", root21, TD_DIRSEP, p1); + char ap22[128] = {0}; + snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1); + char ap23[128] = {0}; + snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1); + EXPECT_NE(taosDirExist(ap00), 0); + EXPECT_NE(taosDirExist(ap01), 0); + EXPECT_NE(taosDirExist(ap10), 0); + EXPECT_NE(taosDirExist(ap11), 0); + EXPECT_NE(taosDirExist(ap12), 0); + EXPECT_NE(taosDirExist(ap20), 0); + EXPECT_NE(taosDirExist(ap21), 0); + EXPECT_NE(taosDirExist(ap22), 0); + EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_EQ(tfsMkdir(pTfs, p1), 0); + EXPECT_EQ(taosDirExist(ap00), 0); + EXPECT_EQ(taosDirExist(ap01), 0); + EXPECT_EQ(taosDirExist(ap10), 0); + EXPECT_EQ(taosDirExist(ap11), 0); + EXPECT_EQ(taosDirExist(ap12), 0); + EXPECT_EQ(taosDirExist(ap20), 0); + EXPECT_EQ(taosDirExist(ap21), 0); + EXPECT_EQ(taosDirExist(ap22), 0); + EXPECT_EQ(taosDirExist(ap23), 0); + EXPECT_EQ(tfsRmdir(pTfs, p1), 0); + EXPECT_NE(taosDirExist(ap00), 0); + EXPECT_NE(taosDirExist(ap01), 0); + EXPECT_NE(taosDirExist(ap10), 0); + EXPECT_NE(taosDirExist(ap11), 0); + EXPECT_NE(taosDirExist(ap12), 0); + EXPECT_NE(taosDirExist(ap20), 0); + EXPECT_NE(taosDirExist(ap21), 0); + EXPECT_NE(taosDirExist(ap22), 0); + EXPECT_NE(taosDirExist(ap23), 0); + + char p2[] = "p2"; + char _ap21[128] = {0}; + snprintf(_ap21, 128, "%s%s%s", root21, TD_DIRSEP, p2); + SDiskID did = {0}; + did.level = 2; + did.id = 1; + EXPECT_NE(taosDirExist(_ap21), 0); + EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); + EXPECT_EQ(taosDirExist(_ap21), 0); + + char p3[] = "p3/p2/p1/p0"; + char _ap12[128] = {0}; + snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3); + did.level = 1; + did.id = 2; + EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(tfsMkdir(pTfs, p3), 0); + EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); + EXPECT_EQ(taosDirExist(_ap12), 0); + EXPECT_EQ(tfsRmdir(pTfs, p3), 0); + EXPECT_NE(taosDirExist(_ap12), 0); + + char p45[] = "p5"; + char p44[] = "p4"; + char p4[] = "p4/p2/p1/p0"; + char _ap22[128] = {0}; + snprintf(_ap22, 128, "%s%s%s", root22, TD_DIRSEP, p4); + did.level = 2; + did.id = 2; + + EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); + EXPECT_EQ(taosDirExist(_ap22), 0); + EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRmdir(pTfs, p4), 0); + EXPECT_NE(taosDirExist(_ap22), 0); + } + + //------------- File -----------------// + { + STfsFile file0; + STfsFile file1; + STfsFile file2; + STfsFile file3; + STfsFile file4; + SDiskID did0 = {0}; + SDiskID did1 = {0}; + SDiskID did2 = {0}; + SDiskID did3 = {0}; + SDiskID did4 = {0}; + did3.id = 1; + did4.level = 1; + tfsInitFile(pTfs, &file0, did0, "fname"); + tfsInitFile(pTfs, &file1, did1, "fname"); + tfsInitFile(pTfs, &file2, did2, "fnamex"); + tfsInitFile(pTfs, &file3, did3, "fname"); + tfsInitFile(pTfs, &file4, did4, "fname"); + + EXPECT_TRUE(tfsIsSameFile(&file0, &file1)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file2)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file3)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file4)); + + { + char n1[] = "t3/t1.json"; + char n2[] = "t3/t2.json"; + STfsFile f1 = {0}; + STfsFile f2 = {0}; + SDiskID did; + did1.level = 1; + did1.id = 2; + did2.level = 2; + did2.id = 3; + + tfsInitFile(pTfs, &f1, did1, n1); + tfsInitFile(pTfs, &f2, did2, n2); + + EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); + + FILE *fp = fopen(f1.aname, "w"); + ASSERT_NE(fp, nullptr); + fwrite("12345678", 1, 5, fp); + fclose(fp); + + char base[128] = {0}; + tfsBasename(&f1, base); + char dir[128] = {0}; + tfsDirname(&f1, dir); + + EXPECT_STREQ(base, "t1.json"); + + char fulldir[128]; + snprintf(fulldir, 128, "%s%s%s", root12, TD_DIRSEP, "t3"); + EXPECT_STREQ(dir, fulldir); + + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); + + char af2[128] = {0}; + snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2); + EXPECT_EQ(taosDirExist(af2), 0); + tfsRemoveFile(&f2); + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_EQ(pf1->did.level, 1); + EXPECT_EQ(pf1->did.id, 2); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_EQ(pf2, nullptr); + + tfsClosedir(pDir); + } + + EXPECT_NE(taosDirExist(af2), 0); + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_GT(pf1->did.level, 0); + EXPECT_GT(pf1->did.id, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_GT(pf1->did.level, 0); + EXPECT_GT(pf1->did.id, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf3 = tfsReaddir(pDir); + EXPECT_EQ(pf3, nullptr); + + tfsClosedir(pDir); + } + } + } + tfsClose(pTfs); } \ No newline at end of file diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index e00363b28f..4471f4e273 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -124,18 +124,18 @@ echo "supportVnodes 128" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 143" >> $TAOS_CFG -echo "dDebugFlag 143" >> $TAOS_CFG -echo "vDebugFlag 143" >> $TAOS_CFG -echo "tsdbDebugFlag 143" >> $TAOS_CFG -echo "cDebugFlag 143" >> $TAOS_CFG -echo "jnidebugFlag 143" >> $TAOS_CFG -echo "qdebugFlag 143" >> $TAOS_CFG -echo "rpcDebugFlag 143" >> $TAOS_CFG +echo "mDebugFlag 0" >> $TAOS_CFG +echo "dDebugFlag 0" >> $TAOS_CFG +echo "vDebugFlag 0" >> $TAOS_CFG +echo "tsdbDebugFlag 0" >> $TAOS_CFG +echo "cDebugFlag 0" >> $TAOS_CFG +echo "jnidebugFlag 0" >> $TAOS_CFG +echo "qdebugFlag 0" >> $TAOS_CFG +echo "rpcDebugFlag 0" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 143" >> $TAOS_CFG -echo "sdebugFlag 143" >> $TAOS_CFG -echo "wdebugFlag 143" >> $TAOS_CFG +echo "udebugFlag 0" >> $TAOS_CFG +echo "sdebugFlag 0" >> $TAOS_CFG +echo "wdebugFlag 0" >> $TAOS_CFG echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG From f5d84caecbe2260c9ac63478506330c9d0baa22d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 20 Jan 2022 22:48:20 -0800 Subject: [PATCH 033/118] minor changes --- tests/script/sh/deploy.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 4471f4e273..e00363b28f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -124,18 +124,18 @@ echo "supportVnodes 128" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG -echo "mDebugFlag 0" >> $TAOS_CFG -echo "dDebugFlag 0" >> $TAOS_CFG -echo "vDebugFlag 0" >> $TAOS_CFG -echo "tsdbDebugFlag 0" >> $TAOS_CFG -echo "cDebugFlag 0" >> $TAOS_CFG -echo "jnidebugFlag 0" >> $TAOS_CFG -echo "qdebugFlag 0" >> $TAOS_CFG -echo "rpcDebugFlag 0" >> $TAOS_CFG +echo "mDebugFlag 143" >> $TAOS_CFG +echo "dDebugFlag 143" >> $TAOS_CFG +echo "vDebugFlag 143" >> $TAOS_CFG +echo "tsdbDebugFlag 143" >> $TAOS_CFG +echo "cDebugFlag 143" >> $TAOS_CFG +echo "jnidebugFlag 143" >> $TAOS_CFG +echo "qdebugFlag 143" >> $TAOS_CFG +echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 0" >> $TAOS_CFG -echo "sdebugFlag 0" >> $TAOS_CFG -echo "wdebugFlag 0" >> $TAOS_CFG +echo "udebugFlag 143" >> $TAOS_CFG +echo "sdebugFlag 143" >> $TAOS_CFG +echo "wdebugFlag 143" >> $TAOS_CFG echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG From e1d9fa73b0e2f4c00390cafd8a88047ac877e945 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 15:16:17 +0800 Subject: [PATCH 034/118] [td-11818] Support create topic. --- include/libs/executor/executor.h | 40 ++++++++++----------- source/libs/executor/src/executor.c | 46 ++++++++++++++++++++++--- source/libs/executor/src/executorimpl.c | 20 +++++++++-- 3 files changed, 79 insertions(+), 27 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 36cc0f2665..457245e9a3 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -29,23 +29,23 @@ struct SSubplan; /** * Create the exec task for streaming mode * @param pMsg - * @param pStreamBlockReadHandle + * @param streamReadHandle * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle); +qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* streamReadHandle); -void qStreamExecTaskSetInput(qTaskInfo_t qHandle, void* input); +int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); /** * Create the exec task object according to task json - * @param tsdb + * @param readHandle * @param vgId * @param pTaskInfoMsg * @param pTaskInfo * @param qId * @return */ -int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +int32_t qCreateExecTask(void* readHandle, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, @@ -62,63 +62,63 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); * this function will be blocked to wait for the query execution completed or paused, * in which case enough results have been produced already. * - * @param qinfo + * @param tinfo * @return */ -int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext); +int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext); /** * * Retrieve the actual results to fill the response message payload. * Note that this function must be executed after qRetrieveQueryResultInfo is invoked. * - * @param qinfo qinfo object + * @param tinfo tinfo object * @param pRsp response message * @param contLen payload length * @return */ -//int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) - * @param qinfo + * @param tinfo * @return */ -void* qGetResultRetrieveMsg(qTaskInfo_t qinfo); +void* qGetResultRetrieveMsg(qTaskInfo_t tinfo); /** * kill the ongoing query and free the query handle and corresponding resources automatically - * @param qinfo qhandle + * @param tinfo qhandle * @return */ -int32_t qKillTask(qTaskInfo_t qinfo); +int32_t qKillTask(qTaskInfo_t tinfo); /** * kill the ongoing query asynchronously - * @param qinfo qhandle + * @param tinfo qhandle * @return */ -int32_t qAsyncKillTask(qTaskInfo_t qinfo); +int32_t qAsyncKillTask(qTaskInfo_t tinfo); /** * return whether query is completed or not - * @param qinfo + * @param tinfo * @return */ -int32_t qIsTaskCompleted(qTaskInfo_t qinfo); +int32_t qIsTaskCompleted(qTaskInfo_t tinfo); /** * destroy query info structure * @param qHandle */ -void qDestroyTask(qTaskInfo_t qHandle); +void qDestroyTask(qTaskInfo_t tinfo); /** * Get the queried table uid * @param qHandle * @return */ -int64_t qGetQueriedTableUid(qTaskInfo_t qHandle); +int64_t qGetQueriedTableUid(qTaskInfo_t tinfo); /** * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. @@ -145,7 +145,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param type operation type: ADD|DROP * @return */ -int32_t qUpdateQueriedTableIdList(qTaskInfo_t qinfo, int64_t uid, int32_t type); +int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type); //================================================================================================ // query handle management diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 49bf42f383..ccc1620264 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,12 +14,50 @@ */ #include "executor.h" +#include "tq.h" +#include "executorimpl.h" #include "planner.h" -void qStreamExecTaskSetInput(qTaskInfo_t qHandle, void* input) {} +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input) { + ASSERT(pOperator != NULL); + if (pOperator->operatorType != OP_StreamScan) { + if (pOperator->numOfDownstream > 0) { -qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* pStreamBlockReadHandle) { - if (pMsg == NULL || pStreamBlockReadHandle == NULL) { + if (pOperator->numOfDownstream > 1) { // not handle this in join query + return TSDB_CODE_QRY_APP_ERROR; + } + + return doSetStreamBlock(pOperator->pDownstream[0], input); + } + } else { + SStreamBlockScanInfo* pInfo = pOperator->info; + tqReadHandleSetMsg(pInfo->readerHandle, input, 0); + return TSDB_CODE_SUCCESS; + } +} + +int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { + if (tinfo == NULL) { + return TSDB_CODE_QRY_APP_ERROR; + } + + if (input == NULL) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to set the stream block data, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); + } else { + qDebug("set the stream block successfully, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); + } + + return code; +} + +qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle) { + if (pMsg == NULL || streamReadHandle == NULL) { return NULL; } @@ -37,7 +75,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* pStreamBlockRead } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(pStreamBlockReadHandle, 0, plan, &pTaskInfo, NULL); + code = qCreateExecTask(streamReadHandle, 0, plan, &pTaskInfo, NULL); if (code != TSDB_CODE_SUCCESS) { // TODO: destroy SSubplan & pTaskInfo terrno = code; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0ed480ed15..3b01c319e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5407,7 +5407,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } -SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5417,10 +5417,21 @@ SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32 return NULL; } - pInfo->readerHandle = pStreamBlockHandle; + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo); + SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + + taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId); + } + + // TODO set the extract column id to streamHandle + // pColList + + pInfo->readerHandle = streamReadHandle; pOperator->name = "StreamBlockScanOperator"; - pOperator->operatorType = OP_StreamBlockScan; + pOperator->operatorType = OP_StreamScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -7704,6 +7715,9 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); + } else if (pPhyNode->info.type == OP_StreamScan) { + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pTaskInfo); } } From ecdd6784f6f20e24015709ed0b5e23ec2eefd273 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 15:36:24 +0800 Subject: [PATCH 035/118] [td-11818] refactor. --- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/executor/src/executor.c | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 52c541dcfd..59c9828693 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -635,7 +635,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; void* task = pHandle->buffer.output[pos].task; - qStreamExecTaskSetInput(task, pCont); + qSetStreamInput(task, pCont); SSDataBlock* pDataBlock; uint64_t ts; if (qExecTask(task, &pDataBlock, &ts) < 0) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ccc1620264..ee96ac4a71 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -18,17 +18,20 @@ #include "executorimpl.h" #include "planner.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t reqId) { ASSERT(pOperator != NULL); if (pOperator->operatorType != OP_StreamScan) { - if (pOperator->numOfDownstream > 0) { - - if (pOperator->numOfDownstream > 1) { // not handle this in join query - return TSDB_CODE_QRY_APP_ERROR; - } - - return doSetStreamBlock(pOperator->pDownstream[0], input); + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator to set the input data block, reqId:0x%" PRIx64, reqId); + return TSDB_CODE_QRY_APP_ERROR; } + + if (pOperator->numOfDownstream > 1) { // not handle this in join query + qError("join not supported for stream block scan, reqId:0x%" PRIx64, reqId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return doSetStreamBlock(pOperator->pDownstream[0], input, reqId); } else { SStreamBlockScanInfo* pInfo = pOperator->info; tqReadHandleSetMsg(pInfo->readerHandle, input, 0); @@ -46,7 +49,8 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input); + + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("failed to set the stream block data, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); } else { From ecdb23c9dbe7f69ae5b5c4c4957821799de3af75 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 20 Jan 2022 23:39:54 -0800 Subject: [PATCH 036/118] Remove warnings and adjust dependencies --- include/util/compare.h | 68 ++++++++++++++++++------------------- source/util/src/compare.c | 16 ++++----- source/util/src/tskiplist.c | 12 +++++-- tests/test/c/create_table.c | 4 +-- 4 files changed, 53 insertions(+), 47 deletions(-) diff --git a/include/util/compare.h b/include/util/compare.h index 461552ca7b..70a8134b35 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_COMPARE_H -#define _TD_UTIL_COMPARE_H +#ifndef _TD_UTIL_COMPARE_H_ +#define _TD_UTIL_COMPARE_H_ #ifdef __cplusplus extern "C" { @@ -35,67 +35,65 @@ extern "C" { #define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) #define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) -#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' } +#define PATTERN_COMPARE_INFO_INITIALIZER \ + { '%', '_' } typedef struct SPatternCompareInfo { char matchAll; // symbol for match all wildcard, default: '%' char matchOne; // symbol for match one wildcard, default: '_' } SPatternCompareInfo; -int patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); -int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); - -int32_t taosArrayCompareString(const void* a, const void* b); +int32_t taosArrayCompareString(const void *a, const void *b); int32_t setCompareBytes1(const void *pLeft, const void *pRight); - int32_t setCompareBytes2(const void *pLeft, const void *pRight); - int32_t setCompareBytes4(const void *pLeft, const void *pRight); int32_t setCompareBytes8(const void *pLeft, const void *pRight); +int32_t compareInt8Val(const void *pLeft, const void *pRight); +int32_t compareInt16Val(const void *pLeft, const void *pRight); int32_t compareInt32Val(const void *pLeft, const void *pRight); int32_t compareInt64Val(const void *pLeft, const void *pRight); -int32_t compareInt16Val(const void *pLeft, const void *pRight); - -int32_t compareInt8Val(const void *pLeft, const void *pRight); - +int32_t compareUint8Val(const void *pLeft, const void *pRight); +int32_t compareUint16Val(const void *pLeft, const void *pRight); int32_t compareUint32Val(const void *pLeft, const void *pRight); int32_t compareUint64Val(const void *pLeft, const void *pRight); -int32_t compareUint16Val(const void *pLeft, const void *pRight); - -int32_t compareUint8Val(const void* pLeft, const void* pRight); - int32_t compareFloatVal(const void *pLeft, const void *pRight); - int32_t compareDoubleVal(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); - int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); -int32_t compareStrRegexComp(const void* pLeft, const void* pRight); -int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight); -int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight); -int32_t compareFindItemInSet(const void *pLeft, const void* pRight); + +int32_t compareStrRegexComp(const void *pLeft, const void *pRight); +int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); +int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); + +int32_t compareFindItemInSet(const void *pLeft, const void *pRight); + int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); -int32_t compareInt16ValDesc(const void* pLeft, const void* pRight); -int32_t compareInt32ValDesc(const void* pLeft, const void* pRight); -int32_t compareInt64ValDesc(const void* pLeft, const void* pRight); -int32_t compareFloatValDesc(const void* pLeft, const void* pRight); -int32_t compareDoubleValDesc(const void* pLeft, const void* pRight); -int32_t compareUint8ValDesc(const void* pLeft, const void* pRight); -int32_t compareUint16ValDesc(const void* pLeft, const void* pRight); -int32_t compareUint32ValDesc(const void* pLeft, const void* pRight); -int32_t compareUint64ValDesc(const void* pLeft, const void* pRight); -int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight); -int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight); +int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); +int32_t compareInt32ValDesc(const void *pLeft, const void *pRight); +int32_t compareInt64ValDesc(const void *pLeft, const void *pRight); + +int32_t compareFloatValDesc(const void *pLeft, const void *pRight); +int32_t compareDoubleValDesc(const void *pLeft, const void *pRight); + +int32_t compareUint8ValDesc(const void *pLeft, const void *pRight); +int32_t compareUint16ValDesc(const void *pLeft, const void *pRight); +int32_t compareUint32ValDesc(const void *pLeft, const void *pRight); +int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); + +int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); +int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_COMPARE_H*/ +#endif /*_TD_UTIL_COMPARE_H_*/ diff --git a/source/util/src/compare.c b/source/util/src/compare.c index f2d320fde0..a1c1625c34 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -16,13 +16,13 @@ #define _GNU_SOURCE #define _XOPEN_SOURCE #define _DEFAULT_SOURCE - #include "os.h" -#include "types.h" + #include "compare.h" -#include "ulog.h" -#include "thash.h" #include "regex.h" +#include "thash.h" +#include "types.h" +#include "ulog.h" int32_t setCompareBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -228,7 +228,7 @@ int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { * '_': Matches one character * */ -int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { +int32_t patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { char c, c1; int32_t i = 0; @@ -289,7 +289,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { +int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { wchar_t c, c1; wchar_t matchOne = L'_'; // "_" wchar_t matchAll = L'%'; // "%" @@ -360,11 +360,11 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { memcpy(str, varDataVal(pLeft), sz); str[sz] = 0; - int errCode = 0; + int32_t errCode = 0; regex_t regex; char msgbuf[256] = {0}; - int cflags = REG_EXTENDED; + int32_t cflags = REG_EXTENDED; if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 00263d7bce..328d3da5a4 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -13,11 +13,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tskiplist.h" + #include "os.h" + #include "compare.h" -#include "ulog.h" +#include "tskiplist.h" #include "tutil.h" +#include "ulog.h" static int initForwardBackwardPtr(SSkipList *pSkipList); static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); @@ -52,11 +54,17 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ pSkipList->flags = flags; pSkipList->keyFn = fn; pSkipList->seed = rand(); + +#if 0 + // the function getkeycomparfunc is defined in common if (comparFn == NULL) { pSkipList->comparFn = getKeyComparFunc(keyType, TSDB_ORDER_ASC); } else { pSkipList->comparFn = comparFn; } +#else + pSkipList->comparFn = comparFn; +#endif if (initForwardBackwardPtr(pSkipList) < 0) { tSkipListDestroy(pSkipList); diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index b8d8123380..74dd3f4535 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -182,7 +182,7 @@ void *threadFunc(void *param) { exit(1); } - pError("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", + pPrint("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); @@ -190,7 +190,7 @@ void *threadFunc(void *param) { pInfo->tableBeginIndex += startOffset; pInfo->tableEndIndex += startOffset; - pError("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", + pPrint("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); From 8a4444e5e1510424434f4c7bbd7622a84e9e4d09 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 15:42:12 +0800 Subject: [PATCH 037/118] [td-11818] Remove printf. --- source/client/src/clientImpl.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 42f9378a4e..159a92b0ab 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -666,8 +666,6 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { if (pMsg->contLen > 0) { buf.pData = calloc(1, pMsg->contLen); - printf("create------------>%p\n", buf.pData); - if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; From 84ca79c5d68566c3f3ba850c057cf3acd6e2265c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 15:52:25 +0800 Subject: [PATCH 038/118] [td-11818] Add log for create table. --- source/dnode/vnode/src/vnd/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index d1b529f7fb..b109608193 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -84,7 +84,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } - vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); + vInfo("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); free(pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { free(pCreateTbReq->stbCfg.pSchema); From 35749cb374279728b2ddc27d4ef8f3ddb322773b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 16:12:02 +0800 Subject: [PATCH 039/118] fix dependency for vnode --- include/common/tmsg.h | 19 +++++---- include/util/tcoding.h | 16 ++++--- source/dnode/vnode/inc/tq.h | 35 ++++------------ source/dnode/vnode/inc/vnode.h | 32 +++++++++++++- source/dnode/vnode/src/inc/tqInt.h | 1 + source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/tq/tq.c | 56 +++++++++++++++---------- source/dnode/vnode/src/vnd/vnodeMain.c | 2 +- source/libs/executor/src/executor.c | 8 ++-- source/libs/executor/src/executorimpl.c | 6 ++- source/libs/scheduler/src/scheduler.c | 8 ++-- 11 files changed, 110 insertions(+), 74 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b468456cb7..62f55609ce 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1529,17 +1529,22 @@ typedef struct SMqSetCVgReq { } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { - int32_t tlen = sizeof(SSubQueryMsg) + pMsg->contentLen; - if (buf == NULL) return tlen; - memcpy(*buf, pMsg, tlen); - *buf = POINTER_SHIFT(*buf, tlen); + int32_t tlen = 0; + tlen += taosEncodeFixedU64(buf, pMsg->sId); + tlen += taosEncodeFixedU64(buf, pMsg->queryId); + tlen += taosEncodeFixedU64(buf, pMsg->taskId); + tlen += taosEncodeFixedU32(buf, pMsg->contentLen); + tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { - int32_t tlen = sizeof(SSubQueryMsg) + ((SSubQueryMsg*)buf)->contentLen; - memcpy(pMsg, buf, tlen); - return POINTER_SHIFT(buf, tlen); + buf = taosDecodeFixedU64(buf, &pMsg->sId); + buf = taosDecodeFixedU64(buf, &pMsg->queryId); + buf = taosDecodeFixedU64(buf, &pMsg->taskId); + buf = taosDecodeFixedU32(buf, &pMsg->contentLen); + buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); + return buf; } static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 8198787048..4bb0bde4bb 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -372,9 +372,10 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { } // ---- binary -static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valueLen) { +static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { int tlen = 0; + tlen += taosEncodeVariantI32(buf, valueLen); if (buf != NULL) { memcpy(*buf, value, valueLen); *buf = POINTER_SHIFT(*buf, valueLen); @@ -384,14 +385,19 @@ static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valu return tlen; } -static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int valueLen) { - uint64_t size = 0; +static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valueLen) { *value = malloc((size_t)valueLen); if (*value == NULL) return NULL; - memcpy(*value, buf, (size_t)size); + memcpy(*value, buf, (size_t)valueLen); - return POINTER_SHIFT(buf, size); + return POINTER_SHIFT(buf, valueLen); +} + +static FORCE_INLINE void *taosDecodeBinaryTo(void *buf, void *value, int32_t valueLen) { + + memcpy(value, buf, (size_t)valueLen); + return POINTER_SHIFT(buf, valueLen); } #endif diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index f49542b5ec..9cd6c3d365 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -17,11 +17,12 @@ #define _TD_TQ_H_ #include "common.h" +#include "executor.h" +#include "vnode.h" #include "mallocator.h" #include "meta.h" #include "os.h" #include "scheduler.h" -#include "executor.h" #include "taoserror.h" #include "tlist.h" #include "tmsg.h" @@ -148,10 +149,10 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - qTaskInfo_t task; + int8_t status; + int64_t offset; + void* dst; + qTaskInfo_t task; } STqTaskItem; // new version @@ -184,10 +185,6 @@ typedef struct STqQueryMsg { struct STqQueryMsg* next; } STqQueryMsg; -typedef struct STqCfg { - // TODO -} STqCfg; - typedef struct STqMemRef { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; @@ -284,6 +281,7 @@ typedef struct STQ { STqMemRef tqMemRef; STqMetaStore* tqMeta; SWal* pWal; + SMeta* pMeta; } STQ; typedef struct STqMgmt { @@ -298,7 +296,7 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type @@ -320,23 +318,6 @@ int tqSendLaunchQuery(STqMsgItem*, int64_t offset); int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq); -typedef struct STqReadHandle { - int64_t ver; - SSubmitMsg* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - SMeta* pMeta; - SArray* pColumnIdList; -} STqReadHandle; - -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList); -void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle* pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); -// return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 499972f476..bb0ee8dfc4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -22,7 +22,6 @@ #include "meta.h" #include "tarray.h" #include "tfs.h" -#include "tq.h" #include "tsdb.h" #include "wal.h" @@ -35,6 +34,12 @@ typedef struct SVnode SVnode; typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef struct STqCfg { + // TODO + int32_t reserved; +} STqCfg; + + typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; @@ -61,6 +66,16 @@ typedef struct { PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeOpt; +typedef struct STqReadHandle { + int64_t ver; + SSubmitMsg* pMsg; + SSubmitBlk* pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; + SMeta* pMeta; + SArray* pColumnIdList; +} STqReadHandle; + /* ------------------------ SVnode ------------------------ */ /** * @brief Initialize the vnode module @@ -180,6 +195,21 @@ int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); +/* ------------------------- TQ QUERY -------------------------- */ + +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta); + +static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColumnIdList) { + pReadHandle->pColumnIdList = pColumnIdList; +} + +void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); +bool tqNextDataBlock(STqReadHandle* pHandle); +int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); +// return SArray +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); + + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index b4e1f57384..2b4200fce5 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -17,6 +17,7 @@ #define _TD_TQ_INT_H_ #include "tq.h" +#include "meta.h" #include "tlog.h" #include "trpc.h" #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index be32ed6829..1fa65b2a73 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -24,6 +24,7 @@ #include "tlockfree.h" #include "tmacro.h" #include "wal.h" +#include "tq.h" #include "vnode.h" diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 52c541dcfd..eca02c867c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -50,7 +50,7 @@ void tqCleanUp() { taosTmrCleanUp(tqMgmt.timer); } -STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -58,6 +58,8 @@ STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory } pTq->path = strdup(path); pTq->tqConfig = tqConfig; + pTq->pWal = pWal; + pTq->pMeta = pMeta; #if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); @@ -610,48 +612,52 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SMqCVConsumeReq* pReq = pMsg->pCont; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; - int64_t offset = pReq->offset; + int64_t reqOffset = pReq->offset; + int64_t fetchOffset = reqOffset; int64_t blockingTime = pReq->blockingTime; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0 ; i < sz; i++) { - STqTopicHandle *pHandle = taosArrayGet(pConsumer->topics, i); + STqTopicHandle *pTopic = taosArrayGet(pConsumer->topics, i); - int8_t pos = offset % TQ_BUFFER_SIZE; - int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); + int8_t pos = fetchOffset % TQ_BUFFER_SIZE; + int8_t old = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1); if (old == 1) { // do nothing continue; } - if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { - // TODO + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + return -1; } - SWalHead* pHead = pHandle->pReadhandle->pHead; - while (pHead->head.msgType != TDMT_VND_SUBMIT) { + SWalHead* pHead = pTopic->pReadhandle->pHead; + while (1) { // read until find TDMT_VND_SUBMIT + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + return -1; + } } SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; - void* task = pHandle->buffer.output[pos].task; + void* task = pTopic->buffer.output[pos].task; - qStreamExecTaskSetInput(task, pCont); + qSetStreamInput(task, pCont); SSDataBlock* pDataBlock; uint64_t ts; if (qExecTask(task, &pDataBlock, &ts) < 0) { } // TODO: launch query and get output data - pHandle->buffer.output[pos].dst = pDataBlock; - if (pHandle->buffer.firstOffset == -1 - || pReq->offset < pHandle->buffer.firstOffset) { - pHandle->buffer.firstOffset = pReq->offset; + pTopic->buffer.output[pos].dst = pDataBlock; + if (pTopic->buffer.firstOffset == -1 + || pReq->offset < pTopic->buffer.firstOffset) { + pTopic->buffer.firstOffset = pReq->offset; } - if (pHandle->buffer.lastOffset == -1 - || pReq->offset > pHandle->buffer.lastOffset) { - pHandle->buffer.lastOffset = pReq->offset; + if (pTopic->buffer.lastOffset == -1 + || pReq->offset > pTopic->buffer.lastOffset) { + pTopic->buffer.lastOffset = pReq->offset; } - atomic_store_8(&pHandle->buffer.output[pos].status, 1); + atomic_store_8(&pTopic->buffer.output[pos].status, 1); // put output into rsp } @@ -681,16 +687,20 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; + pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); + if (pTopic->pReadhandle == NULL) { + + } for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, NULL); + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, pReadHandle); } - pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); // write mq meta return 0; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) { +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; @@ -698,7 +708,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) { pReadHandle->pMeta = pMeta; pReadHandle->pMsg = NULL; pReadHandle->ver = -1; - pReadHandle->pColumnIdList = pColumnIdList; + pReadHandle->pColumnIdList = NULL; return NULL; } diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index c4bbd93eda..6bbf3b959d 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -127,7 +127,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, pVnode->pWal, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ccc1620264..55f6b75fa2 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -62,10 +62,10 @@ qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle } // print those info into log - pMsg->sId = be64toh(pMsg->sId); - pMsg->queryId = be64toh(pMsg->queryId); - pMsg->taskId = be64toh(pMsg->taskId); - pMsg->contentLen = ntohl(pMsg->contentLen); + pMsg->sId = pMsg->sId; + pMsg->queryId = pMsg->queryId; + pMsg->taskId = pMsg->taskId; + pMsg->contentLen = pMsg->contentLen; struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(pMsg->msg, &plan); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3b01c319e4..d9b3f5f4a9 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -27,6 +27,7 @@ #include "thash.h" #include "ttypes.h" #include "query.h" +#include "vnode.h" #include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) @@ -5425,8 +5426,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId); } - // TODO set the extract column id to streamHandle - // pColList + // set the extract column id to streamHandle + tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); pInfo->readerHandle = streamReadHandle; @@ -5438,6 +5439,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp pOperator->numOfOutput = numOfOutput; pOperator->exec = doStreamBlockScan; pOperator->pTaskInfo = pTaskInfo; + return pOperator; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ddfa73f0a5..46450d78ea 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1485,11 +1485,11 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { pMsg->header.vgId = htonl(tInfo.addr.nodeId); - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(plan->id.queryId); - pMsg->taskId = htobe64(schGenUUID()); + pMsg->sId = schMgmt.sId; + pMsg->queryId = plan->id.queryId; + pMsg->taskId = schGenUUID(); pMsg->taskType = TASK_TYPE_PERSISTENT; - pMsg->contentLen = htonl(msgLen); + pMsg->contentLen = msgLen; memcpy(pMsg->msg, msg, msgLen); tInfo.msg = pMsg; From 99984adb45d784b086f6362c7104062ccbb98a88 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 16:13:11 +0800 Subject: [PATCH 040/118] fix dependency for vnode --- include/util/tcoding.h | 1 - source/libs/executor/src/executor.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 4bb0bde4bb..c105ce1ab9 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -375,7 +375,6 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { int tlen = 0; - tlen += taosEncodeVariantI32(buf, valueLen); if (buf != NULL) { memcpy(*buf, value, valueLen); *buf = POINTER_SHIFT(*buf, valueLen); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 55f6b75fa2..b6683e6043 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -62,10 +62,12 @@ qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle } // print those info into log +#if 0 pMsg->sId = pMsg->sId; pMsg->queryId = pMsg->queryId; pMsg->taskId = pMsg->taskId; pMsg->contentLen = pMsg->contentLen; +#endif struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(pMsg->msg, &plan); From 4a60cd962dd21f29a18e317d788d9ae0979954fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 16:15:35 +0800 Subject: [PATCH 041/118] [td-11818] change create table log. --- source/dnode/vnode/src/vnd/vnodeWrite.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index b109608193..9b1f80f329 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,8 +83,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error + vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } - vInfo("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); free(pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { free(pCreateTbReq->stbCfg.pSchema); @@ -95,6 +95,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { free(pCreateTbReq->ntbCfg.pSchema); } } + + vInfo("vgId:%d process create %"PRIzu" tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); break; From 8cdd7023c64cac2129bb07f4f9a8c5e0b59ffbaa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 21 Jan 2022 00:32:58 -0800 Subject: [PATCH 042/118] remove warnings --- source/common/src/ttime.c | 8 ++++---- source/libs/sync/src/raft_handle_vote_message.c | 2 +- source/libs/sync/src/sync.c | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 5c6b9ff6f3..f9755a43b9 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -100,12 +100,12 @@ int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t t } else if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, time, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])(timestr, time, timePrec); + return (*parseLocaltimeFp[day_light])((char*)timestr, time, timePrec); } } -bool checkTzPresent(const char *str, int32_t len) { - char *seg = forwardToTimeStringEnd(str); +bool checkTzPresent(const char* str, int32_t len) { + char* seg = forwardToTimeStringEnd((char*)str); int32_t seg_len = len - (int32_t)(seg - str); char *c = &seg[seg_len - 1]; @@ -267,7 +267,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch #endif int64_t fraction = 0; - str = forwardToTimeStringEnd(timestr); + str = forwardToTimeStringEnd((char*)timestr); if ((str[0] == 'Z' || str[0] == 'z') && str[1] == '\0') { /* utc time, no millisecond, return directly*/ diff --git a/source/libs/sync/src/raft_handle_vote_message.c b/source/libs/sync/src/raft_handle_vote_message.c index 0219e39df9..4d940732dc 100644 --- a/source/libs/sync/src/raft_handle_vote_message.c +++ b/source/libs/sync/src/raft_handle_vote_message.c @@ -37,7 +37,7 @@ int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { if (pRespMsg == NULL) { return 0; } - syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d"\ + syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d" "[logterm: %" PRId64 ", index: %" PRId64 "] at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, lastTerm, lastIndex, pRaft->voteFor, grant ? "grant" : "reject", diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index e49b1d7983..321b03d2ee 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -23,8 +23,8 @@ SSyncManager* gSyncManager = NULL; #define SYNC_ACTIVITY_TIMER 5 #define SYNC_SERVER_WORKER 2 -static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg); static int syncInitRpcClient(SSyncManager* syncManager); @@ -119,7 +119,7 @@ SSyncNode* syncStart(const SSyncInfo* pInfo) { return NULL; } - pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pInfo->vgId, gSyncManager->syncTimerManager); + pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)((int64_t)pInfo->vgId), gSyncManager->syncTimerManager); // start raft pNode->raft.pNode = pNode; @@ -176,12 +176,12 @@ int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode) { } // process rpc rsp message from other sync server -static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet) { +static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } // process rpc message from other sync server -static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { +static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } @@ -195,7 +195,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn } assert(pSyncCfg->selfIndex < pSyncCfg->replica && pSyncCfg->selfIndex >= 0); const SNodeInfo* pNode = &(pSyncCfg->nodeInfo[pSyncCfg->replica]); - char buffer[20] = {'\0'}; + char buffer[156] = {'\0'}; snprintf(buffer, sizeof(buffer), "%s:%d", &(pNode->nodeFqdn[0]), pNode->nodePort); size_t len = strlen(buffer); void** ppRpcServer = taosHashGet(gSyncManager->rpcServerTable, buffer, len); @@ -287,7 +287,7 @@ static void *syncWorkerMain(void *argv) { } static void syncNodeTick(void *param, void *tmrId) { - SyncGroupId vgId = (SyncGroupId)param; + SyncGroupId vgId = (SyncGroupId)((int64_t)param); SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &vgId, sizeof(SyncGroupId*)); if (ppNode == NULL) { return; @@ -298,5 +298,5 @@ static void syncNodeTick(void *param, void *tmrId) { syncRaftTick(&pNode->raft); pthread_mutex_unlock(&pNode->mutex); - pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pNode->vgId, gSyncManager->syncTimerManager); + pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)(int64_t)pNode->vgId, gSyncManager->syncTimerManager); } \ No newline at end of file From 36e99c2259e4976e4e5645e941159507cc42fafe Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 21 Jan 2022 16:55:08 +0800 Subject: [PATCH 043/118] [modify] --- tests/script/sh/massiveTable/setupDnodes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index e45c7724ba..37fdbaf784 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -70,7 +70,7 @@ createNewCfgFile() { echo "dataDir ${dataDir}" >> ${cfgFile} echo "logDir ${logDir}" >> ${cfgFile} echo "serverPort ${serverPort}" >> ${cfgFile} - + echo "numOfLogLines 100000000" >> ${cfgFile} echo "supportVnodes 1024" >> ${cfgFile} #echo "asyncLog 0" >> ${cfgFile} echo "telemetryReporting 0" >> ${cfgFile} From e9984c992d582933ad99f58d5cfc893f6d807588 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 09:13:22 +0000 Subject: [PATCH 044/118] refact --- source/libs/tdb/src/db/tdb_mpool.c | 21 ++++++++++----------- source/libs/tdb/src/inc/tdb_mpool.h | 7 +++---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index e368801900..69b9a8a12f 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -47,24 +47,24 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TD_DLIST_INIT(&mp->freeList); - mp->pages = (pg_t **)calloc(mp->npages, sizeof(pg_t *)); + mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); if (mp->pages == NULL) { tdbError("failed to malloc memory pool pages"); goto _err; } for (frame_id_t i = 0; i < mp->npages; i++) { - mp->pages[i] = (pg_t *)calloc(1, MP_PAGE_SIZE(pgsize)); - if (mp->pages[i] == NULL) { + mp->pages[i].p = malloc(pgsize); + if (mp->pages[i].p == NULL) { goto _err; } - taosInitRWLatch(&mp->pages[i]->rwLatch); - mp->pages[i]->frameid = i; - mp->pages[i]->pgid = TDB_IVLD_PGID; + taosInitRWLatch(&mp->pages[i].rwLatch); + mp->pages[i].frameid = i; + mp->pages[i].pgid = TDB_IVLD_PGID; // add new page to the free list - TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), mp->pages[i], free); + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); } #define PGTAB_FACTOR 1.0 @@ -90,7 +90,7 @@ int tdbMPoolClose(TDB_MPOOL *mp) { tfree(mp->pgtab.hashtab); if (mp->pages) { for (int i = 0; i < mp->npages; i++) { - tfree(mp->pages[i]); + tfree(mp->pages[i].p); } free(mp->pages); @@ -182,7 +182,7 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { if (pagep) { // page is found // todo: pin the page and return - *(void **)addr = pagep->data; + *(void **)addr = pagep->p; return 0; } @@ -191,7 +191,6 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { if (pagep) { // has free page TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); - // todo: load the page from file and pin the page } else { // no free page available // pagep = tdbMpoolEvict(mp); @@ -208,7 +207,7 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { // load page from the disk if a container page is available // TODO: load the page from the disk - if (tdbMPoolFileReadPage(mpf, pgno, pagep->data) < 0) { + if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { return -1; } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index c92038dfc8..0c7613da63 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -32,14 +32,13 @@ typedef struct pg_t { frame_id_t frameid; pgid_t pgid; uint8_t dirty; + uint8_t rbit; int32_t pinRef; pg_free_dlist_node_t free; pg_hash_dlist_node_t hash; - uint8_t data[]; + void * p; } pg_t; -#define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) - typedef TD_DLIST(pg_t) pg_list_t; typedef struct { SRWLatch latch; @@ -49,7 +48,7 @@ struct TDB_MPOOL { int64_t cachesize; pgsize_t pgsize; int32_t npages; - pg_t ** pages; + pg_t * pages; pg_list_t freeList; struct { int32_t nbucket; From daa29bca42e3d91787d1d45db563a1fa8cdae60d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 09:35:33 +0000 Subject: [PATCH 045/118] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 36 ++++++++++++++++++++++++++--- source/libs/tdb/src/inc/tdb_mpool.h | 11 +++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 69b9a8a12f..a6df79c9b3 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -21,6 +21,7 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -44,6 +45,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->cachesize = cachesize; mp->pgsize = pgsize; mp->npages = cachesize / pgsize; + mp->clockHand = 0; TD_DLIST_INIT(&mp->freeList); @@ -193,10 +195,11 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); } else { // no free page available - // pagep = tdbMpoolEvict(mp); + tdbMPoolClockEvictPage(mp, &pagep); if (pagep) { - } else { - // TODO: Cannot find a page to evict + if (pagep->dirty) { + // TODO: Handle dirty page eviction + } } } @@ -346,4 +349,31 @@ static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { // TODO: handle error return 0; +} + +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { + pg_t * pagep; + frame_id_t och; + + *pagepp = NULL; + och = mp->clockHand; + + do { + pagep = mp->pages + mp->clockHand; + mp->clockHand = (mp->clockHand + 1) % mp->npages; + + if (pagep->pinRef == 0) { + if (pagep->rbit == 1) { + pagep->rbit = 0; + } else { + break; + } + } + + if (mp->clockHand == och) { + return; + } + } while (1); + + *pagepp = pagep; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 0c7613da63..d951040d8d 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -45,11 +45,12 @@ typedef struct { TD_DLIST(TDB_MPFILE); } mpf_bucket_t; struct TDB_MPOOL { - int64_t cachesize; - pgsize_t pgsize; - int32_t npages; - pg_t * pages; - pg_list_t freeList; + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; + frame_id_t clockHand; struct { int32_t nbucket; pg_list_t *hashtab; From 314e1e1af423094e1ae68abd1e5c1a974dd119d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 10:14:39 +0000 Subject: [PATCH 046/118] more tdb --- source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/src/db/tdb_mpool.c | 2 +- source/libs/tdb/src/inc/tdb_mpool.h | 4 +++- source/libs/tdb/test/CMakeLists.txt | 4 ++++ source/libs/tdb/test/tdbMPoolTest.cpp | 31 +++++++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 4 ++-- 6 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 source/libs/tdb/test/tdbMPoolTest.cpp diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index bb8904117c..3cb5a65572 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -18,5 +18,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - # add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index a6df79c9b3..9b67f405a9 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -225,7 +225,7 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { return 0; } -int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { // TODO return 0; } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d951040d8d..37c82f3833 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -76,6 +76,7 @@ struct TDB_MPFILE { // TDB_MPOOL int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbMPoolClose(TDB_MPOOL *mp); +int tdbMPoolSync(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); @@ -83,7 +84,8 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf); int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileSync(TDB_MPFILE *mpf); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 2d77c1f4e9..7fbfaf5506 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -1,3 +1,7 @@ +# tdbMPoolTest +add_executable(tdbMPoolTest "tdbMPoolTest.cpp") +target_link_libraries(tdbMPoolTest tdb gtest gtest_main) + # tdbTest add_executable(tdbTest "tdbTest.cpp") target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file diff --git a/source/libs/tdb/test/tdbMPoolTest.cpp b/source/libs/tdb/test/tdbMPoolTest.cpp new file mode 100644 index 0000000000..17381759fb --- /dev/null +++ b/source/libs/tdb/test/tdbMPoolTest.cpp @@ -0,0 +1,31 @@ +#include "gtest/gtest.h" + +#include + +#include "tdb_mpool.h" + +TEST(tdb_mpool_test, test1) { + TDB_MPOOL * mp; + TDB_MPFILE *mpf; + pgno_t pgno; + void * pgdata; + + // open mp + tdbMPoolOpen(&mp, 16384, 4096); + + // open mpf + tdbMPoolFileOpen(&mpf, "test.db", mp); + +#define TEST1_TOTAL_PAGES 100 + for (int i = 0; i < TEST1_TOTAL_PAGES; i++) { + tdbMPoolFileNewPage(mpf, &pgno, pgdata); + + *(pgno_t *)pgdata = i; + } + + // close mpf + tdbMPoolFileClose(mpf); + + // close mp + tdbMPoolClose(mp); +} diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index f27e17f1ca..113bb2560f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,8 +3,8 @@ #include "tdb.h" TEST(tdb_api_test, tdb_create_open_close_db_test) { - int ret; - TDB *dbp; + // int ret; + // TDB *dbp; // tdbCreateDB(&dbp, TDB_BTREE_T); From 2e14f6345343136f31d4bcb595ec49bcc98579f3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 21 Jan 2022 02:19:40 -0800 Subject: [PATCH 047/118] Remove warnings --- 2.0/src/query/tests/astTest.cpp | 4 +- 2.0/src/query/tests/cSortTest.cpp | 5 +- 2.0/src/query/tests/histogramTest.cpp | 3 + 2.0/src/query/tests/patternMatchTest.cpp | 3 + 2.0/src/query/tests/percentileTest.cpp | 3 + 2.0/src/query/tests/rangeMergeTest.cpp | 3 + 2.0/src/query/tests/resultBufferTest.cpp | 3 + 2.0/src/query/tests/tsBufTest.cpp | 3 + 2.0/src/query/tests/unitTest.cpp | 2 + CMakeLists.txt | 4 +- source/client/test/clientTests.cpp | 7 +- source/common/test/commonTests.cpp | 5 +- source/dnode/vnode/inc/tq.h | 2 + source/dnode/vnode/src/meta/metaTbCfg.c | 2 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 5 +- source/libs/catalog/test/catalogTests.cpp | 441 ++++++++---------- source/libs/executor/test/executorTests.cpp | 7 +- source/libs/parser/test/mockCatalog.cpp | 6 + source/libs/parser/test/parserTests.cpp | 7 +- source/libs/parser/test/plannerTest.cpp | 7 +- source/libs/parser/test/tokenizerTest.cpp | 4 +- source/libs/planner/test/plannerTests.cpp | 6 +- source/libs/qcom/src/querymsg.c | 8 +- source/libs/qcom/test/queryTest.cpp | 5 +- source/libs/qworker/inc/qworkerInt.h | 3 +- source/libs/qworker/test/qworkerTests.cpp | 15 +- source/libs/scheduler/inc/schedulerInt.h | 13 +- source/libs/scheduler/test/schedulerTests.cpp | 18 +- source/util/test/encodeTest.cpp | 15 +- 29 files changed, 327 insertions(+), 282 deletions(-) diff --git a/2.0/src/query/tests/astTest.cpp b/2.0/src/query/tests/astTest.cpp index b7df749983..1109d25f29 100644 --- a/2.0/src/query/tests/astTest.cpp +++ b/2.0/src/query/tests/astTest.cpp @@ -632,4 +632,6 @@ void exprSerializeTest2() { TEST(testCase, astTest) { // exprSerializeTest2(); } -#endif \ No newline at end of file +#endif + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/cSortTest.cpp b/2.0/src/query/tests/cSortTest.cpp index aa5aa89afc..64e3b41b49 100644 --- a/2.0/src/query/tests/cSortTest.cpp +++ b/2.0/src/query/tests/cSortTest.cpp @@ -5,6 +5,7 @@ #include "tsdb.h" #include "qExtbuffer.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -121,4 +122,6 @@ TEST(testCase, columnsort_test) { printf("\n"); destroyColumnModel(pModel); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/histogramTest.cpp b/2.0/src/query/tests/histogramTest.cpp index 0266ecffc1..ff5ac08cdc 100644 --- a/2.0/src/query/tests/histogramTest.cpp +++ b/2.0/src/query/tests/histogramTest.cpp @@ -6,6 +6,7 @@ #include "taos.h" #include "qHistogram.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -140,3 +141,5 @@ TEST(testCase, heapsort) { // // free(pEntry); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/patternMatchTest.cpp b/2.0/src/query/tests/patternMatchTest.cpp index 091604c65c..3b46c91bc3 100644 --- a/2.0/src/query/tests/patternMatchTest.cpp +++ b/2.0/src/query/tests/patternMatchTest.cpp @@ -6,6 +6,7 @@ #include "qAggMain.h" #include "tcompare.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -84,3 +85,5 @@ TEST(testCase, patternMatchTest) { ret = patternMatch("%9", str, 2, &info); EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/percentileTest.cpp b/2.0/src/query/tests/percentileTest.cpp index 1b6951201a..7c4d85b3ae 100644 --- a/2.0/src/query/tests/percentileTest.cpp +++ b/2.0/src/query/tests/percentileTest.cpp @@ -7,6 +7,7 @@ #include "qPercentile.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -255,3 +256,5 @@ TEST(testCase, percentileTest) { unsignedDataTest(); largeDataTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/rangeMergeTest.cpp b/2.0/src/query/tests/rangeMergeTest.cpp index f7fc558ccf..9bbf47e56d 100644 --- a/2.0/src/query/tests/rangeMergeTest.cpp +++ b/2.0/src/query/tests/rangeMergeTest.cpp @@ -7,6 +7,7 @@ #include "qFilter.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -365,3 +366,5 @@ TEST(testCase, rangeMergeTest) { intDataTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp index 54ac0bf4e5..9724b98f7c 100644 --- a/2.0/src/query/tests/resultBufferTest.cpp +++ b/2.0/src/query/tests/resultBufferTest.cpp @@ -6,6 +6,7 @@ #include "taos.h" #include "tsdb.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -161,3 +162,5 @@ TEST(testCase, resultBufferTest) { writeDownTest(); recyclePageTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/tsBufTest.cpp b/2.0/src/query/tests/tsBufTest.cpp index 04c5a15252..62ffa785c6 100644 --- a/2.0/src/query/tests/tsBufTest.cpp +++ b/2.0/src/query/tests/tsBufTest.cpp @@ -9,6 +9,7 @@ #include "ttoken.h" #include "tutil.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wunused-but-set-variable" @@ -513,3 +514,5 @@ TEST(testCase, tsBufTest) { mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/unitTest.cpp b/2.0/src/query/tests/unitTest.cpp index 1ed4cde406..57f66ef466 100644 --- a/2.0/src/query/tests/unitTest.cpp +++ b/2.0/src/query/tests/unitTest.cpp @@ -4,6 +4,7 @@ #include "taos.h" #include "tsdb.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -910,3 +911,4 @@ TEST(testCase, getTempFilePath_test) { printf("%s\n", path); } +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 99e48006b1..0383cc8aed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 736a47273f..b0f7645d82 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -17,8 +17,9 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -678,4 +679,6 @@ TEST(testCase, create_topic_Test) { // // taos_free_result(pRes); // taos_close(pConn); -//} \ No newline at end of file +//} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 2f821ee8b9..b91b6b06f2 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,7 +1,8 @@ #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wunused-but-set-variable" @@ -94,3 +95,5 @@ TEST(testCase, toInteger_test) { ret = toInteger(s, strlen(s), 10, &val, &sign); ASSERT_EQ(ret, -1); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 8c4effa221..2685ac4964 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -320,6 +320,8 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); +int tqSetCursor(STQ*, STqSetCurReq* pMsg); + #if 0 int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); int tqSetCursor(STQ*, STqSetCurReq* pMsg); diff --git a/source/dnode/vnode/src/meta/metaTbCfg.c b/source/dnode/vnode/src/meta/metaTbCfg.c index 4e02b64ce0..d71d319a79 100644 --- a/source/dnode/vnode/src/meta/metaTbCfg.c +++ b/source/dnode/vnode/src/meta/metaTbCfg.c @@ -32,7 +32,7 @@ size_t metaEncodeTbObjFromTbOptions(const STbCfg *pTbOptions, void *pBuf, size_t switch (pTbOptions->type) { case META_SUPER_TABLE: tlen += taosEncodeFixedU64(ppBuf, pTbOptions->stbCfg.suid); - tlen += tdEncodeSchema(ppBuf, pTbOptions->stbCfg.pTagSchema); + tlen += tdEncodeSchema(ppBuf, (STSchema *)pTbOptions->stbCfg.pTagSchema); // TODO: encode schema version array break; case META_CHILD_TABLE: diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index dd1e5ba9ae..9390e19523 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -19,7 +19,10 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); } +int vnodeQueryOpen(SVnode *pVnode) { + return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, + (putReqToQueryQFp)vnodePutReqToVQueryQ); +} int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("query message is processing"); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 436593f9d6..a01c3bcf5d 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -16,27 +16,28 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#include "os.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "os.h" +#pragma GCC diagnostic ignored "-Wformat" +#include "addr_any.h" +#include "catalog.h" +#include "stub.h" #include "taos.h" #include "tdef.h" -#include "tvariant.h" -#include "catalog.h" #include "tep.h" #include "trpc.h" -#include "stub.h" -#include "addr_any.h" - - +#include "tvariant.h" namespace { -extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist); +extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, + int32_t *exist); extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); void ctgTestSetPrepareTableMeta(); @@ -44,9 +45,9 @@ void ctgTestSetPrepareCTableMeta(); void ctgTestSetPrepareSTableMeta(); void ctgTestSetPrepareMultiSTableMeta(); -bool ctgTestStop = false; -bool ctgTestEnableSleep = false; -bool ctgTestDeadLoop = false; +bool ctgTestStop = false; +bool ctgTestEnableSleep = false; +bool ctgTestDeadLoop = false; int32_t ctgTestPrintNum = 200000; int32_t ctgTestMTRunSec = 30; @@ -61,14 +62,13 @@ int32_t ctgTestSuid = 2; int64_t ctgTestDbId = 33; uint64_t ctgTestClusterId = 0x1; -char *ctgTestDbname = "1.db1"; -char *ctgTestTablename = "table1"; -char *ctgTestCTablename = "ctable1"; -char *ctgTestSTablename = "stable1"; - +char *ctgTestDbname = "1.db1"; +char *ctgTestTablename = "table1"; +char *ctgTestCTablename = "ctable1"; +char *ctgTestSTablename = "stable1"; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(sizeof(SCreateDbReq)); + SCreateDbReq *pReq = (SCreateDbReq *)rpcMallocCont(sizeof(SCreateDbReq)); strcpy(pReq->db, "1.db1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -89,7 +89,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { pReq->update = 0; pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; rpcMsg.contLen = sizeof(SCreateDbReq); @@ -103,8 +103,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { } void ctgTestInitLogFile() { - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; tsAsyncLog = 0; @@ -113,7 +113,6 @@ void ctgTestInitLogFile() { if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } - } int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { @@ -152,10 +151,10 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { output->tbMeta->tableInfo.numOfColumns = ctgTestColNum; output->tbMeta->tableInfo.numOfTags = ctgTestTagNum; - + output->tbMeta->sversion = ctgTestSVersion; output->tbMeta->tversion = ctgTestTVersion; - + SSchema *s = NULL; s = &output->tbMeta->schema[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; @@ -174,18 +173,17 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { s->colId = 3; s->bytes = 12; strcpy(s->name, "tag1s"); - } void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; - int32_t vgNum = 0; - SVgroupInfo vgInfo = {0}; - + int32_t vgNum = 0; + SVgroupInfo vgInfo = {0}; + dbVgroup->vgVersion = vgVersion++; - + ctgTestCurrentVgVersion = dbVgroup->vgVersion; - + dbVgroup->hashMethod = 0; dbVgroup->dbId = ctgTestDbId; dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -210,9 +208,9 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { } void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - SUseDbRsp *rspMsg = NULL; //todo + SUseDbRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(SUseDbRsp) + ctgTestVgNum * sizeof(SVgroupInfo); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (SUseDbRsp *)pRsp->pCont; @@ -224,7 +222,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM rspMsg->uid = htobe64(ctgTestDbId); SVgroupInfo *vg = NULL; - uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; for (int32_t i = 0; i < ctgTestVgNum; ++i) { vg = &rspMsg->vgroupInfo[i]; @@ -245,13 +243,10 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM return; } - - - void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -279,15 +274,14 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM s->colId = htonl(2); s->bytes = htonl(4); strcpy(s->name, "col1"); - + return; } - void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -323,15 +317,13 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc s->bytes = htonl(12); strcpy(s->name, "tag1s"); - return; } - void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -367,15 +359,14 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc s->bytes = htonl(12); strcpy(s->name, "tag1s"); - return; } void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo static int32_t idx = 1; - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -412,55 +403,50 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, strcpy(s->name, "tag1s"); ++idx; - + return; } - - void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareTableMeta(); - + return; } - void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareCTableMeta(); - + return; } void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareSTableMeta(); - + return; } void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareMultiSTableMeta(); - + return; } - - void ctgTestSetPrepareDbVgroups() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroups); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroups); } } @@ -470,10 +456,10 @@ void ctgTestSetPrepareTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareTableMeta); } } @@ -483,10 +469,10 @@ void ctgTestSetPrepareCTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareCTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareCTableMeta); } } @@ -496,10 +482,10 @@ void ctgTestSetPrepareSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareSTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareSTableMeta); } } @@ -509,38 +495,36 @@ void ctgTestSetPrepareMultiSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareMultiSTableMeta); } } } - void ctgTestSetPrepareDbVgroupsAndNormalMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta); } } } - void ctgTestSetPrepareDbVgroupsAndChildMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta); } } @@ -550,10 +534,10 @@ void ctgTestSetPrepareDbVgroupsAndSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta); } } @@ -563,25 +547,24 @@ void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta); } } } - -} +} // namespace void *ctgTestGetDbVgroupThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - void *mockPointer = (void *)0x1; - SArray *vgList = NULL; - int32_t n = 0; - + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + void *mockPointer = (void *)0x1; + SArray *vgList = NULL; + int32_t n = 0; + while (!ctgTestStop) { code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); if (code) { @@ -593,7 +576,7 @@ void *ctgTestGetDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand()%5); + usleep(rand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); @@ -604,11 +587,11 @@ void *ctgTestGetDbVgroupThread(void *param) { } void *ctgTestSetDbVgroupThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; - int32_t n = 0; - + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + SDBVgroupInfo dbVgroup = {0}; + int32_t n = 0; + while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); @@ -616,8 +599,8 @@ void *ctgTestSetDbVgroupThread(void *param) { assert(0); } - if (ctgTestEnableSleep) { - usleep(rand()%5); + if (ctgTestEnableSleep) { + usleep(rand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -625,20 +608,19 @@ void *ctgTestSetDbVgroupThread(void *param) { } return NULL; - } void *ctgTestGetCtableMetaThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - int32_t n = 0; - STableMeta* tbMeta = NULL; - int32_t exist = 0; + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + int32_t n = 0; + STableMeta *tbMeta = NULL; + int32_t exist = 0; SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); - + while (!ctgTestStop) { code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist); if (code || 0 == exist) { @@ -648,9 +630,9 @@ void *ctgTestGetCtableMetaThread(void *param) { tfree(tbMeta); if (ctgTestEnableSleep) { - usleep(rand()%5); + usleep(rand() % 5); } - + if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); } @@ -660,22 +642,22 @@ void *ctgTestGetCtableMetaThread(void *param) { } void *ctgTestSetCtableMetaThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; - int32_t n = 0; + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + SDBVgroupInfo dbVgroup = {0}; + int32_t n = 0; STableMetaOutput output = {0}; ctgTestBuildCTableMetaOutput(&output); - + while (!ctgTestStop) { code = ctgUpdateTableMetaCache(pCtg, &output); if (code) { assert(0); } - if (ctgTestEnableSleep) { - usleep(rand()%5); + if (ctgTestEnableSleep) { + usleep(rand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -685,21 +667,19 @@ void *ctgTestSetCtableMetaThread(void *param) { tfree(output.tbMeta); return NULL; - } - TEST(tableMeta, normalTable) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; ctgTestSetPrepareDbVgroups(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -715,7 +695,6 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.numOfEps, 3); - ctgTestSetPrepareTableMeta(); STableMeta *tableMeta = NULL; @@ -742,19 +721,19 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stb = NULL; - uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; - int32_t i = 0; + uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; + int32_t i = 0; while (i < 5) { ++i; code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum); ASSERT_EQ(code, 0); code = catalogGetExpiredSTables(pCtg, &stb, &stbNum); ASSERT_EQ(code, 0); - + if (dbNum) { - printf("got expired db,dbId:%"PRId64"\n", dbs->dbId); + printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); free(dbs); dbs = NULL; } else { @@ -762,7 +741,7 @@ TEST(tableMeta, normalTable) { } if (stbNum) { - printf("got expired stb,suid:%"PRId64"\n", stb->suid); + printf("got expired stb,suid:%" PRId64 "\n", stb->suid); free(stb); stb = NULL; } else { @@ -773,7 +752,7 @@ TEST(tableMeta, normalTable) { allStbNum += stbNum; sleep(2); } - + ASSERT_EQ(allDbNum, 1); ASSERT_EQ(allStbNum, 0); @@ -781,18 +760,18 @@ TEST(tableMeta, normalTable) { } TEST(tableMeta, childTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - + code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -838,19 +817,19 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stb = NULL; - uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; - int32_t i = 0; + uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; + int32_t i = 0; while (i < 5) { ++i; code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum); ASSERT_EQ(code, 0); code = catalogGetExpiredSTables(pCtg, &stb, &stbNum); ASSERT_EQ(code, 0); - + if (dbNum) { - printf("got expired db,dbId:%"PRId64"\n", dbs->dbId); + printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); free(dbs); dbs = NULL; } else { @@ -858,7 +837,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%"PRId64"\n", stb->suid); + printf("got expired stb,suid:%" PRId64 "\n", stb->suid); free(stb); stb = NULL; } else { @@ -869,18 +848,17 @@ TEST(tableMeta, childTableCase) { allStbNum += stbNum; sleep(2); } - + ASSERT_EQ(allDbNum, 1); ASSERT_EQ(allStbNum, 1); - catalogDestroy(); } TEST(tableMeta, superTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; ctgTestSetPrepareDbVgroupsAndSuperMeta(); @@ -889,7 +867,7 @@ TEST(tableMeta, superTableCase) { int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -940,19 +918,19 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stb = NULL; - uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; - int32_t i = 0; + uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; + int32_t i = 0; while (i < 5) { ++i; code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum); ASSERT_EQ(code, 0); code = catalogGetExpiredSTables(pCtg, &stb, &stbNum); ASSERT_EQ(code, 0); - + if (dbNum) { - printf("got expired db,dbId:%"PRId64"\n", dbs->dbId); + printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); free(dbs); dbs = NULL; } else { @@ -960,7 +938,7 @@ TEST(tableMeta, superTableCase) { } if (stbNum) { - printf("got expired stb,suid:%"PRId64"\n", stb->suid); + printf("got expired stb,suid:%" PRId64 "\n", stb->suid); free(stb); stb = NULL; } else { @@ -971,19 +949,18 @@ TEST(tableMeta, superTableCase) { allStbNum += stbNum; sleep(2); } - + ASSERT_EQ(allDbNum, 1); ASSERT_EQ(allStbNum, 1); - catalogDestroy(); } TEST(tableDistVgroup, normalTable) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndNormalMeta(); @@ -992,8 +969,8 @@ TEST(tableDistVgroup, normalTable) { int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -1012,17 +989,17 @@ TEST(tableDistVgroup, normalTable) { } TEST(tableDistVgroup, childTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1040,15 +1017,14 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(vgInfo->vgId, 9); ASSERT_EQ(vgInfo->numOfEps, 4); - catalogDestroy(); } TEST(tableDistVgroup, superTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndSuperMeta(); @@ -1057,7 +1033,7 @@ TEST(tableDistVgroup, superTableCase) { int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -1078,24 +1054,23 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(vgInfo->vgId, 3); ASSERT_EQ(vgInfo->numOfEps, 3); - catalogDestroy(); } TEST(dbVgroup, getSetDbVgroupCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndNormalMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1109,7 +1084,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); - + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); @@ -1139,27 +1114,27 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(pvgInfo->vgId, 8); ASSERT_EQ(pvgInfo->numOfEps, 3); taosArrayDestroy(vgList); - + catalogDestroy(); } TEST(multiThread, getSetDbVgroupCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - + ctgTestSetPrepareDbVgroups(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1187,28 +1162,28 @@ TEST(multiThread, getSetDbVgroupCase) { break; } } - + ctgTestStop = true; sleep(1); - + catalogDestroy(); } TEST(multiThread, ctableMeta) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestStop = false; ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1235,25 +1210,24 @@ TEST(multiThread, ctableMeta) { break; } } - + ctgTestStop = true; sleep(1); - + catalogDestroy(); } - TEST(rentTest, allRent) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestStop = false; - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stable = NULL; - uint32_t num = 0; + uint32_t num = 0; ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); @@ -1265,7 +1239,6 @@ TEST(rentTest, allRent) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); @@ -1285,41 +1258,37 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - + code = catalogGetExpiredDBs(pCtg, &dbs, &num); ASSERT_EQ(code, 0); printf("%d - expired dbNum:%d\n", i, num); if (dbs) { - printf("%d - expired dbId:%"PRId64", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); + printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); free(dbs); dbs = NULL; } - + code = catalogGetExpiredSTables(pCtg, &stable, &num); ASSERT_EQ(code, 0); printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%"PRId64", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; } printf("*************************************************\n"); - + sleep(2); } - + catalogDestroy(); } - - -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - - - \ No newline at end of file +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index c528d879a3..5e0b641b28 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -17,8 +17,9 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -219,4 +220,6 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo, &sinkHandle); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index e8d975c22e..d68b04a384 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -18,8 +18,14 @@ #include #include "stub.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" + #include "addr_any.h" +#pragma GCC diagnostic pop + namespace { void generateTestT1(MockCatalogService* mcs) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 4847b50082..b408e15abe 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -17,8 +17,9 @@ #include #include #include "tglobal.h" -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -781,4 +782,6 @@ TEST(testCase, create_user_Test) { ASSERT_NE(output, nullptr); destroySqlInfo(&info1); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index a278e0053c..43daff9fec 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -17,8 +17,9 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -203,4 +204,6 @@ TEST(testCase, displayPlan) { // Projection(cols: [ts #0], [a #1], [b #2]) filters:(nil) // TableScan(t.1abc #110) time_range: -9223372036854775808 - 9223372036854775807 -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 7ed052e8cc..888eeae73b 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -1,7 +1,8 @@ #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -722,3 +723,4 @@ TEST(testCase, extractMeta_test) { destroySqlInfo(&info1); } +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 4b408e67db..765651a31a 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -22,8 +22,8 @@ #include "parser.h" #include "mockCatalog.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" - #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -102,4 +102,6 @@ TEST(testCase, planner_test) { // destroyQueryInfo(pQueryInfo); // qParserCleanupMetaRequestInfo(&req); // destroySqlInfo(&info1); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 2cde28baf9..52e632ffbb 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -18,6 +18,9 @@ #include "query.h" #include "trpc.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-truncation" + int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize) = {0}; @@ -288,7 +291,4 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } - - - - +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index 8fc6b7e529..9ca7442d55 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -18,8 +18,9 @@ #include "tmsg.h" #include "query.h" #include "trpc.h" -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -82,3 +83,5 @@ TEST(testCase, error_in_async_test) { usleep(1000); printf("Error code:%d after asynchronously exec function\n", code); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 5f9b33f7e3..15c7c0d64b 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -219,8 +219,7 @@ typedef struct SQWorkerMgmt { } \ } while (0) - - +int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); #ifdef __cplusplus } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d1cc9f03d1..0ed5c0c816 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -16,11 +16,17 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + #include "os.h" #include "taos.h" @@ -461,11 +467,11 @@ void *controlThread(void *param) { } void *queryQueueThread(void *param) { - + return NULL; } void *fetchQueueThread(void *param) { - + return NULL; } @@ -783,5 +789,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } - - +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 6b047eb96e..a82c75d295 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -146,12 +146,15 @@ typedef struct SSchJob { #define SCH_SET_JOB_TYPE(pAttr, type) (pAttr)->queryJob = ((type) != QUERY_TYPE_MODIFY) #define SCH_JOB_NEED_FETCH(pAttr) ((pAttr)->queryJob) -#define SCH_JOB_ELOG(param, ...) qError("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) -#define SCH_JOB_DLOG(param, ...) qDebug("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_ELOG(param, ...) qError("QID:%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_DLOG(param, ...) qDebug("QID:%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) \ + qError("QID:%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) \ + qDebug("QID:%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) \ + qWarn("QID:%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 1425ac0e6c..fb13149713 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -16,11 +16,7 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" #include "taos.h" @@ -30,6 +26,16 @@ #include "scheduler.h" #include "tep.h" #include "trpc.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wliteral-suffix" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wreturn-type" +#pragma GCC diagnostic ignored "-Wformat" + #include "schedulerInt.h" #include "stub.h" #include "addr_any.h" @@ -680,6 +686,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } - - - +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index b178ee0b10..1b13c102a5 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -4,6 +4,11 @@ #include "encode.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshift-count-overflow" +#pragma GCC diagnostic ignored "-Woverflow" +#pragma GCC diagnostic ignored "-Woverflow" + #define BUF_SIZE 64 td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN}; @@ -354,10 +359,10 @@ TEST(td_encode_test, compound_struct_encode_test) { SCoder encoder, decoder; uint8_t * buf1; int32_t buf1size; - uint8_t * buf2; + uint8_t *buf2; int32_t buf2size; - SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = "Hello"}; - SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = "Hello", .A_d = 67, .A_e = 13}; + SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = (char *)"Hello"}; + SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = (char *)"Hello", .A_d = 67, .A_e = 13}; SFinalReq_v1 req1 = {.pA = &sa1, .v_a = 15, .v_b = 35}; SFinalReq_v2 req2 = {.pA = &sa2, .v_a = 15, .v_b = 32, .v_c = 37}; SFinalReq_v1 dreq11, dreq21; @@ -430,4 +435,6 @@ TEST(td_encode_test, compound_struct_encode_test) { GTEST_ASSERT_EQ(dreq21.v_a, req2.v_a); GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b); tCoderClear(&decoder); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file From b76470aaab225f7d430c1a4190a6ec129828d27a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 21 Jan 2022 18:27:00 +0800 Subject: [PATCH 048/118] add traft demo code --- contrib/test/traft/CMakeLists.txt | 3 +- contrib/test/traft/cluster/Makefile.2 | 16 + .../traft/{make_cluster => cluster}/clear.sh | 0 contrib/test/traft/cluster/node10000.c | 117 +++ .../test/traft/cluster/node10000_restart.c | 113 +++ contrib/test/traft/cluster/node10001.c | 117 +++ .../test/traft/cluster/node10001_restart.c | 113 +++ contrib/test/traft/cluster/node10002.c | 117 +++ .../test/traft/cluster/node10002_restart.c | 113 +++ .../test/traft/join_into_vgroup/Makefile.2 | 10 + .../clear.sh | 0 .../join_into_vgroup/node_follower10000.c | 113 +++ .../join_into_vgroup/node_follower10001.c | 113 +++ .../traft/join_into_vgroup/node_leader10002.c | 144 ++++ .../test/traft/make_cluster/CMakeLists.txt | 11 - contrib/test/traft/make_cluster/common.h | 23 - contrib/test/traft/make_cluster/config.c | 64 -- contrib/test/traft/make_cluster/config.h | 31 - contrib/test/traft/make_cluster/console.c | 202 ------ contrib/test/traft/make_cluster/console.h | 19 - contrib/test/traft/make_cluster/raftMain.c | 81 --- contrib/test/traft/make_cluster/raftServer.c | 286 -------- contrib/test/traft/make_cluster/raftServer.h | 66 -- contrib/test/traft/make_cluster/simpleHash.c | 218 ------ contrib/test/traft/make_cluster/simpleHash.h | 61 -- contrib/test/traft/make_cluster/util.c | 45 -- contrib/test/traft/make_cluster/util.h | 17 - .../traft/rebalance_leader/CMakeLists.txt | 7 - contrib/test/traft/rebalance_leader/common.h | 36 - .../test/traft/rebalance_leader/raftMain.c | 678 ------------------ .../test/traft/rebalance_leader/raftServer.c | 224 ------ .../test/traft/rebalance_leader/raftServer.h | 70 -- contrib/test/traft/single_node/CMakeLists.txt | 6 + contrib/test/traft/single_node/clear.sh | 4 + contrib/test/traft/single_node/cmd | 6 + contrib/test/traft/single_node/singleNode.c | 111 +++ 36 files changed, 1214 insertions(+), 2141 deletions(-) create mode 100644 contrib/test/traft/cluster/Makefile.2 rename contrib/test/traft/{make_cluster => cluster}/clear.sh (100%) create mode 100644 contrib/test/traft/cluster/node10000.c create mode 100644 contrib/test/traft/cluster/node10000_restart.c create mode 100644 contrib/test/traft/cluster/node10001.c create mode 100644 contrib/test/traft/cluster/node10001_restart.c create mode 100644 contrib/test/traft/cluster/node10002.c create mode 100644 contrib/test/traft/cluster/node10002_restart.c create mode 100644 contrib/test/traft/join_into_vgroup/Makefile.2 rename contrib/test/traft/{rebalance_leader => join_into_vgroup}/clear.sh (100%) create mode 100644 contrib/test/traft/join_into_vgroup/node_follower10000.c create mode 100644 contrib/test/traft/join_into_vgroup/node_follower10001.c create mode 100644 contrib/test/traft/join_into_vgroup/node_leader10002.c delete mode 100644 contrib/test/traft/make_cluster/CMakeLists.txt delete mode 100644 contrib/test/traft/make_cluster/common.h delete mode 100644 contrib/test/traft/make_cluster/config.c delete mode 100644 contrib/test/traft/make_cluster/config.h delete mode 100644 contrib/test/traft/make_cluster/console.c delete mode 100644 contrib/test/traft/make_cluster/console.h delete mode 100644 contrib/test/traft/make_cluster/raftMain.c delete mode 100644 contrib/test/traft/make_cluster/raftServer.c delete mode 100644 contrib/test/traft/make_cluster/raftServer.h delete mode 100644 contrib/test/traft/make_cluster/simpleHash.c delete mode 100644 contrib/test/traft/make_cluster/simpleHash.h delete mode 100644 contrib/test/traft/make_cluster/util.c delete mode 100644 contrib/test/traft/make_cluster/util.h delete mode 100644 contrib/test/traft/rebalance_leader/CMakeLists.txt delete mode 100644 contrib/test/traft/rebalance_leader/common.h delete mode 100644 contrib/test/traft/rebalance_leader/raftMain.c delete mode 100644 contrib/test/traft/rebalance_leader/raftServer.c delete mode 100644 contrib/test/traft/rebalance_leader/raftServer.h create mode 100644 contrib/test/traft/single_node/CMakeLists.txt create mode 100644 contrib/test/traft/single_node/clear.sh create mode 100644 contrib/test/traft/single_node/cmd create mode 100644 contrib/test/traft/single_node/singleNode.c diff --git a/contrib/test/traft/CMakeLists.txt b/contrib/test/traft/CMakeLists.txt index d165c6d4dc..32e391646a 100644 --- a/contrib/test/traft/CMakeLists.txt +++ b/contrib/test/traft/CMakeLists.txt @@ -1,3 +1,2 @@ -add_subdirectory(rebalance_leader) -add_subdirectory(make_cluster) +add_subdirectory(single_node) diff --git a/contrib/test/traft/cluster/Makefile.2 b/contrib/test/traft/cluster/Makefile.2 new file mode 100644 index 0000000000..0a4f6ff325 --- /dev/null +++ b/contrib/test/traft/cluster/Makefile.2 @@ -0,0 +1,16 @@ +all: + gcc node10000.c -I ../../include/ ../../.libs/libraft.a -o node10000 -luv -llz4 -lpthread -g + gcc node10001.c -I ../../include/ ../../.libs/libraft.a -o node10001 -luv -llz4 -lpthread -g + gcc node10002.c -I ../../include/ ../../.libs/libraft.a -o node10002 -luv -llz4 -lpthread -g + gcc node10000_restart.c -I ../../include/ ../../.libs/libraft.a -o node10000_restart -luv -llz4 -lpthread -g + gcc node10001_restart.c -I ../../include/ ../../.libs/libraft.a -o node10001_restart -luv -llz4 -lpthread -g + gcc node10002_restart.c -I ../../include/ ../../.libs/libraft.a -o node10002_restart -luv -llz4 -lpthread -g +clean: + rm -f node10000 + rm -f node10001 + rm -f node10002 + rm -f node10000_restart + rm -f node10001_restart + rm -f node10002_restart + sh clear.sh + diff --git a/contrib/test/traft/make_cluster/clear.sh b/contrib/test/traft/cluster/clear.sh similarity index 100% rename from contrib/test/traft/make_cluster/clear.sh rename to contrib/test/traft/cluster/clear.sh diff --git a/contrib/test/traft/cluster/node10000.c b/contrib/test/traft/cluster/node10000.c new file mode 100644 index 0000000000..96e0b067c2 --- /dev/null +++ b/contrib/test/traft/cluster/node10000.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // array gives the peer replica infomation. + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10001"); + snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10002"); + uint32_t peersCount = 2; + r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10000_restart.c b/contrib/test/traft/cluster/node10000_restart.c new file mode 100644 index 0000000000..c9538552ed --- /dev/null +++ b/contrib/test/traft/cluster/node10000_restart.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // here only add self. + // peer replica information will restore from wal. + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10001.c b/contrib/test/traft/cluster/node10001.c new file mode 100644 index 0000000000..08636637ac --- /dev/null +++ b/contrib/test/traft/cluster/node10001.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // array gives the peer replica infomation. + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10000"); + snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10002"); + uint32_t peersCount = 2; + r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10001_restart.c b/contrib/test/traft/cluster/node10001_restart.c new file mode 100644 index 0000000000..10d64d76ef --- /dev/null +++ b/contrib/test/traft/cluster/node10001_restart.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // here only add self. + // peer replica information will restore from wal. + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10002.c b/contrib/test/traft/cluster/node10002.c new file mode 100644 index 0000000000..ebc3598075 --- /dev/null +++ b/contrib/test/traft/cluster/node10002.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // array gives the peer replica infomation. + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10000"); + snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10001"); + uint32_t peersCount = 2; + r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10002_restart.c b/contrib/test/traft/cluster/node10002_restart.c new file mode 100644 index 0000000000..d772e97b2b --- /dev/null +++ b/contrib/test/traft/cluster/node10002_restart.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // here only add self. + // peer replica information will restore from wal. + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/join_into_vgroup/Makefile.2 b/contrib/test/traft/join_into_vgroup/Makefile.2 new file mode 100644 index 0000000000..92f5b7269a --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/Makefile.2 @@ -0,0 +1,10 @@ +all: + gcc node_follower10000.c -I ../../include/ ../../.libs/libraft.a -o node_follower10000 -luv -llz4 -lpthread -g + gcc node_follower10001.c -I ../../include/ ../../.libs/libraft.a -o node_follower10001 -luv -llz4 -lpthread -g + gcc node_leader10002.c -I ../../include/ ../../.libs/libraft.a -o node_leader10002 -luv -llz4 -lpthread -g +clean: + rm -f node_follower10000 + rm -f node_follower10001 + rm -f node_leader10002 + sh clear.sh + diff --git a/contrib/test/traft/rebalance_leader/clear.sh b/contrib/test/traft/join_into_vgroup/clear.sh similarity index 100% rename from contrib/test/traft/rebalance_leader/clear.sh rename to contrib/test/traft/join_into_vgroup/clear.sh diff --git a/contrib/test/traft/join_into_vgroup/node_follower10000.c b/contrib/test/traft/join_into_vgroup/node_follower10000.c new file mode 100644 index 0000000000..b684ef199c --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/node_follower10000.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // add one replica + r = addRaftSpare(&raftEnv, 100, pFsm); + assert(r == 0); + + // for test: submit value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/join_into_vgroup/node_follower10001.c b/contrib/test/traft/join_into_vgroup/node_follower10001.c new file mode 100644 index 0000000000..27acd33917 --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/node_follower10001.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // add one replica + r = addRaftSpare(&raftEnv, 100, pFsm); + assert(r == 0); + + // for test: submit value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/join_into_vgroup/node_leader10002.c b/contrib/test/traft/join_into_vgroup/node_leader10002.c new file mode 100644 index 0000000000..b57af7777b --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/node_leader10002.c @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +void joinRaftPeerCb(struct raft_change *req, int status) { + struct raft *r = req->data; + if (status != 0) { + fprintf(stderr, "joinRaftPeerCb error: %s \n", raft_errmsg(r)); + } else { + fprintf(stderr, "joinRaftPeerCb ok \n"); + } + raft_free(req); +} + +int main(int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // add one replica + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + printRaftState(getRaft(&raftEnv, 100)); + + // wait for being leader + while (1) { + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + if (r->state == RAFT_LEADER) { + break; + } + } + + // join peers + r = joinRaftPeer(&raftEnv, 100, "127.0.0.1", 10000, joinRaftPeerCb); + assert(r == 0); + + // wait for join over + sleep(2); + + r = joinRaftPeer(&raftEnv, 100, "127.0.0.1", 10001, joinRaftPeerCb); + assert(r == 0); + + // for test: submit value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/make_cluster/CMakeLists.txt b/contrib/test/traft/make_cluster/CMakeLists.txt deleted file mode 100644 index afd19d5435..0000000000 --- a/contrib/test/traft/make_cluster/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_executable(makeCluster "") -target_sources(makeCluster - PRIVATE - "raftMain.c" - "raftServer.c" - "config.c" - "console.c" - "simpleHash.c" - "util.c" -) -target_link_libraries(makeCluster PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/make_cluster/common.h b/contrib/test/traft/make_cluster/common.h deleted file mode 100644 index df7422033a..0000000000 --- a/contrib/test/traft/make_cluster/common.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef TRAFT_COMMON_H -#define TRAFT_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#define COMMAND_LEN 512 -#define MAX_CMD_COUNT 10 -#define TOKEN_LEN 128 -#define MAX_PEERS_COUNT 19 - -#define HOST_LEN 64 -#define ADDRESS_LEN (HOST_LEN * 2) -#define BASE_DIR_LEN 128 - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/make_cluster/config.c b/contrib/test/traft/make_cluster/config.c deleted file mode 100644 index 3b96839fd9..0000000000 --- a/contrib/test/traft/make_cluster/config.c +++ /dev/null @@ -1,64 +0,0 @@ -#include "config.h" -#include -#include -#include - -void addrToString(const char *host, uint16_t port, char *addr, int len) { snprintf(addr, len, "%s:%hu", host, port); } - -void parseAddr(const char *addr, char *host, int len, uint16_t *port) { - char *tmp = (char *)malloc(strlen(addr) + 1); - strcpy(tmp, addr); - - char *context; - char *separator = ":"; - char *token = strtok_r(tmp, separator, &context); - if (token) { - snprintf(host, len, "%s", token); - } - - token = strtok_r(NULL, separator, &context); - if (token) { - sscanf(token, "%hu", port); - } - - free(tmp); -} - -int parseConf(int argc, char **argv, RaftServerConfig *pConf) { - memset(pConf, 0, sizeof(*pConf)); - - int option_index, option_value; - option_index = 0; - static struct option long_options[] = {{"help", no_argument, NULL, 'h'}, - {"addr", required_argument, NULL, 'a'}, - {"dir", required_argument, NULL, 'd'}, - {NULL, 0, NULL, 0}}; - - while ((option_value = getopt_long(argc, argv, "ha:d:", long_options, &option_index)) != -1) { - switch (option_value) { - case 'a': { - parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); - break; - } - - case 'd': { - snprintf(pConf->baseDir, sizeof(pConf->baseDir), "%s", optarg); - break; - } - - case 'h': { - return -2; - } - - default: { return -2; } - } - } - - return 0; -} - -void printConf(RaftServerConfig *pConf) { - printf("\n---printConf: \n"); - printf("me: [%s:%hu] \n", pConf->me.host, pConf->me.port); - printf("dataDir: [%s] \n\n", pConf->baseDir); -} diff --git a/contrib/test/traft/make_cluster/config.h b/contrib/test/traft/make_cluster/config.h deleted file mode 100644 index 13c43d0d28..0000000000 --- a/contrib/test/traft/make_cluster/config.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef TRAFT_CONFIG_H -#define TRAFT_CONFIG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "common.h" - -typedef struct { - char host[HOST_LEN]; - uint16_t port; -} Addr; - -typedef struct { - Addr me; - char baseDir[BASE_DIR_LEN]; -} RaftServerConfig; - -void addrToString(const char *host, uint16_t port, char *addr, int len); -void parseAddr(const char *addr, char *host, int len, uint16_t *port); -int parseConf(int argc, char **argv, RaftServerConfig *pConf); -void printConf(RaftServerConfig *pConf); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/make_cluster/console.c b/contrib/test/traft/make_cluster/console.c deleted file mode 100644 index b00550c681..0000000000 --- a/contrib/test/traft/make_cluster/console.c +++ /dev/null @@ -1,202 +0,0 @@ -#include "console.h" -#include -#include -#include -#include "raftServer.h" -#include "util.h" - -void printHelp() { - printf("---------------------\n"); - printf("help: \n\n"); - printf("create a vgroup with 3 replicas: \n"); - printf("create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 \n"); - printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10002 \n"); - printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10001 \n"); - printf("\n"); - printf("create a vgroup with only one replica: \n"); - printf("create vnode voter vid 200 \n"); - printf("\n"); - printf("add vnode into vgroup: \n"); - printf("create vnode spare vid 100 ---- run at 127.0.0.1:10003\n"); - printf("join vnode vid 100 addr 127.0.0.1:10003 ---- run at leader of vgroup 100\n"); - printf("\n"); - printf("run \n"); - printf("put 0 key value \n"); - printf("get 0 key \n"); - printf("---------------------\n"); -} - -void console(RaftServer *pRaftServer) { - while (1) { - int ret; - char cmdBuf[COMMAND_LEN]; - memset(cmdBuf, 0, sizeof(cmdBuf)); - printf("(console)> "); - char *retp = fgets(cmdBuf, COMMAND_LEN, stdin); - if (!retp) { - exit(-1); - } - - int pos = strlen(cmdBuf); - if (cmdBuf[pos - 1] == '\n') { - cmdBuf[pos - 1] = '\0'; - } - - if (strncmp(cmdBuf, "", COMMAND_LEN) == 0) { - continue; - } - - char cmds[MAX_CMD_COUNT][TOKEN_LEN]; - memset(cmds, 0, sizeof(cmds)); - - int cmdCount; - cmdCount = splitString(cmdBuf, " ", cmds, MAX_CMD_COUNT); - - if (strcmp(cmds[0], "create") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[3], "vid") == 0) { - uint16_t vid; - sscanf(cmds[4], "%hu", &vid); - - if (strcmp(cmds[2], "voter") == 0) { - char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; - memset(peers, 0, sizeof(peers)); - uint32_t peersCount = 0; - - if (strcmp(cmds[5], "peers") == 0 && cmdCount > 6) { - // create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 - for (int i = 6; i < cmdCount; ++i) { - snprintf(peers[i - 6], ADDRESS_LEN, "%s", cmds[i]); - peersCount++; - } - } else { - // create vnode voter vid 200 - } - ret = addRaftVoter(pRaftServer, peers, peersCount, vid); - if (ret == 0) { - printf("create vnode voter ok \n"); - } else { - printf("create vnode voter error \n"); - } - } else if (strcmp(cmds[2], "spare") == 0) { - ret = addRaftSpare(pRaftServer, vid); - if (ret == 0) { - printf("create vnode spare ok \n"); - } else { - printf("create vnode spare error \n"); - } - } else { - printHelp(); - } - - } else if (strcmp(cmds[0], "join") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[2], "vid") == 0 && - strcmp(cmds[4], "addr") == 0 && cmdCount == 6) { - // join vnode vid 100 addr 127.0.0.1:10004 - - char * address = cmds[5]; - char host[64]; - uint16_t port; - parseAddr(address, host, sizeof(host), &port); - - uint16_t vid; - sscanf(cmds[3], "%hu", &vid); - - HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); - if (*pp == NULL) { - printf("vid:%hu not found \n", vid); - break; - } - RaftInstance *pRaftInstance = (*pp)->data; - - uint64_t destRaftId = encodeRaftId(host, port, vid); - - struct raft_change *req = raft_malloc(sizeof(*req)); - RaftJoin * pRaftJoin = raft_malloc(sizeof(*pRaftJoin)); - pRaftJoin->r = &pRaftInstance->raft; - pRaftJoin->joinId = destRaftId; - req->data = pRaftJoin; - ret = raft_add(&pRaftInstance->raft, req, destRaftId, address, raftChangeAddCb); - if (ret != 0) { - printf("raft_add error: %s \n", raft_errmsg(&pRaftInstance->raft)); - } - - } else if (strcmp(cmds[0], "dropnode") == 0) { - } else if (strcmp(cmds[0], "state") == 0) { - pRaftServer->raftInstances.print(&pRaftServer->raftInstances); - for (size_t i = 0; i < pRaftServer->raftInstances.length; ++i) { - HashNode *ptr = pRaftServer->raftInstances.table[i]; - if (ptr != NULL) { - while (ptr != NULL) { - RaftInstance *pRaftInstance = ptr->data; - printf("instance vid:%hu raftId:%llu \n", ptr->vgroupId, pRaftInstance->raftId); - printRaftState(&pRaftInstance->raft); - printf("\n"); - ptr = ptr->next; - } - printf("\n"); - } - } - - } else if (strcmp(cmds[0], "put") == 0 && cmdCount == 4) { - uint16_t vid; - sscanf(cmds[1], "%hu", &vid); - char * key = cmds[2]; - char * value = cmds[3]; - HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); - if (*pp == NULL) { - printf("vid:%hu not found \n", vid); - break; - } - RaftInstance *pRaftInstance = (*pp)->data; - - char *raftValue = malloc(TOKEN_LEN * 2 + 3); - snprintf(raftValue, TOKEN_LEN * 2 + 3, "%s--%s", key, value); - putValue(&pRaftInstance->raft, raftValue); - free(raftValue); - - } else if (strcmp(cmds[0], "run") == 0) { - pthread_t tidRaftServer; - pthread_create(&tidRaftServer, NULL, startServerFunc, pRaftServer); - - } else if (strcmp(cmds[0], "get") == 0 && cmdCount == 3) { - uint16_t vid; - sscanf(cmds[1], "%hu", &vid); - char * key = cmds[2]; - HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); - if (*pp == NULL) { - printf("vid:%hu not found \n", vid); - break; - } - RaftInstance * pRaftInstance = (*pp)->data; - SimpleHash * pKV = pRaftInstance->fsm.data; - SimpleHashNode **ppNode = pKV->find_cstr(pKV, key); - if (*ppNode == NULL) { - printf("key:%s not found \n", key); - } else { - printf("find key:%s value:%s \n", key, (char *)((*ppNode)->data)); - } - - } else if (strcmp(cmds[0], "transfer") == 0) { - } else if (strcmp(cmds[0], "state") == 0) { - } else if (strcmp(cmds[0], "snapshot") == 0) { - } else if (strcmp(cmds[0], "exit") == 0) { - exit(0); - - } else if (strcmp(cmds[0], "quit") == 0) { - exit(0); - - } else if (strcmp(cmds[0], "help") == 0) { - printHelp(); - - } else { - printf("unknown command: %s \n", cmdBuf); - printHelp(); - } - - /* - printf("cmdBuf: [%s] \n", cmdBuf); - printf("cmdCount : %d \n", cmdCount); - for (int i = 0; i < MAX_CMD_COUNT; ++i) { - printf("cmd%d : %s \n", i, cmds[i]); - } - */ - } -} diff --git a/contrib/test/traft/make_cluster/console.h b/contrib/test/traft/make_cluster/console.h deleted file mode 100644 index f9ed12baf5..0000000000 --- a/contrib/test/traft/make_cluster/console.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef TRAFT_CONSOLE_H -#define TRAFT_CONSOLE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "common.h" -#include "raftServer.h" - -void console(RaftServer *pRaftServer); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/make_cluster/raftMain.c b/contrib/test/traft/make_cluster/raftMain.c deleted file mode 100644 index e25636de91..0000000000 --- a/contrib/test/traft/make_cluster/raftMain.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" -#include "config.h" -#include "console.h" -#include "raftServer.h" -#include "simpleHash.h" -#include "util.h" - -const char *exe_name; - -void *startConsoleFunc(void *param) { - RaftServer *pRaftServer = (RaftServer *)param; - console(pRaftServer); - return NULL; -} - -void usage() { - printf("\nusage: \n"); - printf("%s --addr=127.0.0.1:10000 --dir=./data \n", exe_name); - printf("\n"); -} - -RaftServerConfig gConfig; -RaftServer gRaftServer; - -int main(int argc, char **argv) { - srand(time(NULL)); - int32_t ret; - - exe_name = argv[0]; - if (argc < 3) { - usage(); - exit(-1); - } - - ret = parseConf(argc, argv, &gConfig); - if (ret != 0) { - usage(); - exit(-1); - } - printConf(&gConfig); - - if (!dirOK(gConfig.baseDir)) { - ret = mkdir(gConfig.baseDir, 0775); - if (ret != 0) { - fprintf(stderr, "mkdir error, %s \n", gConfig.baseDir); - exit(-1); - } - } - - ret = raftServerInit(&gRaftServer, &gConfig); - if (ret != 0) { - fprintf(stderr, "raftServerInit error \n"); - exit(-1); - } - - /* - pthread_t tidRaftServer; - pthread_create(&tidRaftServer, NULL, startServerFunc, &gRaftServer); - */ - - pthread_t tidConsole; - pthread_create(&tidConsole, NULL, startConsoleFunc, &gRaftServer); - - while (1) { - sleep(10); - } - - return 0; -} diff --git a/contrib/test/traft/make_cluster/raftServer.c b/contrib/test/traft/make_cluster/raftServer.c deleted file mode 100644 index bbf67b9420..0000000000 --- a/contrib/test/traft/make_cluster/raftServer.c +++ /dev/null @@ -1,286 +0,0 @@ -#include "raftServer.h" -#include -#include -#include "common.h" -#include "simpleHash.h" -#include "util.h" - -void *startServerFunc(void *param) { - RaftServer *pRaftServer = (RaftServer *)param; - int32_t r = raftServerStart(pRaftServer); - assert(r == 0); - return NULL; -} - -void raftChangeAssignCb(struct raft_change *req, int status) { - struct raft *r = req->data; - if (status != 0) { - printf("raftChangeAssignCb error: %s \n", raft_errmsg(r)); - } else { - printf("raftChangeAssignCb ok \n"); - } - raft_free(req); -} - -void raftChangeAddCb(struct raft_change *req, int status) { - RaftJoin *pRaftJoin = req->data; - if (status != 0) { - printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r)); - } else { - struct raft_change *req2 = raft_malloc(sizeof(*req2)); - req2->data = pRaftJoin->r; - int ret = raft_assign(pRaftJoin->r, req2, pRaftJoin->joinId, RAFT_VOTER, raftChangeAssignCb); - if (ret != 0) { - printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r)); - } - } - raft_free(req->data); - raft_free(req); -} - -int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { - // get fsm data - SimpleHash *sh = pFsm->data; - - // get commit value - char *msg = (char *)buf->base; - printf("fsm apply: [%s] \n", msg); - char arr[2][TOKEN_LEN]; - int r = splitString(msg, "--", arr, 2); - assert(r == 2); - - // do the value on fsm - sh->insert_cstr(sh, arr[0], arr[1]); - - raft_free(buf->base); - return 0; -} - -void putValueCb(struct raft_apply *req, int status, void *result) { - struct raft *r = req->data; - if (status != 0) { - printf("putValueCb error: %s \n", raft_errmsg(r)); - } else { - printf("putValueCb: %s \n", "ok"); - } - raft_free(req); -} - -void putValue(struct raft *r, const char *value) { - struct raft_buffer buf; - - buf.len = strlen(value) + 1; - buf.base = raft_malloc(buf.len); - snprintf(buf.base, buf.len, "%s", value); - - struct raft_apply *req = raft_malloc(sizeof(*req)); - req->data = r; - int ret = raft_apply(r, req, &buf, 1, putValueCb); - if (ret == 0) { - printf("put %s \n", (char *)buf.base); - } else { - printf("put error: %s \n", raft_errmsg(r)); - } -} - -const char *state2String(unsigned short state) { - if (state == RAFT_UNAVAILABLE) { - return "RAFT_UNAVAILABLE"; - - } else if (state == RAFT_FOLLOWER) { - return "RAFT_FOLLOWER"; - - } else if (state == RAFT_CANDIDATE) { - return "RAFT_CANDIDATE"; - - } else if (state == RAFT_LEADER) { - return "RAFT_LEADER"; - } - return "UNKNOWN_RAFT_STATE"; -} - -void printRaftConfiguration(struct raft_configuration *c) { - printf("configuration: \n"); - for (int i = 0; i < c->n; ++i) { - printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); - } -} - -void printRaftState(struct raft *r) { - printf("----Raft State: -----------\n"); - printf("mem_addr: %p \n", r); - printf("my_id: %llu \n", r->id); - printf("address: %s \n", r->address); - printf("current_term: %llu \n", r->current_term); - printf("voted_for: %llu \n", r->voted_for); - printf("role: %s \n", state2String(r->state)); - printf("commit_index: %llu \n", r->commit_index); - printf("last_applied: %llu \n", r->last_applied); - printf("last_stored: %llu \n", r->last_stored); - - printf("configuration_index: %llu \n", r->configuration_index); - printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); - printRaftConfiguration(&r->configuration); - - printf("----------------------------\n"); -} - -int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid) { - int ret; - - RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance)); - assert(pRaftInstance != NULL); - - // init raftId - pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); - - // init dir - snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir, - pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId); - - if (!dirOK(pRaftInstance->dir)) { - ret = mkdir(pRaftInstance->dir, 0775); - if (ret != 0) { - fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir); - assert(0); - } - } - - // init fsm - pRaftInstance->fsm.data = newSimpleHash(2); - pRaftInstance->fsm.apply = fsmApplyCb; - - // init io - ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport); - if (ret != 0) { - fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft - ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId, - pRaftServer->address); - if (ret != 0) { - fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft_configuration - struct raft_configuration conf; - raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_VOTER); - for (int i = 0; i < peersCount; ++i) { - char * peerAddress = peers[i]; - char host[64]; - uint16_t port; - parseAddr(peerAddress, host, sizeof(host), &port); - uint64_t raftId = encodeRaftId(host, port, vid); - raft_configuration_add(&conf, raftId, peers[i], RAFT_VOTER); - } - raft_bootstrap(&pRaftInstance->raft, &conf); - - // start raft - ret = raft_start(&pRaftInstance->raft); - if (ret != 0) { - fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // add raft instance into raft server - pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance); - - return 0; -} - -int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid) { - int ret; - - RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance)); - assert(pRaftInstance != NULL); - - // init raftId - pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); - - // init dir - snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir, - pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId); - ret = mkdir(pRaftInstance->dir, 0775); - if (ret != 0) { - fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir); - assert(0); - } - - // init fsm - pRaftInstance->fsm.data = newSimpleHash(2); - pRaftInstance->fsm.apply = fsmApplyCb; - - // init io - ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport); - if (ret != 0) { - fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft - ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId, - pRaftServer->address); - if (ret != 0) { - fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft_configuration - struct raft_configuration conf; - raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_SPARE); - raft_bootstrap(&pRaftInstance->raft, &conf); - - // start raft - ret = raft_start(&pRaftInstance->raft); - if (ret != 0) { - fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // add raft instance into raft server - pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance); - - return 0; -} - -int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf) { - int ret; - - // init host, port, address, dir - snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); - pRaftServer->port = pConf->me.port; - snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); - snprintf(pRaftServer->baseDir, sizeof(pRaftServer->baseDir), "%s", pConf->baseDir); - - // init loop - ret = uv_loop_init(&pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); - assert(0); - } - - // init network - ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); - assert(0); - } - - // init raft instance container - initIdHash(&pRaftServer->raftInstances, 2); - - return 0; -} - -int32_t raftServerStart(RaftServer *pRaftServer) { - // start loop - uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); - return 0; -} - -void raftServerStop(RaftServer *pRaftServer) {} diff --git a/contrib/test/traft/make_cluster/raftServer.h b/contrib/test/traft/make_cluster/raftServer.h deleted file mode 100644 index b6dbddb2b7..0000000000 --- a/contrib/test/traft/make_cluster/raftServer.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef TDENGINE_RAFT_SERVER_H -#define TDENGINE_RAFT_SERVER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include "common.h" -#include "config.h" -#include "raft.h" -#include "raft/uv.h" -#include "simpleHash.h" - -typedef struct RaftJoin { - struct raft *r; - raft_id joinId; -} RaftJoin; - -typedef struct { - raft_id raftId; - char dir[BASE_DIR_LEN * 2]; - struct raft_fsm fsm; - struct raft_io io; - struct raft raft; -} RaftInstance; - -typedef struct { - char host[HOST_LEN]; - uint16_t port; - char address[ADDRESS_LEN]; /* Raft instance address */ - char baseDir[BASE_DIR_LEN]; /* Raft instance address */ - - struct uv_loop_s loop; /* UV loop */ - struct raft_uv_transport transport; /* UV I/O backend transport */ - - IdHash raftInstances; /* multi raft instances. traft use IdHash to manager multi vgroup inside, here we can use IdHash - too. */ -} RaftServer; - -void * startServerFunc(void *param); -int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid); -int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid); - -int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf); -int32_t raftServerStart(RaftServer *pRaftServer); -void raftServerStop(RaftServer *pRaftServer); - -int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result); -void putValueCb(struct raft_apply *req, int status, void *result); -void putValue(struct raft *r, const char *value); - -void raftChangeAddCb(struct raft_change *req, int status); - -const char *state2String(unsigned short state); -void printRaftConfiguration(struct raft_configuration *c); -void printRaftState(struct raft *r); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RAFT_SERVER_H diff --git a/contrib/test/traft/make_cluster/simpleHash.c b/contrib/test/traft/make_cluster/simpleHash.c deleted file mode 100644 index 6694843874..0000000000 --- a/contrib/test/traft/make_cluster/simpleHash.c +++ /dev/null @@ -1,218 +0,0 @@ -#include "simpleHash.h" - -uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed) { - // Similar to murmur hash - const uint32_t m = 0xc6a4a793; - const uint32_t r = 24; - const char* limit = data + n; - uint32_t h = seed ^ (n * m); - - // Pick up four bytes at a time - while (data + 4 <= limit) { - // uint32_t w = DecodeFixed32(data); - uint32_t w; - memcpy(&w, data, 4); - - data += 4; - h += w; - h *= m; - h ^= (h >> 16); - } - - // Pick up remaining bytes - switch (limit - data) { - case 3: - h += (unsigned char)(data[2]) << 16; - do { - } while (0); - case 2: - h += (unsigned char)(data[1]) << 8; - do { - } while (0); - case 1: - h += (unsigned char)(data[0]); - h *= m; - h ^= (h >> r); - break; - } - return h; -} - -int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data) { - return insertSimpleHash(ths, key, strlen(key) + 1, data, strlen(data) + 1); -} - -int removeCStrSimpleHash(struct SimpleHash* ths, char* key) { return removeSimpleHash(ths, key, strlen(key) + 1); } - -SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key) { - return findSimpleHash(ths, key, strlen(key) + 1); -} - -int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen) { - SimpleHashNode** pp = ths->find(ths, key, keyLen); - if (*pp != NULL) { - fprintf(stderr, "insertSimpleHash, already has key \n"); - return -1; - } - - SimpleHashNode* node = malloc(sizeof(*node)); - node->hashCode = ths->hashFunc(key, keyLen); - node->key = malloc(keyLen); - node->keyLen = keyLen; - memcpy(node->key, key, keyLen); - node->data = malloc(dataLen); - node->dataLen = dataLen; - memcpy(node->data, data, dataLen); - node->next = NULL; - - // printf("insertSimpleHash: <%s, %ld, %s, %ld, %u> \n", node->key, node->keyLen, node->data, node->dataLen, - // node->hashCode); - - size_t index = node->hashCode & (ths->length - 1); - - SimpleHashNode* ptr = ths->table[index]; - if (ptr != NULL) { - node->next = ptr; - ths->table[index] = node; - - } else { - ths->table[index] = node; - } - ths->elems++; - if (ths->elems > 2 * ths->length) { - ths->resize(ths); - } - - return 0; -} - -int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) { - SimpleHashNode** pp = ths->find(ths, key, keyLen); - if (*pp == NULL) { - fprintf(stderr, "removeSimpleHash, key not exist \n"); - return -1; - } - - SimpleHashNode* del = *pp; - *pp = del->next; - free(del->key); - free(del->data); - free(del); - ths->elems--; - - return 0; -} - -SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) { - uint32_t hashCode = ths->hashFunc(key, keyLen); - // size_t index = hashCode % ths->length; - size_t index = hashCode & (ths->length - 1); - - // printf("findSimpleHash: %s %ld %u \n", key, keyLen, hashCode); - - SimpleHashNode** pp = &(ths->table[index]); - while (*pp != NULL && ((*pp)->hashCode != hashCode || memcmp(key, (*pp)->key, keyLen) != 0)) { - pp = &((*pp)->next); - } - - return pp; -} - -void printCStrSimpleHash(struct SimpleHash* ths) { - printf("\n--- printCStrSimpleHash: elems:%d length:%d \n", ths->elems, ths->length); - for (size_t i = 0; i < ths->length; ++i) { - SimpleHashNode* ptr = ths->table[i]; - if (ptr != NULL) { - printf("%zu: ", i); - while (ptr != NULL) { - printf("<%u, %s, %ld, %s, %ld> ", ptr->hashCode, (char*)ptr->key, ptr->keyLen, (char*)ptr->data, ptr->dataLen); - ptr = ptr->next; - } - printf("\n"); - } - } - printf("---------------\n"); -} - -void destroySimpleHash(struct SimpleHash* ths) { - for (size_t i = 0; i < ths->length; ++i) { - SimpleHashNode* ptr = ths->table[i]; - while (ptr != NULL) { - SimpleHashNode* tmp = ptr; - ptr = ptr->next; - free(tmp->key); - free(tmp->data); - free(tmp); - } - } - - ths->length = 0; - ths->elems = 0; - free(ths->table); - free(ths); -} - -void resizeSimpleHash(struct SimpleHash* ths) { - uint32_t new_length = ths->length; - while (new_length < ths->elems) { - new_length *= 2; - } - - printf("resizeSimpleHash: %p from %u to %u \n", ths, ths->length, new_length); - - SimpleHashNode** new_table = malloc(new_length * sizeof(SimpleHashNode*)); - memset(new_table, 0, new_length * sizeof(SimpleHashNode*)); - - uint32_t count = 0; - for (uint32_t i = 0; i < ths->length; i++) { - if (ths->table[i] == NULL) { - continue; - } - - SimpleHashNode* it = ths->table[i]; - while (it != NULL) { - SimpleHashNode* move_node = it; - it = it->next; - - // move move_node - move_node->next = NULL; - size_t index = move_node->hashCode & (new_length - 1); - - SimpleHashNode* ptr = new_table[index]; - if (ptr != NULL) { - move_node->next = ptr; - new_table[index] = move_node; - } else { - new_table[index] = move_node; - } - count++; - } - } - - assert(ths->elems == count); - free(ths->table); - ths->table = new_table; - ths->length = new_length; -} - -uint32_t simpleHashFunc(const char* key, size_t keyLen) { return mySimpleHash(key, keyLen, 1); } - -struct SimpleHash* newSimpleHash(size_t length) { - struct SimpleHash* ths = malloc(sizeof(*ths)); - - ths->length = length; - ths->elems = 0; - ths->table = malloc(length * sizeof(SimpleHashNode*)); - memset(ths->table, 0, length * sizeof(SimpleHashNode*)); - - ths->insert = insertSimpleHash; - ths->remove = removeSimpleHash; - ths->find = findSimpleHash; - ths->insert_cstr = insertCStrSimpleHash; - ths->remove_cstr = removeCStrSimpleHash; - ths->find_cstr = findCStrSimpleHash; - ths->print_cstr = printCStrSimpleHash; - ths->destroy = destroySimpleHash; - ths->resize = resizeSimpleHash; - ths->hashFunc = simpleHashFunc; -} diff --git a/contrib/test/traft/make_cluster/simpleHash.h b/contrib/test/traft/make_cluster/simpleHash.h deleted file mode 100644 index c6fcd93888..0000000000 --- a/contrib/test/traft/make_cluster/simpleHash.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef __SIMPLE_HASH_H__ -#define __SIMPLE_HASH_H__ - -#include -#include -#include -#include -#include - -uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed); - -typedef struct SimpleHashNode { - uint32_t hashCode; - void* key; - size_t keyLen; - void* data; - size_t dataLen; - struct SimpleHashNode* next; -} SimpleHashNode; - -typedef struct SimpleHash { - // public: - - int (*insert)(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen); - int (*remove)(struct SimpleHash* ths, char* key, size_t keyLen); - SimpleHashNode** (*find)(struct SimpleHash* ths, char* key, size_t keyLen); - - // wrapper - int (*insert_cstr)(struct SimpleHash* ths, char* key, char* data); - int (*remove_cstr)(struct SimpleHash* ths, char* key); - SimpleHashNode** (*find_cstr)(struct SimpleHash* ths, char* key); - - void (*print_cstr)(struct SimpleHash* ths); - void (*destroy)(struct SimpleHash* ths); - - uint32_t length; - uint32_t elems; - - // private: - void (*resize)(struct SimpleHash* ths); - uint32_t (*hashFunc)(const char* key, size_t keyLen); - - SimpleHashNode** table; - -} SimpleHash; - -int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data); -int removeCStrSimpleHash(struct SimpleHash* ths, char* key); -SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key); -void printCStrSimpleHash(struct SimpleHash* ths); - -int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen); -int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen); -SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen); -void destroySimpleHash(struct SimpleHash* ths); -void resizeSimpleHash(struct SimpleHash* ths); -uint32_t simpleHashFunc(const char* key, size_t keyLen); - -struct SimpleHash* newSimpleHash(size_t length); - -#endif diff --git a/contrib/test/traft/make_cluster/util.c b/contrib/test/traft/make_cluster/util.c deleted file mode 100644 index ff704f3660..0000000000 --- a/contrib/test/traft/make_cluster/util.c +++ /dev/null @@ -1,45 +0,0 @@ -#include "util.h" -#include -#include -#include - -int dirOK(const char *path) { - DIR *dir = opendir(path); - if (dir != NULL) { - closedir(dir); - return 1; - } else { - return 0; - } -} - -int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr) { - if (n_arr <= 0) { - return -1; - } - - char *tmp = (char *)malloc(strlen(str) + 1); - strcpy(tmp, str); - char *context; - int n = 0; - - char *token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - strncpy(arr[n], token, TOKEN_LEN); - n++; - - while (1) { - token = strtok_r(NULL, separator, &context); - if (!token || n >= n_arr) { - goto ret; - } - strncpy(arr[n], token, TOKEN_LEN); - n++; - } - -ret: - free(tmp); - return n; -} diff --git a/contrib/test/traft/make_cluster/util.h b/contrib/test/traft/make_cluster/util.h deleted file mode 100644 index fb4ccb9c5c..0000000000 --- a/contrib/test/traft/make_cluster/util.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef TRAFT_UTIL_H -#define TRAFT_UTIL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "common.h" - -int dirOK(const char *path); -int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/rebalance_leader/CMakeLists.txt b/contrib/test/traft/rebalance_leader/CMakeLists.txt deleted file mode 100644 index 92640bdd80..0000000000 --- a/contrib/test/traft/rebalance_leader/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_executable(rebalanceLeader "") -target_sources(rebalanceLeader - PRIVATE - "raftMain.c" - "raftServer.c" -) -target_link_libraries(rebalanceLeader PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/rebalance_leader/common.h b/contrib/test/traft/rebalance_leader/common.h deleted file mode 100644 index 0229c29cf7..0000000000 --- a/contrib/test/traft/rebalance_leader/common.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef TDENGINE_COMMON_H -#define TDENGINE_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#define MAX_INSTANCE_NUM 100 - -#define MAX_PEERS 10 -#define COMMAND_LEN 1024 -#define TOKEN_LEN 128 -#define DIR_LEN 256 -#define HOST_LEN 64 -#define ADDRESS_LEN (HOST_LEN + 16) - -typedef struct { - char host[HOST_LEN]; - uint32_t port; -} Addr; - -typedef struct { - Addr me; - Addr peers[MAX_PEERS]; - int peersCount; - char dir[DIR_LEN]; - char dataDir[DIR_LEN + HOST_LEN * 2]; -} SRaftServerConfig; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_COMMON_H diff --git a/contrib/test/traft/rebalance_leader/raftMain.c b/contrib/test/traft/rebalance_leader/raftMain.c deleted file mode 100644 index 70dc191997..0000000000 --- a/contrib/test/traft/rebalance_leader/raftMain.c +++ /dev/null @@ -1,678 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "raftServer.h" -#include "common.h" - -const char *exe_name; - -typedef struct LeaderState { - char address[48]; - int leaderCount; - -} LeaderState; - -#define NODE_COUNT 3 -LeaderState leaderStates[NODE_COUNT]; - -void printLeaderCount() { - for (int i = 0; i < NODE_COUNT; ++i) { - printf("%s: leaderCount:%d \n", leaderStates[i].address, leaderStates[i].leaderCount); - } -} - -void updateLeaderStates(SRaftServer *pRaftServer) { - for (int i = 0; i < pRaftServer->instance[0].raft.configuration.n; ++i) { - snprintf(leaderStates[i].address, sizeof(leaderStates[i].address), "%s", pRaftServer->instance[0].raft.configuration.servers[i].address); - leaderStates[i].leaderCount = 0; - } - - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - struct raft *r = &pRaftServer->instance[i].raft; - - char leaderAddress[128]; - memset(leaderAddress, 0, sizeof(leaderAddress)); - - if (r->state == RAFT_LEADER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); - } else if (r->state == RAFT_FOLLOWER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); - } - - for (int j = 0; j < NODE_COUNT; j++) { - if (strcmp(leaderAddress, leaderStates[j].address) == 0) { - leaderStates[j].leaderCount++; - } - } - } -} - - -void raftTransferCb(struct raft_transfer *req) { - SRaftServer *pRaftServer = req->data; - raft_free(req); - - //printf("raftTransferCb: \n"); - updateLeaderStates(pRaftServer); - //printLeaderCount(); - - int myLeaderCount; - for (int i = 0; i < NODE_COUNT; ++i) { - if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { - myLeaderCount = leaderStates[i].leaderCount; - } - } - - //printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); - if (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { - struct raft *r; - for (int j = 0; j < pRaftServer->instanceCount; ++j) { - if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { - r = &pRaftServer->instance[j].raft; - break; - } - } - - struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); - transfer->data = pRaftServer; - - uint64_t destRaftId; - int minIndex = -1; - int minLeaderCount = myLeaderCount; - for (int j = 0; j < NODE_COUNT; ++j) { - if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) { - continue; - } - - if (leaderStates[j].leaderCount <= minLeaderCount) { - minLeaderCount = leaderStates[j].leaderCount; - minIndex = j; - } - } - - - char myHost[48]; - uint16_t myPort; - uint16_t myVid; - decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); - - - //printf("raftTransferCb transfer leader: vid[%u] choose: index:%d, leaderStates[%d].address:%s, leaderStates[%d].leaderCount:%d \n", minIndex, minIndex, leaderStates[minIndex].address, minIndex, leaderStates[minIndex].leaderCount); - - char *destAddress = leaderStates[minIndex].address; - - char tokens[MAX_PEERS][MAX_TOKEN_LEN]; - splitString(destAddress, ":", tokens, 2); - char *destHost = tokens[0]; - uint16_t destPort = atoi(tokens[1]); - destRaftId = encodeRaftId(destHost, destPort, myVid); - - printf("\nraftTransferCb transfer leader: vgroupId:%u from:%s:%u --> to:%s:%u ", myVid, myHost, myPort, destHost, destPort); - fflush(stdout); - - raft_transfer(r, transfer, destRaftId, raftTransferCb); - } - -} - - -void parseAddr(const char *addr, char *host, int len, uint32_t *port) { - char* tmp = (char*)malloc(strlen(addr) + 1); - strcpy(tmp, addr); - - char* context; - char* separator = ":"; - char* token = strtok_r(tmp, separator, &context); - if (token) { - snprintf(host, len, "%s", token); - } - - token = strtok_r(NULL, separator, &context); - if (token) { - sscanf(token, "%u", port); - } - - free(tmp); -} - -// only parse 3 tokens -int parseCommand3(const char* str, char* token1, char* token2, char* token3, int len) -{ - char* tmp = (char*)malloc(strlen(str) + 1); - strcpy(tmp, str); - - char* context; - char* separator = " "; - int n = 0; - - char* token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token1, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token2, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token3, token, len); - n++; - } - -ret: - return n; - free(tmp); -} - -// only parse 4 tokens -int parseCommand4(const char* str, char* token1, char* token2, char* token3, char *token4, int len) -{ - char* tmp = (char*)malloc(strlen(str) + 1); - strcpy(tmp, str); - - char* context; - char* separator = " "; - int n = 0; - - char* token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token1, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token2, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token3, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token4, token, len); - n++; - } - -ret: - return n; - free(tmp); -} - -void *startServerFunc(void *param) { - SRaftServer *pServer = (SRaftServer*)param; - int32_t r = raftServerStart(pServer); - assert(r == 0); - - return NULL; -} - -// Console --------------------------------- -const char* state2String(unsigned short state) { - if (state == RAFT_UNAVAILABLE) { - return "RAFT_UNAVAILABLE"; - - } else if (state == RAFT_FOLLOWER) { - return "RAFT_FOLLOWER"; - - } else if (state == RAFT_CANDIDATE) { - return "RAFT_CANDIDATE"; - - } else if (state == RAFT_LEADER) { - return "RAFT_LEADER"; - - } - return "UNKNOWN_RAFT_STATE"; -} - - -void printRaftState2(struct raft *r) { - char leaderAddress[128]; - memset(leaderAddress, 0, sizeof(leaderAddress)); - - if (r->state == RAFT_LEADER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); - } else if (r->state == RAFT_FOLLOWER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); - } - - for (int i = 0; i < r->configuration.n; ++i) { - char tmpAddress[128]; - snprintf(tmpAddress, sizeof(tmpAddress), "%s", r->configuration.servers[i].address); - - uint64_t raftId = r->configuration.servers[i].id; - char host[128]; - uint16_t port; - uint16_t vid; - decodeRaftId(raftId, host, 128, &port, &vid); - - char buf[512]; - memset(buf, 0, sizeof(buf)); - if (strcmp(tmpAddress, leaderAddress) == 0) { - snprintf(buf, sizeof(buf), "<%s:%u-%u-LEADER>\t", host, port, vid); - } else { - snprintf(buf, sizeof(buf), "<%s:%u-%u-FOLLOWER>\t", host, port, vid); - } - printf("%s", buf); - } - printf("\n"); -} - -void printRaftConfiguration(struct raft_configuration *c) { - printf("configuration: \n"); - for (int i = 0; i < c->n; ++i) { - printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); - } -} - -void printRaftState(struct raft *r) { - printf("----Raft State: -----------\n"); - printf("mem_addr: %p \n", r); - printf("my_id: %llu \n", r->id); - printf("address: %s \n", r->address); - printf("current_term: %llu \n", r->current_term); - printf("voted_for: %llu \n", r->voted_for); - printf("role: %s \n", state2String(r->state)); - printf("commit_index: %llu \n", r->commit_index); - printf("last_applied: %llu \n", r->last_applied); - printf("last_stored: %llu \n", r->last_stored); - - printf("configuration_index: %llu \n", r->configuration_index); - printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); - printRaftConfiguration(&r->configuration); - - printf("----------------------------\n"); -} - -void putValueCb(struct raft_apply *req, int status, void *result) { - raft_free(req); - struct raft *r = req->data; - if (status != 0) { - printf("putValueCb: %s \n", raft_errmsg(r)); - } else { - printf("putValueCb: %s \n", "ok"); - } -} - -void putValue(struct raft *r, const char *value) { - struct raft_buffer buf; - - buf.len = TOKEN_LEN;; - buf.base = raft_malloc(buf.len); - snprintf(buf.base, buf.len, "%s", value); - - struct raft_apply *req = raft_malloc(sizeof(struct raft_apply)); - req->data = r; - int ret = raft_apply(r, req, &buf, 1, putValueCb); - if (ret == 0) { - printf("put %s \n", (char*)buf.base); - } else { - printf("put error: %s \n", raft_errmsg(r)); - } -} - -void getValue(const char *key) { - char *ptr = getKV(key); - if (ptr) { - printf("get value: [%s] \n", ptr); - } else { - printf("value not found for key: [%s] \n", key); - } -} - -void console(SRaftServer *pRaftServer) { - while (1) { - char cmd_buf[COMMAND_LEN]; - memset(cmd_buf, 0, sizeof(cmd_buf)); - printf("(console)> "); - char *ret = fgets(cmd_buf, COMMAND_LEN, stdin); - if (!ret) { - exit(-1); - } - - int pos = strlen(cmd_buf); - if(cmd_buf[pos - 1] == '\n') { - cmd_buf[pos - 1] = '\0'; - } - - if (strncmp(cmd_buf, "", COMMAND_LEN) == 0) { - continue; - } - - char cmd[TOKEN_LEN]; - memset(cmd, 0, sizeof(cmd)); - - char param1[TOKEN_LEN]; - memset(param1, 0, sizeof(param1)); - - char param2[TOKEN_LEN]; - memset(param2, 0, sizeof(param2)); - - char param3[TOKEN_LEN]; - memset(param2, 0, sizeof(param2)); - - parseCommand4(cmd_buf, cmd, param1, param2, param3, TOKEN_LEN); - if (strcmp(cmd, "addnode") == 0) { - printf("not support \n"); - - /* - char host[HOST_LEN]; - uint32_t port; - parseAddr(param1, host, HOST_LEN, &port); - uint64_t rid = raftId(host, port); - - struct raft_change *req = raft_malloc(sizeof(*req)); - int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL); - if (r != 0) { - printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft)); - } - printf("add node: %lu %s \n", rid, param1); - - struct raft_change *req2 = raft_malloc(sizeof(*req2)); - r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL); - if (r != 0) { - printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft)); - } - */ - - } else if (strcmp(cmd, "dropnode") == 0) { - printf("not support \n"); - - } else if (strcmp(cmd, "quit") == 0 || strcmp(cmd, "exit") == 0) { - exit(0); - - } else if (strcmp(cmd, "rebalance") == 0 && strcmp(param1, "leader") == 0) { - - /* - updateLeaderStates(pRaftServer); - - int myLeaderCount; - for (int i = 0; i < NODE_COUNT; ++i) { - if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { - myLeaderCount = leaderStates[i].leaderCount; - } - } - - while (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { - printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); - - struct raft *r; - for (int j = 0; j < pRaftServer->instanceCount; ++j) { - if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { - r = &pRaftServer->instance[j].raft; - } - } - - struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); - transfer->data = pRaftServer; - - uint64_t destRaftId; - int minIndex = -1; - int minLeaderCount = myLeaderCount; - for (int j = 0; j < NODE_COUNT; ++j) { - if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; - - printf("-----leaderStates[%d].leaderCount:%d \n", j, leaderStates[j].leaderCount); - if (leaderStates[j].leaderCount <= minLeaderCount) { - minIndex = j; - printf("++++ assign minIndex : %d \n", minIndex); - } - } - - printf("minIndex:%d minLeaderCount:%d \n", minIndex, minLeaderCount); - - char myHost[48]; - uint16_t myPort; - uint16_t myVid; - decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); - - char *destAddress = leaderStates[minIndex].address; - - char tokens[MAX_PEERS][MAX_TOKEN_LEN]; - splitString(destAddress, ":", tokens, 2); - char *destHost = tokens[0]; - uint16_t destPort = atoi(tokens[1]); - destRaftId = encodeRaftId(destHost, destPort, myVid); - - printf("destHost:%s destPort:%u myVid:%u", destHost, destPort, myVid); - raft_transfer(r, transfer, destRaftId, raftTransferCb); - sleep(1); - - for (int i = 0; i < NODE_COUNT; ++i) { - if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { - myLeaderCount = leaderStates[i].leaderCount; - } - } - } - */ - - - int leaderCount = 0; - - struct raft *firstR; - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - struct raft *r = &pRaftServer->instance[i].raft; - if (r->state == RAFT_LEADER) { - leaderCount++; - firstR = r; - } - } - - if (leaderCount > pRaftServer->instanceCount / NODE_COUNT) { - struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); - transfer->data = pRaftServer; - raft_transfer(firstR, transfer, 0, raftTransferCb); - } - - - } else if (strcmp(cmd, "put") == 0) { - char buf[256]; - uint16_t vid; - sscanf(param1, "%hu", &vid); - snprintf(buf, sizeof(buf), "%s--%s", param2, param3); - putValue(&pRaftServer->instance[vid].raft, buf); - - } else if (strcmp(cmd, "get") == 0) { - getValue(param1); - - } else if (strcmp(cmd, "transfer") == 0) { - uint16_t vid; - sscanf(param1, "%hu", &vid); - - struct raft_transfer transfer; - raft_transfer(&pRaftServer->instance[vid].raft, &transfer, 0, NULL); - - - } else if (strcmp(cmd, "state") == 0) { - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - printf("instance %d: ", i); - printRaftState(&pRaftServer->instance[i].raft); - } - - } else if (strcmp(cmd, "leader") == 0 && strcmp(param1, "state") == 0) { - updateLeaderStates(pRaftServer); - printf("\n--------------------------------------------\n"); - printLeaderCount(); - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - printRaftState2(&pRaftServer->instance[i].raft); - } - printf("--------------------------------------------\n"); - - } else if (strcmp(cmd, "snapshot") == 0) { - printf("not support \n"); - - } else if (strcmp(cmd, "help") == 0) { - printf("addnode \"127.0.0.1:8888\" \n"); - printf("dropnode \"127.0.0.1:8888\" \n"); - printf("put key value \n"); - printf("get key \n"); - printf("state \n"); - - } else { - printf("unknown command: [%s], type \"help\" to see help \n", cmd); - } - - //printf("cmd_buf: [%s] \n", cmd_buf); - } -} - -void *startConsoleFunc(void *param) { - SRaftServer *pServer = (SRaftServer*)param; - console(pServer); - return NULL; -} - -// Config --------------------------------- -void usage() { - printf("\nusage: \n"); - printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name); - printf("\n"); - printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name); - printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name); - printf("%s --me=127.0.0.1:10002 --peers=127.0.0.1:10000,127.0.0.1:10001 --dir=./data \n", exe_name); - printf("\n"); -} - -void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { - memset(pConf, 0, sizeof(*pConf)); - - int option_index, option_value; - option_index = 0; - static struct option long_options[] = { - {"help", no_argument, NULL, 'h'}, - {"peers", required_argument, NULL, 'p'}, - {"me", required_argument, NULL, 'm'}, - {"dir", required_argument, NULL, 'd'}, - {NULL, 0, NULL, 0} - }; - - while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) { - switch (option_value) { - case 'm': { - parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); - break; - } - - case 'p': { - char tokens[MAX_PEERS][MAX_TOKEN_LEN]; - int peerCount = splitString(optarg, ",", tokens, MAX_PEERS); - pConf->peersCount = peerCount; - for (int i = 0; i < peerCount; ++i) { - Addr *pAddr = &pConf->peers[i]; - parseAddr(tokens[i], pAddr->host, sizeof(pAddr->host), &pAddr->port); - } - break; - } - - - case 'd': { - snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg); - break; - } - - case 'h': { - usage(); - exit(-1); - } - - default: { - usage(); - exit(-1); - } - } - } - snprintf(pConf->dataDir, sizeof(pConf->dataDir), "%s/%s_%u", pConf->dir, pConf->me.host, pConf->me.port); -} - -void printConf(SRaftServerConfig *pConf) { - printf("\nconf: \n"); - printf("me: %s:%u \n", pConf->me.host, pConf->me.port); - printf("peersCount: %d \n", pConf->peersCount); - for (int i = 0; i < pConf->peersCount; ++i) { - Addr *pAddr = &pConf->peers[i]; - printf("peer%d: %s:%u \n", i, pAddr->host, pAddr->port); - } - printf("dataDir: %s \n\n", pConf->dataDir); - -} - - -int main(int argc, char **argv) { - srand(time(NULL)); - int32_t ret; - - exe_name = argv[0]; - if (argc < 3) { - usage(); - exit(-1); - } - - SRaftServerConfig conf; - parseConf(argc, argv, &conf); - printConf(&conf); - - signal(SIGPIPE, SIG_IGN); - - /* - char cmd_buf[COMMAND_LEN]; - snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", conf.dataDir); - system(cmd_buf); - */ - - - struct raft_fsm fsm; - initFsm(&fsm); - - SRaftServer raftServer; - ret = raftServerInit(&raftServer, &conf, &fsm); - assert(ret == 0); - - pthread_t tidRaftServer; - pthread_create(&tidRaftServer, NULL, startServerFunc, &raftServer); - - pthread_t tidConsole; - pthread_create(&tidConsole, NULL, startConsoleFunc, &raftServer); - - while (1) { - sleep(10); - } - - return 0; -} diff --git a/contrib/test/traft/rebalance_leader/raftServer.c b/contrib/test/traft/rebalance_leader/raftServer.c deleted file mode 100644 index 165d3c9023..0000000000 --- a/contrib/test/traft/rebalance_leader/raftServer.c +++ /dev/null @@ -1,224 +0,0 @@ -#include -#include -#include "common.h" -#include "raftServer.h" - -//char *keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);; -//char *values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); - - -char keys[MAX_KV_LEN][MAX_RECORD_COUNT]; -char values[MAX_KV_LEN][MAX_RECORD_COUNT]; -int writeIndex = 0; - -void initStore() { -} - -void destroyStore() { - //free(keys); - //free(values); -} - -void putKV(const char *key, const char *value) { - if (writeIndex < MAX_RECORD_COUNT) { - strncpy(keys[writeIndex], key, MAX_KV_LEN); - strncpy(values[writeIndex], value, MAX_KV_LEN); - writeIndex++; - } -} - -char *getKV(const char *key) { - for (int i = 0; i < MAX_RECORD_COUNT; ++i) { - if (strcmp(keys[i], key) == 0) { - return values[i]; - } - } - return NULL; -} - - -int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr) -{ - if (n_arr <= 0) { - return -1; - } - - char* tmp = (char*)malloc(strlen(str) + 1); - strcpy(tmp, str); - char* context; - int n = 0; - - char* token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - strncpy(arr[n], token, MAX_TOKEN_LEN); - n++; - - while (1) { - token = strtok_r(NULL, separator, &context); - if (!token || n >= n_arr) { - goto ret; - } - strncpy(arr[n], token, MAX_TOKEN_LEN); - n++; - } - -ret: - free(tmp); - return n; -} - -/* -uint64_t raftId(const char *host, uint32_t port) { - uint32_t host_uint32 = (uint32_t)inet_addr(host); - assert(host_uint32 != (uint32_t)-1); - uint64_t code = ((uint64_t)host_uint32) << 32 | port; - return code; -} -*/ - - -/* -uint64_t encodeRaftId(const char *host, uint16_t port, uint16_t vid) { - uint64_t raftId; - uint32_t host_uint32 = (uint32_t)inet_addr(host); - assert(host_uint32 != (uint32_t)-1); - - raftId = (((uint64_t)host_uint32) << 32) | (((uint32_t)port) << 16) | vid; - return raftId; -} - -void decodeRaftId(uint64_t raftId, char *host, int32_t len, uint16_t *port, uint16_t *vid) { - uint32_t host32 = (uint32_t)((raftId >> 32) & 0x00000000FFFFFFFF); - - struct in_addr addr; - addr.s_addr = host32; - snprintf(host, len, "%s", inet_ntoa(addr)); - - *port = (uint16_t)((raftId & 0x00000000FFFF0000) >> 16); - *vid = (uint16_t)(raftId & 0x000000000000FFFF); -} -*/ - - - - -int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm) { - int ret; - - snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); - pRaftServer->port = pConf->me.port; - snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); - //strncpy(pRaftServer->dir, pConf->dataDir, sizeof(pRaftServer->dir)); - - ret = uv_loop_init(&pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); - assert(0); - } - - ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); - assert(0); - } - - - uint16_t vid; - pRaftServer->instanceCount = 20; - - - for (int i = 0; i < pRaftServer->instanceCount; ++i) - { - //vid = 0; - vid = i; - - - pRaftServer->instance[vid].raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); - snprintf(pRaftServer->instance[vid].dir, sizeof(pRaftServer->instance[vid].dir), "%s_%llu", pConf->dataDir, pRaftServer->instance[vid].raftId); - - char cmd_buf[COMMAND_LEN]; - snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", pRaftServer->instance[vid].dir); - system(cmd_buf); - sleep(1); - - pRaftServer->instance[vid].fsm = pFsm; - - ret = raft_uv_init(&pRaftServer->instance[vid].io, &pRaftServer->loop, pRaftServer->instance[vid].dir, &pRaftServer->transport); - if (ret != 0) { - fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); - assert(0); - } - - ret = raft_init(&pRaftServer->instance[vid].raft, &pRaftServer->instance[vid].io, pRaftServer->instance[vid].fsm, pRaftServer->instance[vid].raftId, pRaftServer->address); - if (ret != 0) { - fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); - assert(0); - } - - struct raft_configuration conf; - raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftServer->instance[vid].raftId, pRaftServer->address, RAFT_VOTER); - printf("add myself: %llu - %s \n", pRaftServer->instance[vid].raftId, pRaftServer->address); - for (int i = 0; i < pConf->peersCount; ++i) { - const Addr *pAddr = &pConf->peers[i]; - - raft_id rid = encodeRaftId(pAddr->host, pAddr->port, vid); - - char addrBuf[ADDRESS_LEN]; - snprintf(addrBuf, sizeof(addrBuf), "%s:%u", pAddr->host, pAddr->port); - raft_configuration_add(&conf, rid, addrBuf, RAFT_VOTER); - printf("add peers: %llu - %s \n", rid, addrBuf); - } - - raft_bootstrap(&pRaftServer->instance[vid].raft, &conf); - - } - - - - - - - - return 0; -} - -int32_t raftServerStart(SRaftServer *pRaftServer) { - int ret; - - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - ret = raft_start(&pRaftServer->instance[i].raft); - if (ret != 0) { - fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[i].raft)); - } - - } - - - uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); -} - - -void raftServerClose(SRaftServer *pRaftServer) { - -} - - -int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { - char *msg = (char*)buf->base; - printf("fsm apply: %s \n", msg); - - char arr[2][MAX_TOKEN_LEN]; - splitString(msg, "--", arr, 2); - putKV(arr[0], arr[1]); - - return 0; -} - -int32_t initFsm(struct raft_fsm *fsm) { - initStore(); - fsm->apply = fsmApplyCb; - return 0; -} diff --git a/contrib/test/traft/rebalance_leader/raftServer.h b/contrib/test/traft/rebalance_leader/raftServer.h deleted file mode 100644 index 5ea43985c9..0000000000 --- a/contrib/test/traft/rebalance_leader/raftServer.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef TDENGINE_RAFT_SERVER_H -#define TDENGINE_RAFT_SERVER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include "raft.h" -#include "raft/uv.h" -#include "common.h" - - -// simulate a db store, just for test -#define MAX_KV_LEN 20 -#define MAX_RECORD_COUNT 16 - - -//char *keys; -//char *values; -//int writeIndex; - -void initStore(); -void destroyStore(); -void putKV(const char *key, const char *value); -char *getKV(const char *key); - -typedef struct { - char dir[DIR_LEN + HOST_LEN * 4]; /* Data dir of UV I/O backend */ - raft_id raftId; /* For vote */ - struct raft_fsm *fsm; /* Sample application FSM */ - struct raft raft; /* Raft instance */ - struct raft_io io; /* UV I/O backend */ - -} SInstance; - -typedef struct { - char host[HOST_LEN]; - uint32_t port; - char address[ADDRESS_LEN]; /* Raft instance address */ - - struct uv_loop_s loop; /* UV loop */ - struct raft_uv_transport transport; /* UV I/O backend transport */ - - SInstance instance[MAX_INSTANCE_NUM]; - int32_t instanceCount; - -} SRaftServer; - -#define MAX_TOKEN_LEN 32 -int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr); - -int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm); -int32_t raftServerStart(SRaftServer *pRaftServer); -void raftServerClose(SRaftServer *pRaftServer); - - -int initFsm(struct raft_fsm *fsm); - - - - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RAFT_SERVER_H diff --git a/contrib/test/traft/single_node/CMakeLists.txt b/contrib/test/traft/single_node/CMakeLists.txt new file mode 100644 index 0000000000..666ce271b8 --- /dev/null +++ b/contrib/test/traft/single_node/CMakeLists.txt @@ -0,0 +1,6 @@ +add_executable(singleNode "") +target_sources(singleNode + PRIVATE + "singleNode.c" +) +target_link_libraries(singleNode PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/single_node/clear.sh b/contrib/test/traft/single_node/clear.sh new file mode 100644 index 0000000000..398b3088f2 --- /dev/null +++ b/contrib/test/traft/single_node/clear.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -rf 127.0.0.1* +rm -rf ./data diff --git a/contrib/test/traft/single_node/cmd b/contrib/test/traft/single_node/cmd new file mode 100644 index 0000000000..ff8a156f64 --- /dev/null +++ b/contrib/test/traft/single_node/cmd @@ -0,0 +1,6 @@ +all: + gcc singleNode.c -I ../../include/ ../../.libs/libraft.a -o singleNode -luv -llz4 -lpthread -g +clean: + rm -f singleNode + sh clear.sh + diff --git a/contrib/test/traft/single_node/singleNode.c b/contrib/test/traft/single_node/singleNode.c new file mode 100644 index 0000000000..d22af30e45 --- /dev/null +++ b/contrib/test/traft/single_node/singleNode.c @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +//#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 38000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has one replica + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} From a1c6c94c0b3558b99a06965d2fd40e4a47fc3ef7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 19:00:09 +0800 Subject: [PATCH 049/118] consume skip ununsed table --- include/common/tmsg.h | 4 +- source/dnode/vnode/inc/vnode.h | 11 ++- source/dnode/vnode/src/tq/tq.c | 114 +++++++++++++++++----------- source/libs/executor/src/executor.c | 1 + 4 files changed, 82 insertions(+), 48 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 62f55609ce..bb53c6ddfa 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1590,8 +1590,8 @@ typedef struct SMqCVConsumeReq { typedef struct SMqConsumeRspBlock { int32_t bodyLen; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char body[]; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char body[]; } SMqConsumeRspBlock; typedef struct SMqCVConsumeRsp { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index bb0ee8dfc4..ab538ff12d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,12 +68,13 @@ typedef struct { typedef struct STqReadHandle { int64_t ver; + int64_t tbUid; SSubmitMsg* pMsg; SSubmitBlk* pBlock; SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; SMeta* pMeta; - SArray* pColumnIdList; + SArray* pColIdList; } STqReadHandle; /* ------------------------ SVnode ------------------------ */ @@ -199,8 +200,12 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta); -static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColumnIdList) { - pReadHandle->pColumnIdList = pColumnIdList; +static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { + pReadHandle->pColIdList = pColIdList; +} + +static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, int64_t tbUid) { + pHandle->tbUid = tbUid; } void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index eca02c867c..4a6f55564c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,6 +15,7 @@ #include "tqInt.h" #include "tqMetaStore.h" +#include "tcompare.h" // static // read next version data @@ -424,7 +425,7 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ /*if (code < 0) {*/ - // TODO: error + // TODO: error /*}*/ // get msgType // if submitblk @@ -610,61 +611,91 @@ int tqItemSSize() { int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SMqCVConsumeReq* pReq = pMsg->pCont; + SRpcMsg rpcMsg; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; int64_t reqOffset = pReq->offset; int64_t fetchOffset = reqOffset; int64_t blockingTime = pReq->blockingTime; + int rspLen = 0; + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - int sz = taosArrayGetSize(pConsumer->topics); + int sz = taosArrayGetSize(pConsumer->topics); - for (int i = 0 ; i < sz; i++) { - STqTopicHandle *pTopic = taosArrayGet(pConsumer->topics, i); + for (int i = 0; i < sz; i++) { + STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); - int8_t pos = fetchOffset % TQ_BUFFER_SIZE; - int8_t old = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1); - if (old == 1) { - // do nothing - continue; - } - if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { - return -1; - } - SWalHead* pHead = pTopic->pReadhandle->pHead; + int8_t pos; + int8_t skip = 0; + SWalHead* pHead; while (1) { - // read until find TDMT_VND_SUBMIT - if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { - return -1; + pos = fetchOffset % TQ_BUFFER_SIZE; + skip = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1); + if (skip == 1) { + // do nothing + break; } + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + // check err + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + skip = 1; + break; + } + // read until find TDMT_VND_SUBMIT + pHead = pTopic->pReadhandle->pHead; + if (pHead->head.msgType == TDMT_VND_SUBMIT) { + break; + } + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + skip = 1; + break; + } + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + fetchOffset++; } + if (skip == 1) continue; SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; - void* task = pTopic->buffer.output[pos].task; + qTaskInfo_t task = pTopic->buffer.output[pos].task; qSetStreamInput(task, pCont); - SSDataBlock* pDataBlock; - uint64_t ts; - if (qExecTask(task, &pDataBlock, &ts) < 0) { + //SArray + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + while (1) { + SSDataBlock* pDataBlock; + uint64_t ts; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + break; + } + if (pDataBlock != NULL) { + taosArrayPush(pRes, pDataBlock); + } else { + break; + } } - // TODO: launch query and get output data - pTopic->buffer.output[pos].dst = pDataBlock; - if (pTopic->buffer.firstOffset == -1 - || pReq->offset < pTopic->buffer.firstOffset) { + + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + + if (taosArrayGetSize(pRes) == 0) { + taosArrayDestroy(pRes); + fetchOffset++; + continue; + } + + pTopic->buffer.output[pos].dst = pRes; + if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) { pTopic->buffer.firstOffset = pReq->offset; } - if (pTopic->buffer.lastOffset == -1 - || pReq->offset > pTopic->buffer.lastOffset) { + if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) { pTopic->buffer.lastOffset = pReq->offset; } - atomic_store_8(&pTopic->buffer.output[pos].status, 1); - // put output into rsp } // launch query // get result - SMqCvConsumeRsp* pRsp; return 0; } @@ -673,14 +704,14 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { if (pConsumer == NULL) { return -1; } - + STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); if (pTopic == NULL) { free(pConsumer); return -1; } - strcpy(pTopic->topicName, pReq->topicName); - strcpy(pTopic->cgroup, pReq->cgroup); + strcpy(pTopic->topicName, pReq->topicName); + strcpy(pTopic->cgroup, pReq->cgroup); strcpy(pTopic->sql, pReq->sql); strcpy(pTopic->logicalPlan, pReq->logicalPlan); strcpy(pTopic->physicalPlan, pReq->physicalPlan); @@ -689,7 +720,6 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { pTopic->buffer.lastOffset = -1; pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); if (pTopic->pReadhandle == NULL) { - } for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; @@ -708,7 +738,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->pMeta = pMeta; pReadHandle->pMsg = NULL; pReadHandle->ver = -1; - pReadHandle->pColumnIdList = NULL; + pReadHandle->pColIdList = NULL; return NULL; } @@ -720,20 +750,18 @@ void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ve } bool tqNextDataBlock(STqReadHandle* pHandle) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; + while (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) >= 0) { + if (pHandle->tbUid == pHandle->pBlock->uid) return true; } - return true; + return false; } int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - SMemRow row; - int32_t sversion = pHandle->pBlock->sversion; - SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false); - pBlockInfo->numOfCols = pSchema->nCols; + /*int32_t sversion = pHandle->pBlock->sversion;*/ + /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ + pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); pBlockInfo->rows = pHandle->pBlock->numOfRows; pBlockInfo->uid = pHandle->pBlock->uid; - // TODO: filter out unused column return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b6683e6043..b4aa506df3 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -29,6 +29,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input) { return doSetStreamBlock(pOperator->pDownstream[0], input); } + return TSDB_CODE_QRY_APP_ERROR; } else { SStreamBlockScanInfo* pInfo = pOperator->info; tqReadHandleSetMsg(pInfo->readerHandle, input, 0); From ff0200ae2885cf4789b2a9b9a3af2025dd5db1d8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 21 Jan 2022 19:52:01 +0800 Subject: [PATCH 050/118] feature/qnode --- include/libs/scheduler/scheduler.h | 8 +- include/util/taoserror.h | 2 +- source/client/src/clientImpl.c | 8 +- source/dnode/mgmt/impl/src/dndTransport.c | 1 + source/dnode/mgmt/impl/src/dndVnodes.c | 2 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 4 +- source/libs/qworker/inc/qworkerInt.h | 21 +- source/libs/qworker/inc/qworkerMsg.h | 1 + source/libs/qworker/src/qworker.c | 542 +++++++++--------- source/libs/qworker/src/qworkerMsg.c | 9 +- source/libs/qworker/test/qworkerTests.cpp | 321 ++++++++++- source/libs/scheduler/src/scheduler.c | 49 +- source/libs/scheduler/test/schedulerTests.cpp | 20 +- source/util/src/terror.c | 2 +- 14 files changed, 646 insertions(+), 344 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 3262b9437c..2eb768713e 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -80,7 +80,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); /** * Fetch query result from the remote query executor @@ -88,7 +88,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, * @param data * @return */ -int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); +int32_t schedulerFetchRows(struct SSchJob *pJob, void **data); /** @@ -102,7 +102,7 @@ int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); * Free the query job * @param pJob */ -void scheduleFreeJob(void *pJob); +void schedulerFreeJob(void *pJob); void schedulerDestroy(void); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 570c1d8375..8c048690c0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -354,7 +354,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) //"Scheduler not exist") #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) //"Task not exist") #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) //"Task already exist") -#define TSDB_CODE_QRY_RES_CACHE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task result cache not exist") +#define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task context not exist") #define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) //"Task cancelled") #define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) //"Task dropped") #define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) //"Task cancelling") diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 34ab1fb05a..1726a066d3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -242,12 +242,12 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = scheduleExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); + int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { if (pRequest->body.pQueryJob != NULL) { - scheduleFreeJob(pRequest->body.pQueryJob); + schedulerFreeJob(pRequest->body.pQueryJob); } } @@ -263,7 +263,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { strcpy(addr.epAddr[0].fqdn, "localhost"); taosArrayPush(execNode, &addr); - return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob); + return schedulerAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob); } typedef struct tmq_t tmq_t; @@ -714,7 +714,7 @@ void* doFetchRow(SRequestObj* pRequest) { return NULL; } - int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + int32_t code = schedulerFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; return NULL; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index f4fda75bd8..63ed87c0a3 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -116,6 +116,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { // Requests handled by VNODE pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 0d4b9c803d..d4ff853e85 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -892,7 +892,7 @@ int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) return -1; - int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, false); + int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, false); dndReleaseVnode(pDnode, pVnode); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index dd1e5ba9ae..79ec7fda26 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -22,7 +22,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vTrace("query message is processing"); + vTrace("message in query queue is processing"); switch (pMsg->msgType) { case TDMT_VND_QUERY: @@ -36,7 +36,7 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vTrace("fetch message is processed"); + vTrace("message in fetch queue is processing"); switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 9a33268472..9ecce3f5f9 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -22,17 +22,17 @@ extern "C" { #include "tlockfree.h" -#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 -#define QWORKER_DEFAULT_TASK_NUMBER 10000 -#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 - +#define QW_DEFAULT_SCHEDULER_NUMBER 10000 +#define QW_DEFAULT_TASK_NUMBER 10000 +#define QW_DEFAULT_SCH_TASK_NUMBER 10000 +#define QW_DEFAULT_SHORT_RUN_TIMES 2 enum { QW_PHASE_PRE_QUERY = 1, QW_PHASE_POST_QUERY, - QW_PHASE_PRE_CQUERY, - QW_PHASE_POST_CQUERY, QW_PHASE_PRE_FETCH, QW_PHASE_POST_FETCH, + QW_PHASE_PRE_CQUERY, + QW_PHASE_POST_CQUERY, }; enum { @@ -133,7 +133,7 @@ typedef struct SQWorkerMgmt { int8_t nodeType; int32_t nodeId; SRWLatch schLock; - SRWLatch ctxLock; + //SRWLatch ctxLock; SHashObj *schHash; //key: schedulerId, value: SQWSchStatus SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx void *nodeObj; @@ -144,6 +144,8 @@ typedef struct SQWorkerMgmt { #define QW_IDS() sId, qId, tId #define QW_FPARAMS() mgmt, QW_IDS() +#define QW_GET_EVENT_VALUE(ctx, event) atomic_load_8(&(ctx)->events[event]) + #define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED) #define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED) #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) @@ -151,9 +153,10 @@ typedef struct SQWorkerMgmt { #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) -#define QW_SET_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) +#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) +#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) -#define QW_IN_EXECUTOR(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_FETCH) +#define QW_IS_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index 7ecc2b2b20..7735e1a1ee 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -30,6 +30,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwBuildAndSendDropRsp(void *connection, int32_t code); +int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len); int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index cedb3fa926..bcfbafd4c9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -11,7 +11,7 @@ SQWDebug gQWDebug = {0}; -int32_t qwValidateStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t oriStatus, int8_t newStatus) { +int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { int32_t code = 0; if (oriStatus == newStatus) { @@ -35,6 +35,7 @@ int32_t qwValidateStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ break; case JOB_TASK_STATUS_EXECUTING: if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED @@ -77,7 +78,7 @@ _return: QW_RET(code); } -int32_t qwSetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskStatus *task, int8_t status) { +int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { int32_t code = 0; int8_t origStatus = 0; @@ -99,7 +100,7 @@ int32_t qwSetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t } -int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAddSchedulerImpl(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { @@ -125,7 +126,7 @@ int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } -int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { while (true) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); @@ -152,11 +153,11 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, u return TSDB_CODE_SUCCESS; } -int32_t qwAcquireAddScheduler(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireAddScheduler(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(QW_FPARAMS(), rwType, sch, QW_NOT_EXIST_ADD); } -int32_t qwAcquireScheduler(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireScheduler(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(QW_FPARAMS(), rwType, sch, QW_NOT_EXIST_RET_ERR); } @@ -165,7 +166,7 @@ void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { } -int32_t qwAcquireTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { +int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -181,7 +182,7 @@ int32_t qwAcquireTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint -int32_t qwAddTaskStatusImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { +int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { int32_t code = 0; char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -215,7 +216,7 @@ int32_t qwAddTaskStatusImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status) { +int32_t qwAddTaskStatus(QW_FPARAMS_DEF, int32_t status) { SQWSchStatus *tsch = NULL; int32_t code = 0; QW_ERR_RET(qwAcquireAddScheduler(QW_FPARAMS(), QW_READ, &tsch)); @@ -230,7 +231,7 @@ _return: } -int32_t qwAddAcquireTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus *sch, int32_t status, SQWTaskStatus **task) { +int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, int32_t status, SQWTaskStatus **task) { return qwAddTaskStatusImpl(QW_FPARAMS(), sch, rwType, status, task); } @@ -240,7 +241,7 @@ void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { } -int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { +int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -249,26 +250,26 @@ int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ if (NULL == (*ctx)) { //QW_UNLOCK(rwType, &mgmt->ctxLock); QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } return TSDB_CODE_SUCCESS; } -int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { +int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { QW_TASK_ELOG("ctx not in ctxHash, ctxHashSize:%d", taosHashGetSize(mgmt->ctxHash)); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool acquire, int32_t status, SQWTaskCtx **ctx) { +int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, int32_t status, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -281,7 +282,7 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ if (HASH_NODE_EXIST(code)) { if (acquire && ctx) { - QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } else { @@ -296,7 +297,7 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); if (acquire && ctx) { - QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } @@ -304,17 +305,17 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, NULL)); } -int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { +int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTaskCtxImpl(QW_FPARAMS(), true, 0, ctx); } -int32_t qwAddGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { +int32_t qwAddGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, ctx); } @@ -356,14 +357,14 @@ void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { // Note: NEED CTX HASH LOCKED BEFORE ENTRANCE -int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType) { +int32_t qwDropTaskCtx(QW_FPARAMS_DEF, int32_t rwType) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx octx; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == ctx) { - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } octx = *ctx; @@ -371,13 +372,15 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t atomic_store_ptr(&ctx->taskHandle, NULL); atomic_store_ptr(&ctx->sinkHandle, NULL); + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); + if (rwType) { QW_UNLOCK(rwType, &ctx->lock); } if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } if (octx.taskHandle) { @@ -394,7 +397,7 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t } -int32_t qwDropTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -429,7 +432,7 @@ _return: QW_RET(code); } -int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status) { +int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -447,12 +450,13 @@ _return: QW_RET(code); } -int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType, bool execOnce) { +int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType, bool shortRun) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; uint64_t useconds = 0; int32_t i = 0; + int32_t leftRun = QW_DEFAULT_SHORT_RUN_TIMES; while (true) { QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); @@ -484,7 +488,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); - if (execOnce || (!qcontinue)) { + if (!qcontinue) { + break; + } + + if (shortRun && ((--leftRun) <= 0)) { break; } } @@ -576,29 +584,35 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void return TSDB_CODE_SUCCESS; } - -int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { +int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; int8_t status = 0; SQWTaskCtx *ctx = NULL; bool locked = false; - void *readyConnection = NULL; void *dropConnection = NULL; void *cancelConnection = NULL; QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); + output->needStop = false; + + if (QW_PHASE_PRE_QUERY == phase) { + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); + } else { + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + } + + QW_LOCK(QW_WRITE, &ctx->lock); + locked = true; + + atomic_store_32(&ctx->phase, phase); + switch (phase) { case QW_PHASE_PRE_QUERY: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - atomic_store_32(&ctx->phase, phase); atomic_store_8(&ctx->taskType, input->taskType); - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_ELOG("task already cancelled at wrong phase, phase:%d", phase); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_ELOG("task already cancelled/dropped at wrong phase, phase:%d", phase); output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; @@ -611,18 +625,27 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_SET_RSP_CODE(ctx, output->rspCode); dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it locked = false; } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - output->needStop = true; - QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - + output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_SET_RSP_CODE(ctx, output->rspCode); + + cancelConnection = ctx->cancelConnection; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed at wrong phase, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); } if (!output->needStop) { @@ -630,64 +653,55 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } break; } - case QW_PHASE_POST_QUERY: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - ctx->taskHandle = input->taskHandle; - ctx->sinkHandle = input->sinkHandle; - - if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { - ctx->emptyRes = true; + case QW_PHASE_PRE_FETCH: { + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropped, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - - if (input->code) { - output->rspCode = input->code; + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); output->needStop = true; - - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); output->needStop = true; - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - qwFreeTask(QW_FPARAMS(), ctx); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { - readyConnection = ctx->readyConnection; - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } - if (!output->needStop) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->taskStatus)); + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + QW_TASK_WLOG("last fetch not finished, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_DUPLICATTED_OPERATION; + QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); + } + + if (!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_READY)) { + QW_TASK_ELOG("query rsp are not ready, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_MSG_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); } break; - } - case QW_PHASE_PRE_FETCH: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - atomic_store_32(&ctx->phase, phase); - + } + case QW_PHASE_PRE_CQUERY: { if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); output->needStop = true; @@ -714,20 +728,6 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - QW_TASK_WLOG("last fetch not finished, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_DUPLICATTED_OPERATION; - QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); - } - - if (!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_READY)) { - QW_TASK_ELOG("query rsp are not ready, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_MSG_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); - } - if (ctx->rspCode) { QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); output->needStop = true; @@ -736,160 +736,156 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } break; } - case QW_PHASE_POST_FETCH: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (input->code) { - output->rspCode = input->code; - } - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("start to drop task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("start to cancel task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - cancelConnection = ctx->cancelConnection; - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } - case QW_PHASE_PRE_CQUERY: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - atomic_store_32(&ctx->phase, phase); - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } - case QW_PHASE_POST_CQUERY: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (input->code) { - output->rspCode = input->code; - } - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("start to drop task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("start to cancel task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - cancelConnection = ctx->cancelConnection; - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } } _return: - if (output->rspCode) { - QW_SET_RSP_CODE(ctx, output->rspCode); + if (ctx) { + if (output->rspCode) { + QW_UPDATE_RSP_CODE(ctx, output->rspCode); + } + + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + qwReleaseTaskCtx(mgmt, ctx); } - if (locked) { - atomic_store_32(&ctx->phase, phase); - - QW_UNLOCK(QW_WRITE, &ctx->lock); + if (code) { + output->needStop = true; + if (TSDB_CODE_SUCCESS == output->rspCode) { + output->rspCode = code; + } } + if (dropConnection) { + qwBuildAndSendDropRsp(dropConnection, output->rspCode); + QW_TASK_DLOG("drop msg rsped, code:%x", output->rspCode); + } + + if (cancelConnection) { + qwBuildAndSendCancelRsp(cancelConnection, output->rspCode); + QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); + } + + QW_SCH_TASK_DLOG("end to handle event at phase %d", phase); + + QW_RET(code); +} + + +int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { + int32_t code = 0; + int8_t status = 0; + SQWTaskCtx *ctx = NULL; + bool locked = false; + void *readyConnection = NULL; + void *dropConnection = NULL; + void *cancelConnection = NULL; + + QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); + + output->needStop = false; + + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + locked = true; + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropped, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + } + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (input->code) { + output->rspCode = input->code; + } + + if (QW_PHASE_POST_QUERY == phase) { + ctx->taskHandle = input->taskHandle; + ctx->sinkHandle = input->sinkHandle; + + if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { + ctx->emptyRes = true; + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { + readyConnection = ctx->readyConnection; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); + } + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + output->needStop = true; + QW_SET_RSP_CODE(ctx, output->rspCode); + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + + QW_ERR_JRET(output->rspCode); + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + qwFreeTask(QW_FPARAMS(), ctx); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_SET_RSP_CODE(ctx, output->rspCode); + cancelConnection = ctx->cancelConnection; + + QW_ERR_JRET(output->rspCode); + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + + if (QW_PHASE_POST_QUERY == phase && (!output->needStop)) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->taskStatus)); + } + +_return: + if (ctx) { + if (output->rspCode) { + QW_UPDATE_RSP_CODE(ctx, output->rspCode); + } + + atomic_store_32(&ctx->phase, phase); + + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + qwReleaseTaskCtx(mgmt, ctx); } + if (code) { + output->needStop = true; + if (TSDB_CODE_SUCCESS == output->rspCode) { + output->rspCode = code; + } + } + if (readyConnection) { qwBuildAndSendReadyRsp(readyConnection, output->rspCode); QW_TASK_DLOG("ready msg rsped, code:%x", output->rspCode); @@ -924,7 +920,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { input.taskType = taskType; - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); needStop = output.needStop; code = output.rspCode; @@ -973,21 +969,17 @@ _return: QW_TASK_DLOG("query msg rsped, code:%x", rspCode); } - if (needStop) { - QW_RET(rspCode); - } - input.code = rspCode; input.taskHandle = pTaskInfo; input.sinkHandle = sinkHandle; input.taskStatus = rspCode ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_PARTIAL_SUCCEED; - QW_ERR_RET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); + QW_ERR_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); QW_RET(rspCode); } -int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; SQWTaskCtx *ctx = NULL; int8_t phase = 0; @@ -998,6 +990,12 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_LOCK(QW_WRITE, &ctx->lock); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || + QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already cancelled/dropped, phase:%d", phase); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + phase = QW_GET_PHASE(ctx); if (phase == QW_PHASE_PRE_QUERY) { @@ -1019,7 +1017,7 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t _return: if (code && ctx) { - QW_SET_RSP_CODE(ctx, code); + QW_UPDATE_RSP_CODE(ctx, code); } if (ctx) { @@ -1036,7 +1034,7 @@ _return: } -int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { SQWTaskCtx *ctx = NULL; int32_t code = 0; bool queryRsped = false; @@ -1048,7 +1046,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t dataLen = 0; do { - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); needStop = output.needStop; code = output.rspCode; @@ -1100,7 +1098,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t } input.code = code; - qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); needStop = output.needStop; code = output.rspCode; @@ -1110,7 +1108,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t } -int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; int32_t needRsp = true; void *data = NULL; @@ -1126,7 +1124,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t SQWPhaseInput input = {0}; SQWPhaseOutput output = {0}; - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, &output)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, &output)); needStop = output.needStop; code = output.rspCode; @@ -1154,7 +1152,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t locked = true; // RC WARNING - if (QW_IN_EXECUTOR(ctx)) { + if (QW_IS_QUERY_RUNNING(ctx)) { atomic_store_8(&ctx->queryContinue, 1); } else if (0 == atomic_load_8(&ctx->queryInQueue)) { QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); @@ -1162,6 +1160,8 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t atomic_store_8(&ctx->queryInQueue, 1); QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + + QW_TASK_DLOG("schedule query in queue, phase:%d", ctx->phase); } } @@ -1173,7 +1173,11 @@ _return: input.code = code; - qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); + + if (output.rspCode) { + code = output.rspCode; + } if (code) { qwFreeFetchRsp(rsp); @@ -1190,7 +1194,7 @@ _return: } -int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; bool needRsp = false; SQWTaskCtx *ctx = NULL; @@ -1207,7 +1211,7 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } - if (QW_IN_EXECUTOR(ctx)) { + if (QW_IS_QUERY_RUNNING(ctx)) { QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); @@ -1217,8 +1221,12 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); + QW_SET_RSP_CODE(ctx, TSDB_CODE_QRY_TASK_DROPPED); + locked = false; needRsp = true; + + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } if (!needRsp) { @@ -1228,7 +1236,7 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t _return: if (code) { - QW_SET_RSP_CODE(ctx, code); + QW_UPDATE_RSP_CODE(ctx, code); } if (ctx) { @@ -1259,18 +1267,18 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW if (cfg) { mgmt->cfg = *cfg; if (0 == mgmt->cfg.maxSchedulerNum) { - mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; } if (0 == mgmt->cfg.maxTaskNum) { - mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; + mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; } if (0 == mgmt->cfg.maxSchTaskNum) { - mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; } } else { - mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; - mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; - mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; + mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; } mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 0b1200745b..baa4ad2a04 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -306,15 +306,11 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWTaskCtx *handles = NULL; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); - msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; @@ -335,14 +331,13 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ return TSDB_CODE_QRY_INVALID_INPUT; } + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); msg->taskId = be64toh(msg->taskId); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d1cc9f03d1..a94af4f69b 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -38,6 +38,10 @@ namespace { +#define qwtTestQueryQueueSize 1000 +#define qwtTestFetchQueueSize 1000 +#define qwtTestMaxExecTaskUsec 1000000 + bool qwtTestEnableSleep = true; bool qwtTestStop = false; bool qwtTestDeadLoop = true; @@ -45,6 +49,36 @@ int32_t qwtTestMTRunSec = 10; int32_t qwtTestPrintNum = 100000; int32_t qwtTestCaseIdx = 0; int32_t qwtTestCaseNum = 4; +bool qwtTestCaseFinished = false; +tsem_t qwtTestQuerySem; +tsem_t qwtTestFetchSem; + +int32_t qwtTestQueryQueueRIdx = 0; +int32_t qwtTestQueryQueueWIdx = 0; +int32_t qwtTestQueryQueueNum = 0; +SRWLatch qwtTestQueryQueueLock = 0; +struct SRpcMsg *qwtTestQueryQueue[qwtTestQueryQueueSize] = {0}; + +int32_t qwtTestFetchQueueRIdx = 0; +int32_t qwtTestFetchQueueWIdx = 0; +int32_t qwtTestFetchQueueNum = 0; +SRWLatch qwtTestFetchQueueLock = 0; +struct SRpcMsg *qwtTestFetchQueue[qwtTestFetchQueueSize] = {0}; + + +int32_t qwtTestSinkBlockNum = 0; +int32_t qwtTestSinkMaxBlockNum = 0; +bool qwtTestSinkQueryEnd = false; +SRWLatch qwtTestSinkLock = 0; + + +SRpcMsg qwtfetchRpc = {0}; +SResFetchReq qwtfetchMsg = {0}; +SRpcMsg qwtreadyRpc = {0}; +SResReadyReq qwtreadyMsg = {0}; +SRpcMsg qwtdropRpc = {0}; +STaskDropReq qwtdropMsg = {0}; + void qwtInitLogFile() { const char *defaultLogFileNamePrefix = "taosdlog"; @@ -103,30 +137,112 @@ void qwtBuildStatusReqMsg(SSchTasksStatusReq *statusMsg, SRpcMsg *statusRpc) { } int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { + *subplan = 0x1; return 0; } -int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { +int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { + taosWLockLatch(&qwtTestFetchQueueLock); + qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = pMsg; + if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { + qwtTestFetchQueueWIdx = 0; + } + + qwtTestFetchQueueNum++; + + if (qwtTestFetchQueueWIdx == qwtTestFetchQueueRIdx) { + printf("Fetch queue is full"); + assert(0); + } + taosWUnLockLatch(&qwtTestFetchQueueLock); + + tsem_post(&qwtTestFetchSem); + return 0; } +int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { + taosWLockLatch(&qwtTestQueryQueueLock); + qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = pMsg; + if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { + qwtTestQueryQueueWIdx = 0; + } + + qwtTestQueryQueueNum++; + + if (qwtTestQueryQueueWIdx == qwtTestQueryQueueRIdx) { + printf("query queue is full"); + assert(0); + } + taosWUnLockLatch(&qwtTestQueryQueueLock); + + tsem_post(&qwtTestQuerySem); + + return 0; +} + + + void qwtRpcSendResponse(const SRpcMsg *pRsp) { -/* - if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { - SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; - printf("task num:%d\n", rsp->num); - for (int32_t i = 0; i < rsp->num; ++i) { - STaskStatus *task = &rsp->status[i]; - printf("qId:%"PRIx64",tId:%"PRIx64",status:%d\n", task->queryId, task->taskId, task->status); + + switch (pRsp->msgType) { + case TDMT_VND_QUERY_RSP: { + SQueryTableRsp *rsp = (SQueryTableRsp *)pRsp->pCont; + + if (0 == pRsp->code) { + qwtBuildReadyReqMsg(&qwtreadyMsg, &qwtreadyRpc); + qwtPutReqToFetchQueue(0x1, &qwtreadyRpc); + } else { + qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); + qwtPutReqToFetchQueue(0x1, &qwtdropRpc); + } + + break; + } + case TDMT_VND_RES_READY_RSP: { + SResReadyRsp *rsp = (SResReadyRsp *)pRsp->pCont; + + if (0 == pRsp->code) { + qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); + qwtPutReqToFetchQueue(0x1, &qwtfetchRpc); + } else { + qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); + qwtPutReqToFetchQueue(0x1, &qwtdropRpc); + } + break; + } + case TDMT_VND_FETCH_RSP: { + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)pRsp->pCont; + + if (0 == pRsp->code && 0 == rsp->completed) { + qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); + qwtPutReqToFetchQueue(0x1, &qwtfetchRpc); + return; + } + + qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); + qwtPutReqToFetchQueue(0x1, &qwtdropRpc); + + break; + } + case TDMT_VND_DROP_TASK: { + STaskDropRsp *rsp = (STaskDropRsp *)pRsp->pCont; + + qwtTestCaseFinished = true; + break; } } -*/ + return; } int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { - int32_t idx = qwtTestCaseIdx % qwtTestCaseNum; + int32_t idx = abs((++qwtTestCaseIdx) % qwtTestCaseNum); + + qwtTestSinkBlockNum = 0; + qwtTestSinkMaxBlockNum = rand() % 100 + 1; + qwtTestSinkQueryEnd = false; if (0 == idx) { *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; @@ -141,13 +257,30 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTas *pTaskInfo = NULL; *handle = (DataSinkHandle)qwtTestCaseIdx; } - - ++qwtTestCaseIdx; return 0; } int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { + int32_t endExec = 0; + + if (NULL == tinfo) { + *pRes = NULL; + *useconds = 0; + } else { + endExec = rand() % 5; + + if (endExec) { + usleep(rand() % qwtTestMaxExecTaskUsec); + + *pRes = (SSDataBlock*)0x1; + } else { + *pRes = NULL; + usleep(rand() % qwtTestMaxExecTaskUsec); + *useconds = rand() % 10; + } + } + return 0; } @@ -156,21 +289,85 @@ int32_t qwtKillTask(qTaskInfo_t qinfo) { } void qwtDestroyTask(qTaskInfo_t qHandle) { - } int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { + if (NULL == handle || NULL == pInput || NULL == pContinue) { + assert(0); + } + + taosWLockLatch(&qwtTestSinkLock); + + qwtTestSinkBlockNum++; + + if (qwtTestSinkBlockNum >= qwtTestSinkMaxBlockNum) { + *pContinue = true; + } else { + *pContinue = false; + } + taosWUnLockLatch(&qwtTestSinkLock); + return 0; } void qwtEndPut(DataSinkHandle handle, uint64_t useconds) { + if (NULL == handle) { + assert(0); + } + + qwtTestSinkQueryEnd = true; } void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { + static int32_t in = 0; + + if (in > 0) { + assert(0); + } + + atomic_add_fetch_32(&in, 1); + + if (NULL == handle) { + assert(0); + } + + taosWLockLatch(&qwtTestSinkLock); + if (qwtTestSinkBlockNum > 0) { + *pLen = rand() % 100 + 1; + qwtTestSinkBlockNum--; + } else { + *pLen = 0; + } + taosWUnLockLatch(&qwtTestSinkLock); + + *pQueryEnd = qwtTestSinkQueryEnd; + + atomic_sub_fetch_32(&in, 1); } int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { + taosWLockLatch(&qwtTestSinkLock); + if (qwtTestSinkBlockNum > 0) { + qwtTestSinkBlockNum--; + pOutput->numOfRows = rand() % 10 + 1; + pOutput->compressed = 1; + pOutput->pData = malloc(pOutput->numOfRows); + pOutput->queryEnd = qwtTestSinkQueryEnd; + if (qwtTestSinkBlockNum == 0) { + pOutput->bufStatus = DS_BUF_EMPTY; + } else if (qwtTestSinkBlockNum <= qwtTestSinkMaxBlockNum*0.5) { + pOutput->bufStatus = DS_BUF_LOW; + } else { + pOutput->bufStatus = DS_BUF_FULL; + } + pOutput->useconds = rand() % 10 + 1; + pOutput->precision = 1; + } else { + assert(0); + } + taosWUnLockLatch(&qwtTestSinkLock); + return 0; } @@ -438,20 +635,31 @@ void *statusThread(void *param) { } -void *controlThread(void *param) { - SRpcMsg queryRpc = {0}; +void *clientThread(void *param) { int32_t code = 0; uint32_t n = 0; - void *mockPointer = (void *)0x1; void *mgmt = param; + void *mockPointer = (void *)0x1; + SRpcMsg queryRpc = {0}; + + sleep(1); while (!qwtTestStop) { + qwtTestCaseFinished = false; + qwtBuildQueryReqMsg(&queryRpc); - qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + qwtPutReqToQueue(0x1, &queryRpc); + + while (!qwtTestCaseFinished) { + usleep(1); + } + free(queryRpc.pCont); + if (qwtTestEnableSleep) { usleep(rand()%5); } + if (++n % qwtTestPrintNum == 0) { printf("query:%d\n", n); } @@ -461,10 +669,79 @@ void *controlThread(void *param) { } void *queryQueueThread(void *param) { + void *mockPointer = (void *)0x1; + SRpcMsg *queryRpc = NULL; + void *mgmt = param; + + while (!qwtTestStop) { + tsem_wait(&qwtTestQuerySem); + + taosWLockLatch(&qwtTestQueryQueueLock); + if (qwtTestQueryQueueNum <= 0 || qwtTestQueryQueueRIdx == qwtTestQueryQueueWIdx) { + printf("query queue is empty\n"); + assert(0); + } + + queryRpc = qwtTestQueryQueue[qwtTestQueryQueueRIdx++]; + + if (qwtTestQueryQueueRIdx >= qwtTestQueryQueueSize) { + qwtTestQueryQueueRIdx = 0; + } + + qwtTestQueryQueueNum--; + taosWUnLockLatch(&qwtTestQueryQueueLock); + + if (TDMT_VND_QUERY == queryRpc->msgType) { + qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + } else if (TDMT_VND_QUERY_CONTINUE == queryRpc->msgType) { + qWorkerProcessCQueryMsg(mockPointer, mgmt, &queryRpc) + } else { + printf("unknown msg in query queue, type:%d\n", queryRpc->msgType); + assert(0); + } + } } void *fetchQueueThread(void *param) { + void *mockPointer = (void *)0x1; + SRpcMsg *fetchRpc = NULL; + void *mgmt = param; + + while (!qwtTestStop) { + tsem_wait(&qwtTestFetchSem); + + taosWLockLatch(&qwtTestFetchQueueLock); + if (qwtTestFetchQueueNum <= 0 || qwtTestFetchQueueRIdx == qwtTestFetchQueueWIdx) { + printf("Fetch queue is empty\n"); + assert(0); + } + + fetchRpc = qwtTestFetchQueue[qwtTestFetchQueueRIdx++]; + + if (qwtTestFetchQueueRIdx >= qwtTestFetchQueueSize) { + qwtTestFetchQueueRIdx = 0; + } + + qwtTestFetchQueueNum--; + taosWUnLockLatch(&qwtTestFetchQueueLock); + + switch (fetchRpc->msgType) { + case TDMT_VND_FETCH: + qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc); + case TDMT_VND_RES_READY: + qWorkerProcessReadyMsg(mockPointer, mgmt, fetchRpc); + case TDMT_VND_TASKS_STATUS: + qWorkerProcessStatusMsg(mockPointer, mgmt, fetchRpc); + case TDMT_VND_CANCEL_TASK: + qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc); + case TDMT_VND_DROP_TASK: + qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc); + default: + printf("unknown msg type:%d in fetch queue", fetchRpc->msgType); + assert(0); + } + } } @@ -753,13 +1030,16 @@ TEST(rcTest, multithread) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, controlThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, NULL); - pthread_create(&(t3), &thattr, fetchQueueThread, NULL); + pthread_create(&(t1), &thattr, clientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -779,6 +1059,7 @@ TEST(rcTest, multithread) { int main(int argc, char** argv) { + srand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d8904cdfa9..366cc62a7e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -276,9 +276,12 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { int32_t schRecordTaskSucceedNode(SSchTask *pTask) { - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); - - assert(NULL != addr); + int32_t idx = atomic_load_8(&pTask->candidateIdx); + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, idx); + if (NULL == addr) { + SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", idx, taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } pTask->succeedAddr = *addr; @@ -578,9 +581,10 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod tsem_post(&pJob->rspSem); } - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + int32_t code = atomic_load_32(&pJob->errCode); + SCH_ERR_RET(code); - assert(0); + SCH_JOB_ELOG("job errCode is invalid, errCode:%d", code); } @@ -725,7 +729,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); - SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); + SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask)); int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { @@ -738,11 +742,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); if (taskDone < pTask->level->taskNum) { - SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); + SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { - assert(0); + SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum); } if (pTask->level->taskFailed > 0) { @@ -875,19 +879,22 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } atomic_store_ptr(&pJob->res, rsp); - atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); + atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); } + SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + SCH_ERR_JRET(schProcessOnDataFetched(pJob)); break; } case TDMT_VND_DROP_TASK: { // SHOULD NEVER REACH HERE - assert(0); + SCH_TASK_ELOG("invalid status to handle drop task rsp, ref:%d", atomic_load_32(&pJob->ref)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } default: @@ -936,7 +943,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in pTask = *task; - SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode); + SCH_TASK_DLOG("rsp msg received, type:%d, %s, code:%x", msgType, TMSG_INFO(msgType), rspCode); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); @@ -1037,6 +1044,8 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); SCH_ERR_JRET(code); } + + qDebug("QID:%"PRIx64 ",TID:%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; @@ -1296,6 +1305,8 @@ void schDropJobAllTasks(SSchJob *pJob) { } int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { + qDebug("QID:%"PRIx64" job started", pDag->queryId); + if (nodeList && taosArrayGetSize(nodeList) <= 0) { qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); } @@ -1363,7 +1374,7 @@ _return: *(SSchJob **)job = NULL; - scheduleFreeJob(pJob); + schedulerFreeJob(pJob); SCH_RET(code); } @@ -1408,7 +1419,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1425,7 +1436,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1558,7 +1569,7 @@ _return: } -int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { +int32_t schedulerFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1624,11 +1635,12 @@ _return: } *pData = rsp; + SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - SCH_JOB_DLOG("fetch done, code:%x", code); + SCH_JOB_DLOG("fetch done, totalRows:%d, code:%x", pJob->resNumOfRows, code); atomic_sub_fetch_32(&pJob->ref, 1); @@ -1647,7 +1659,7 @@ int32_t scheduleCancelJob(void *job) { SCH_RET(code); } -void scheduleFreeJob(void *job) { +void schedulerFreeJob(void *job) { if (NULL == job) { return; } @@ -1676,7 +1688,8 @@ void scheduleFreeJob(void *job) { usleep(1); } else { - assert(0); + SCH_JOB_ELOG("invalid job ref number, ref:%d", ref); + break; } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 1425ac0e6c..9348339efb 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -328,7 +328,7 @@ void schtFreeQueryJob(int32_t freeThread) { SSchJob *job = atomic_load_ptr(&pQueryJob); if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) { - scheduleFreeJob(job); + schedulerFreeJob(job); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { printf("FreeNum:%d\n", freeNum); @@ -372,7 +372,7 @@ void* schtRunJobThread(void *aa) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &job); assert(code == 0); execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); @@ -466,7 +466,7 @@ void* schtRunJobThread(void *aa) { atomic_store_32(&schtStartFetch, 1); void *data = NULL; - code = scheduleFetchRows(pQueryJob, &data); + code = schedulerFetchRows(pQueryJob, &data); assert(code == 0 || code); if (0 == code) { @@ -476,7 +476,7 @@ void* schtRunJobThread(void *aa) { } data = NULL; - code = scheduleFetchRows(pQueryJob, &data); + code = schedulerFetchRows(pQueryJob, &data); assert(code == 0 || code); schtFreeQueryJob(0); @@ -533,7 +533,7 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); ASSERT_EQ(code, 0); SSchJob *job = (SSchJob *)pJob; @@ -588,7 +588,7 @@ TEST(queryTest, normalCase) { pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job); void *data = NULL; - code = scheduleFetchRows(job, &data); + code = schedulerFetchRows(job, &data); ASSERT_EQ(code, 0); SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; @@ -597,11 +597,11 @@ TEST(queryTest, normalCase) { tfree(data); data = NULL; - code = scheduleFetchRows(job, &data); + code = schedulerFetchRows(job, &data); ASSERT_EQ(code, 0); ASSERT_TRUE(data); - scheduleFreeJob(pJob); + schedulerFreeJob(pJob); schtFreeQueryDag(&dag); @@ -643,11 +643,11 @@ TEST(insertTest, normalCase) { pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); SQueryResult res = {0}; - code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); - scheduleFreeJob(pInsertJob); + schedulerFreeJob(pInsertJob); schedulerDestroy(); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a67d66efb0..3a67b6515b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -353,7 +353,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_INPUT, "invalid input") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SCH_NOT_EXIST, "Scheduler not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_NOT_EXIST, "Task not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_ALREADY_EXIST, "Task already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST, "Task result cache not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST, "Task context not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLED, "Task cancelled") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPED, "Task dropped") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling") From 7a9990404aecb72f8d03c92e5790a4a5f1dff30d Mon Sep 17 00:00:00 2001 From: Shengliang Date: Fri, 21 Jan 2022 20:20:41 +0800 Subject: [PATCH 051/118] fix auth error while create table --- source/dnode/mgmt/impl/src/dndMnode.c | 2 ++ source/dnode/mgmt/impl/src/dndTransport.c | 13 ++++++------- source/dnode/mnode/impl/src/mndAuth.c | 12 ++++++++++-- source/dnode/vnode/src/vnd/vnodeWrite.c | 2 +- source/libs/transport/src/rpcMain.c | 6 +++--- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 32730fc1e3..f952f69f20 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -621,5 +621,7 @@ int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *enc int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); dndReleaseMnode(pDnode, pMnode); + + dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); return code; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index f4fda75bd8..104e702afb 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -171,7 +171,7 @@ static int32_t dndInitClient(SDnode *pDnode) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND-C"; + rpcInit.label = "D-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; @@ -282,12 +282,12 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SDnode *pDnode = parent; if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) { - // dTrace("get internal auth success"); + dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) { - // dTrace("get auth from internal mnode"); + dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } @@ -296,13 +296,12 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char return -1; } - // dDebug("user:%s, send auth msg to other mnodes", user); - SAuthReq *pReq = rpcMallocCont(sizeof(SAuthReq)); tstrncpy(pReq->user, user, TSDB_USER_LEN); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH}; SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, pReq->spi, pReq->encrypt); dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { @@ -314,7 +313,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); *spi = pRsp->spi; *encrypt = pRsp->encrypt; - dDebug("user:%s, success to get user auth from other mnodes", user); + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, pRsp->spi, pRsp->encrypt); } rpcFreeCont(rpcRsp.pCont); @@ -333,7 +332,7 @@ static int32_t dndInitServer(SDnode *pDnode) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = pDnode->cfg.serverPort; - rpcInit.label = "DND-S"; + rpcInit.label = "D-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; rpcInit.sessions = pDnode->cfg.maxShellConns; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 52730412e9..3e9ab99a45 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -28,6 +28,14 @@ void mndCleanupAuth(SMnode *pMnode) {} int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { - mDebug("user:%s, auth req is processed", pReq->user); - return 0; + SAuthReq *pAuth = pReq->rpcMsg.pCont; + + int32_t contLen = sizeof(SAuthRsp); + SAuthRsp *pRsp = rpcMallocCont(contLen); + pReq->pCont = pRsp; + pReq->contLen = contLen; + + int32_t code = mndRetriveAuth(pReq->pMnode, pAuth->user, &pRsp->spi, &pRsp->encrypt, pRsp->secret, pRsp->ckey); + mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, pAuth->spi, pAuth->encrypt, pAuth->user); + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 9b1f80f329..dcdc03adf6 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -96,7 +96,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } - vInfo("vgId:%d process create %"PRIzu" tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); + vDebug("vgId:%d process create %"PRIzu" tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); break; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index f381768a34..d870ae98ab 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -752,8 +752,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, - hashstr); + tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, sid, + hashstr, pConn->spi); } return pConn; @@ -1612,7 +1612,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { } } } else { - tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); + tError("%s, auth spi:%d not matched with received:%d %p", pConn->info, pConn->spi, pHead->spi, pConn); code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; } From 8105801040a0041dca8728f5497f28fadc1a812a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 21 Jan 2022 20:48:00 +0800 Subject: [PATCH 052/118] refactor rpc --- source/libs/transport/src/transCli.c | 83 ++++++++++------ source/libs/transport/src/transSrv.c | 136 +++++++++++++++++++-------- 2 files changed, 148 insertions(+), 71 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f2d844f73d..5a505d0e3e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -21,6 +21,7 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; uv_write_t* writeReq; + void* hostThrd; SConnBuffer readBuf; void* data; queue conn; @@ -45,7 +46,7 @@ typedef struct SCliThrdObj { queue msg; pthread_mutex_t msgMtx; uint64_t nextTimeout; // next timeout - void* shandle; // + void* pTransInst; // } SCliThrdObj; @@ -69,7 +70,7 @@ static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// process data read from server, auth/decompress etc +// process data read from server, auth/decompress etc later static void clientProcessData(SCliConn* conn); // check whether already read complete packet from server static bool clientReadComplete(SConnBuffer* pBuf); @@ -91,7 +92,7 @@ static void* clientThread(void* arg); static void clientProcessData(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = pCtx->ahandle; + SRpcInfo* pRpc = pCtx->pRpc; SRpcMsg rpcMsg; rpcMsg.pCont = conn->readBuf.buf; @@ -104,7 +105,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; - SRpcInfo* pRpc = pThrd->shandle; + SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); @@ -127,7 +128,7 @@ static void clientTimeoutCb(uv_timer_t* handle) { } static void* connCacheCreate(int size) { SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - return false; + return cache; } static void* connCacheDestroy(void* cache) { SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); @@ -153,8 +154,9 @@ static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { if (plist == NULL) { SConnList list; plist = &list; - QUEUE_INIT(&plist->conn); taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist)); + plist = taosHashGet(pCache, key, strlen(key)); + QUEUE_INIT(&plist->conn); } if (QUEUE_IS_EMPTY(&plist->conn)) { @@ -169,8 +171,7 @@ static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); - STransConnCtx* ctx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = ctx->pRpc; + SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); // list already create before @@ -200,10 +201,11 @@ static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); + pBuf->buf = (char*)calloc(1, CAPACITY * sizeof(char)); pBuf->len = 0; pBuf->cap = CAPACITY; pBuf->left = -1; + buf->base = pBuf->buf; buf->len = CAPACITY; } else { @@ -213,7 +215,7 @@ static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, pBuf->buf = realloc(pBuf->buf, pBuf->cap); } else if (pBuf->len + pBuf->left > pBuf->cap) { pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); + pBuf->buf = realloc(pBuf->buf, pBuf->cap); } } buf->base = pBuf->buf + pBuf->len; @@ -227,7 +229,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { - tDebug("alread read complete pack"); + tDebug("alread read complete"); clientProcessData(conn); } else { tDebug("read halp packet, continue to read"); @@ -260,7 +262,12 @@ static void clientWriteCb(uv_write_t* req, int status) { uv_close((uv_handle_t*)pConn->stream, clientDestroy); return; } - + SCliThrdObj* pThrd = pConn->hostThrd; + if (pConn->stream == NULL) { + pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); + pConn->stream->data = pConn; + } uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } @@ -270,35 +277,35 @@ static void clientWrite(SCliConn* pConn) { SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - int msgLen = transMsgLenFromCont(pMsg->contLen); - char* msg = (char*)(pHead); + int msgLen = transMsgLenFromCont(pMsg->contLen); - uv_buf_t wb = uv_buf_init(msg, msgLen); + pHead->msgType = pMsg->msgType; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + + uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); + tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; - if (status != 0) { - tError("failed to connect %s", uv_err_name(status)); - clientConnDestroy(pConn); - return; - } + SCliMsg* pMsg = pConn->data; - SCliMsg* pMsg = pConn->data; - STransConnCtx* pCtx = ((SCliMsg*)(pConn->data))->ctx; - - SRpcMsg rpcMsg; - rpcMsg.ahandle = pCtx->ahandle; + STransConnCtx* pCtx = pMsg->ctx; + SRpcInfo* pRpc = pCtx->pRpc; if (status != 0) { + // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); + tError("failed to connect server, errmsg: %s", uv_strerror(status)); // call user fp later - tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); - SRpcInfo* pRpc = pMsg->ctx->pRpc; + SRpcMsg rpcMsg; + rpcMsg.ahandle = pCtx->ahandle; + // SRpcInfo* pRpc = pMsg->ctx->pRpc; (pRpc->cfp)(NULL, &rpcMsg, NULL); uv_close((uv_handle_t*)req->handle, clientDestroy); return; } + assert(pConn->stream == req->handle); clientWrite(pConn); } @@ -315,17 +322,27 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { // impl later conn->data = pMsg; conn->writeReq->data = conn; + + conn->readBuf.len = 0; + memset(conn->readBuf.buf, 0, conn->readBuf.cap); + conn->readBuf.left = -1; clientWrite(conn); } else { SCliConn* conn = calloc(1, sizeof(SCliConn)); + // read/write stream handle conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); + conn->stream->data = conn; + + // write req handle conn->writeReq = malloc(sizeof(uv_write_t)); + conn->writeReq->data = conn; QUEUE_INIT(&conn->conn); conn->connReq.data = conn; conn->data = pMsg; + conn->hostThrd = pThrd; struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); @@ -359,23 +376,24 @@ static void clientAsyncCb(uv_async_t* handle) { static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; - SRpcInfo* pRpc = pThrd->shandle; - pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; - uv_timer_start(pThrd->pTimer, clientTimeoutCb, pRpc->idleTime * 10, 0); + uv_run(pThrd->loop, UV_RUN_DEFAULT); } void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SClientObj* cli = calloc(1, sizeof(SClientObj)); + SRpcInfo* pRpc = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); @@ -385,8 +403,11 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, pThrd->pTimer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->pTimer); + pThrd->pTimer->data = pThrd; + pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; - pThrd->shandle = shandle; + pThrd->cache = connCacheCreate(1); + pThrd->pTransInst = shandle; int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); if (err == 0) { diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 4542541043..77b5f635f4 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -24,13 +24,15 @@ typedef struct SConn { uv_async_t* pWorkerAsync; queue queue; int ref; - int persist; // persist connection or not - SConnBuffer connBuf; // read buf, - SConnBuffer writeBuf; // write buf + int persist; // persist connection or not + SConnBuffer connBuf; // read buf, int count; - void* shandle; // rpc init - void* ahandle; // + int inType; + void* pTransInst; // rpc init + void* ahandle; // void* hostThrd; + + SRpcMsg sendMsg; // del later char secured; int spi; @@ -48,7 +50,7 @@ typedef struct SWorkThrdObj { uv_async_t* workerAsync; // queue conn; pthread_mutex_t connMtx; - void* shandle; + void* pTransInst; } SWorkThrdObj; typedef struct SServerObj { @@ -66,7 +68,7 @@ typedef struct SServerObj { static const char* notify = "a"; // refactor later -static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen); +static int transAddAuthPart(SConn* pConn, char* msg, int msgLen); static int uvAuthMsg(SConn* pConn, char* msg, int msgLen); @@ -75,10 +77,13 @@ static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_b static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void uvOnTimeoutCb(uv_timer_t* handle); static void uvOnWriteCb(uv_write_t* req, int status); +static void uvOnPipeWriteCb(uv_write_t* req, int status); static void uvOnAcceptCb(uv_stream_t* stream, int status); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvWorkerAsyncCb(uv_async_t* handle); +static void uvPrepareSendData(SConn* conn, uv_buf_t* wb); + // already read complete packet static bool readComplete(SConnBuffer* buf); @@ -135,25 +140,28 @@ static bool readComplete(SConnBuffer* data) { if (msgLen > data->len) { data->left = msgLen - data->len; return false; - } else { + } else if (msgLen == data->len) { return true; + } else if (msgLen < data->len) { + return false; + // handle other packet later } } else { return false; } } -static void uvDoProcess(SRecvInfo* pRecv) { - // impl later - STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; - SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; - SConn* pConn = pRecv->thandle; - tDump(pRecv->msg, pRecv->msgLen); - terrno = 0; - // SRpcReqContext* pContest; - - // do auth and check -} +// static void uvDoProcess(SRecvInfo* pRecv) { +// // impl later +// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; +// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; +// SConn* pConn = pRecv->thandle; +// tDump(pRecv->msg, pRecv->msgLen); +// terrno = 0; +// // SRpcReqContext* pContest; +// +// // do auth and check +//} static int uvAuthMsg(SConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; @@ -222,12 +230,13 @@ static void uvProcessData(SConn* pConn) { p->msgLen = pBuf->len; p->ip = 0; p->port = 0; - p->shandle = pConn->shandle; // + p->shandle = pConn->pTransInst; // p->thandle = pConn; p->chandle = NULL; - // STransMsgHead* pHead = (STransMsgHead*)p->msg; + + pConn->inType = pHead->msgType; assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; @@ -247,7 +256,9 @@ static void uvProcessData(SConn* pConn) { // add compress later // pHead = rpcDecompressRpcMsg(pHead); } else { + pHead->msgLen = htonl(pHead->msgLen); // impl later + // } rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; @@ -257,7 +268,7 @@ static void uvProcessData(SConn* pConn) { rpcMsg.handle = pConn; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); - uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0); + uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth // validate msg type } @@ -277,8 +288,9 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { return; } if (nread != UV_EOF) { - tDebug("Read error %s\n", uv_err_name(nread)); + tDebug("Read error %s", uv_err_name(nread)); } + tDebug("read error %s", uv_err_name(nread)); uv_close((uv_handle_t*)cli, uvConnDestroy); } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -293,16 +305,48 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnWriteCb(uv_write_t* req, int status) { SConn* conn = req->data; + + SConnBuffer* buf = &conn->connBuf; + buf->len = 0; + memset(buf->buf, 0, buf->cap); + buf->left = -1; if (status == 0) { tDebug("data already was written on stream"); } else { + tDebug("failed to write data, %s", uv_err_name(status)); connDestroy(conn); } // opt } +static void uvOnPipeWriteCb(uv_write_t* req, int status) { + if (status == 0) { + tDebug("success to dispatch conn to work thread"); + } else { + tError("fail to dispatch conn to work thread"); + } +} +static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { + // impl later + SRpcMsg* pMsg = &conn->sendMsg; + if (pMsg->pCont == 0) { + pMsg->pCont = (void*)rpcMallocCont(0); + pMsg->contLen = 0; + } + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->msgType = conn->inType + 1; + // add more info + char* msg = (char*)pHead; + int32_t len = transMsgLenFromCont(pMsg->contLen); + if (transCompressMsg(msg, len, NULL)) { + // impl later + } + pHead->msgLen = htonl(len); + wb->base = msg; + wb->len = len; +} void uvWorkerAsyncCb(uv_async_t* handle) { - SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync); + SWorkThrdObj* pThrd = handle->data; SConn* conn = NULL; queue wq; // batch process to avoid to lock/unlock frequently @@ -318,8 +362,8 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("except occurred, do nothing"); return; } - uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); - + uv_buf_t wb; + uvPrepareSendData(conn, &wb); uv_timer_stop(conn->pTimer); uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); @@ -341,8 +385,9 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThreads; + tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); - uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); } @@ -374,7 +419,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(pending == UV_TCP); SConn* pConn = connCreate(); - pConn->shandle = pThrd->shandle; + pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pConn->pTimer); @@ -398,6 +443,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tDebug("new connection created: %d", fd); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { + tDebug("failed to create new connection"); connDestroy(pConn); } } @@ -418,14 +464,12 @@ void* acceptThread(void* arg) { } uv_run(srv->loop, UV_RUN_DEFAULT); } -void* workerThread(void* arg) { - SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; - +static void initWorkThrdObj(SWorkThrdObj* pThrd) { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); // SRpcInfo* pRpc = pThrd->shandle; - uv_pipe_init(pThrd->loop, pThrd->pipe, 0); + uv_pipe_init(pThrd->loop, pThrd->pipe, 1); uv_pipe_open(pThrd->pipe, pThrd->fd); pThrd->pipe->data = pThrd; @@ -435,8 +479,12 @@ void* workerThread(void* arg) { pThrd->workerAsync = malloc(sizeof(uv_async_t)); uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); + pThrd->workerAsync->data = pThrd; uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); +} +void* workerThread(void* arg) { + SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); } @@ -444,34 +492,39 @@ static SConn* connCreate() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); return pConn; } +static void connCloseCb(uv_handle_t* handle) { + // impl later + // +} static void connDestroy(SConn* conn) { if (conn == NULL) { return; } uv_timer_stop(conn->pTimer); free(conn->pTimer); - uv_close((uv_handle_t*)conn->pTcp, NULL); - free(conn->connBuf.buf); + // uv_close((uv_handle_t*)conn->pTcp, connCloseCb); free(conn->pTcp); + free(conn->connBuf.buf); free(conn->pWriter); - free(conn); + // free(conn); // handle } static void uvConnDestroy(uv_handle_t* handle) { SConn* conn = handle->data; connDestroy(conn); } -static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen) { - SRpcHead* pHead = (SRpcHead*)msg; +static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) { + STransMsgHead* pHead = (STransMsgHead*)msg; if (pConn->spi && pConn->secured == 0) { // add auth part pHead->spi = pConn->spi; - SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen); + STransDigestMsg* pDigest = (STransDigestMsg*)(msg + msgLen); pDigest->timeStamp = htonl(taosGetTimestampSec()); msgLen += sizeof(SRpcDigest); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); } else { pHead->spi = 0; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); @@ -502,9 +555,11 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - thrd->shandle = shandle; + thrd->pTransInst = shandle; thrd->fd = fds[0]; thrd->pipe = &(srv->pipe[i][1]); // init read + + initWorkThrdObj(thrd); int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { tDebug("sucess to create worker-thread %d", i); @@ -547,6 +602,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SWorkThrdObj* pThrd = pConn->hostThrd; // opt later + pConn->sendMsg = *pMsg; pthread_mutex_lock(&pThrd->connMtx); QUEUE_PUSH(&pThrd->conn, &pConn->queue); pthread_mutex_unlock(&pThrd->connMtx); From f577c418e852d0714d7aa0628d47ebddedea7a1b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 21 Jan 2022 21:09:43 +0800 Subject: [PATCH 053/118] refactor rpc --- source/libs/transport/src/transCli.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5a505d0e3e..00bc1b621f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -99,6 +99,11 @@ static void clientProcessData(SCliConn* conn) { rpcMsg.contLen = conn->readBuf.len; rpcMsg.ahandle = pCtx->ahandle; (pRpc->cfp)(NULL, &rpcMsg, NULL); + + SCliThrdObj* pThrd = conn->hostThrd; + addConnToCache(pThrd->cache, pCtx->ip, pCtx->port, conn); + free(pCtx->ip); + free(pCtx); // impl } static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); From affa8a7ef275b7304cfc7f7d321d0dc5b3971234 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 21 Jan 2022 22:51:59 +0800 Subject: [PATCH 054/118] refactor rpc --- source/libs/transport/src/transCli.c | 35 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 00bc1b621f..bf395d62e5 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,6 +17,8 @@ #include "transComm.h" +#define CONN_PERSIST_TIME(para) (para * 1000 * 100) + typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; @@ -102,6 +104,9 @@ static void clientProcessData(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; addConnToCache(pThrd->cache, pCtx->ip, pCtx->port, conn); + if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { + uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); + } free(pCtx->ip); free(pCtx); // impl @@ -112,6 +117,7 @@ static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; + tDebug("timeout, try to remove expire conn from connCache"); SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); while (p != NULL) { @@ -128,8 +134,8 @@ static void clientTimeoutCb(uv_timer_t* handle) { p = taosHashIterate((SHashObj*)pThrd->cache, p); } - pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; - uv_timer_start(handle, clientTimeoutCb, pRpc->idleTime * 10, 0); + pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); + uv_timer_start(handle, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } static void* connCacheCreate(int size) { SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -158,8 +164,7 @@ static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { SConnList* plist = taosHashGet(pCache, key, strlen(key)); if (plist == NULL) { SConnList list; - plist = &list; - taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist)); + taosHashPut(pCache, key, strlen(key), (void*)&list, sizeof(list)); plist = taosHashGet(pCache, key, strlen(key)); QUEUE_INIT(&plist->conn); } @@ -177,7 +182,7 @@ static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; - conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); // list already create before assert(plist != NULL); @@ -374,14 +379,13 @@ static void clientAsyncCb(uv_async_t* handle) { clientHandleReq(pMsg, pThrd); count++; if (count >= 2) { - tError("send batch size: %d", count); + tDebug("send batch size: %d", count); } } } static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; - uv_run(pThrd->loop, UV_RUN_DEFAULT); } @@ -409,8 +413,8 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, pThrd->pTimer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->pTimer); pThrd->pTimer->data = pThrd; - pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); pThrd->cache = connCacheCreate(1); pThrd->pTransInst = shandle; @@ -426,16 +430,19 @@ static void clientMsgDestroy(SCliMsg* pMsg) { // impl later free(pMsg); } +static void destroyThrdObj(SCliThrdObj* pThrd) { + pthread_join(pThrd->thread, NULL); + pthread_mutex_destroy(&pThrd->msgMtx); + free(pThrd->cliAsync); + free(pThrd->loop); + free(pThrd); +} +// void taosCloseClient(void* arg) { // impl later SClientObj* cli = arg; for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = cli->pThreadObj[i]; - pthread_join(pThrd->thread, NULL); - pthread_mutex_destroy(&pThrd->msgMtx); - free(pThrd->cliAsync); - free(pThrd->loop); - free(pThrd); + destroyThrdObj(cli->pThreadObj[i]); } free(cli->pThreadObj); free(cli); From fc99fe53de433d85d9cb643bdb1d4c223016a142 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 23:19:54 +0800 Subject: [PATCH 055/118] consume skip ununsed table --- include/common/tmsg.h | 51 ++++++++++++++++++---------- source/client/src/clientImpl.c | 2 +- source/client/src/clientMsgHandler.c | 2 +- source/dnode/vnode/inc/vnode.h | 4 +-- source/dnode/vnode/src/tq/tq.c | 2 +- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bb53c6ddfa..e12a46984f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1579,32 +1579,47 @@ typedef struct SMqSetCVgRsp { char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; -typedef struct SMqCVConsumeReq { +typedef struct SMqConsumeReq { int64_t reqId; int64_t offset; int64_t consumerId; int64_t blockingTime; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; -} SMqCVConsumeReq; +} SMqConsumeReq; -typedef struct SMqConsumeRspBlock { - int32_t bodyLen; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char body[]; -} SMqConsumeRspBlock; +typedef struct SMqColData { + int16_t colId; + int16_t type; + int16_t bytes; + char data[]; +} SMqColData; -typedef struct SMqCVConsumeRsp { - int64_t reqId; - int64_t clientId; - int64_t committedOffset; - int64_t receiveOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t bodyLen; - char topicName[TSDB_TOPIC_FNAME_LEN]; - SMqConsumeRspBlock blocks[]; -} SMqCvConsumeRsp; +typedef struct SMqTbData { + int64_t uid; + int32_t numOfCols; + int32_t numOfRows; + SMqColData colData[]; +} SMqTbData; + +typedef struct SMqTopicBlk { + char topicName[TSDB_TOPIC_FNAME_LEN]; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t bodyLen; + int32_t numOfTb; + SMqTbData tbData[]; +} SMqTopicData; + +typedef struct SMqConsumeRsp { + int64_t reqId; + int64_t clientId; + int32_t bodyLen; + int32_t numOfTopics; + SMqTopicData data[]; +} SMqConsumeRsp; #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 159a92b0ab..9cc66d6879 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -117,7 +117,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - p->pAppHbMgr = appHbMgrInit(p); + /*p->pAppHbMgr = appHbMgrInit(p);*/ taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index ec088eb073..81ea18fe08 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); + /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/ // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ab538ff12d..9e6ecb6e23 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,7 +68,7 @@ typedef struct { typedef struct STqReadHandle { int64_t ver; - int64_t tbUid; + uint64_t tbUid; SSubmitMsg* pMsg; SSubmitBlk* pBlock; SSubmitMsgIter msgIter; @@ -204,7 +204,7 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SA pReadHandle->pColIdList = pColIdList; } -static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, int64_t tbUid) { +static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t tbUid) { pHandle->tbUid = tbUid; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4a6f55564c..b18f50cd3f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -610,7 +610,7 @@ int tqItemSSize() { } int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { - SMqCVConsumeReq* pReq = pMsg->pCont; + SMqConsumeReq* pReq = pMsg->pCont; SRpcMsg rpcMsg; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; From 069d715fe7b2a6dad495063d1f42e88730e4a0e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 23:30:03 +0800 Subject: [PATCH 056/118] [td-11818]update log, fix bug in select * from super_table. --- include/common/tmsg.h | 4 +- include/libs/planner/planner.h | 2 +- source/client/inc/clientInt.h | 1 + source/client/src/clientImpl.c | 66 +++++++++++------------ source/libs/planner/inc/plannerInt.h | 2 +- source/libs/planner/src/physicalPlan.c | 26 ++++++--- source/libs/planner/src/planner.c | 20 ++++++- source/libs/planner/test/phyPlanTests.cpp | 6 +-- source/libs/qworker/inc/qworkerInt.h | 18 +++---- source/libs/scheduler/inc/schedulerInt.h | 10 ++-- source/libs/scheduler/src/scheduler.c | 15 ++---- 11 files changed, 93 insertions(+), 77 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b468456cb7..fe75663ab5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -696,13 +696,13 @@ typedef struct SVgroupInfo { uint32_t hashEnd; int8_t inUse; int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; typedef struct { int32_t vgId; int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SVgroupMsg; typedef struct { diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index ba3539e64e..126cee390c 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -178,7 +178,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); + int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 1d10869e30..296fbba634 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -136,6 +136,7 @@ typedef struct SReqResultInfo { TAOS_ROW row; char **pCol; uint32_t numOfRows; + uint64_t totalRows; uint32_t current; bool completed; } SReqResultInfo; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 159a92b0ab..80184a2346 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,9 +25,9 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); +static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); - static bool stringLengthCheck(const char* str, size_t maxsize) { +static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { return false; } @@ -59,7 +59,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i } static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); -static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema); +static void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { if (taos_init() != TSDB_CODE_SUCCESS) { @@ -202,43 +202,38 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { +int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, SArray* pNodeList) { pRequest->type = pQueryNode->type; - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + SSchema* pSchema = NULL; + int32_t numOfCols = 0; + int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &numOfCols, pNodeList, pRequest->requestId); if (code != 0) { return code; } if (pQueryNode->type == TSDB_SQL_SELECT) { - SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0); - - SSubplan* pPlan = taosArrayGetP(pa, 0); - SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); - setResSchemaInfo(pResInfo, pDataBlockSchema); - + setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); pRequest->type = TDMT_VND_QUERY; } return code; } -void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) { - assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { + assert(pSchema != NULL && numOfCols > 0); - pResInfo->numOfCols = pDataBlockSchema->numOfCols; - pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0])); + pResInfo->numOfCols = numOfCols; + pResInfo->fields = calloc(numOfCols, sizeof(pSchema[0])); for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { - SSchema* pSchema = &pDataBlockSchema->pSchema[i]; - pResInfo->fields[i].bytes = pSchema->bytes; - pResInfo->fields[i].type = pSchema->type; - tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name)); + pResInfo->fields[i].bytes = pSchema[i].bytes; + pResInfo->fields[i].type = pSchema[i].type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); } } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { +int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; @@ -256,14 +251,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return pRequest->code; } - SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr)); - - SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 2}; - addr.epAddr[0].port = 7100; - strcpy(addr.epAddr[0].fqdn, "localhost"); - - taosArrayPush(execNode, &addr); - return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob); + return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); } typedef struct tmq_t tmq_t; @@ -399,7 +387,9 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, // todo check for invalid sql statement and return with error code - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return); + SSchema *schema = NULL; + int32_t numOfCols = 0; + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); pStr = qDagToString(pRequest->body.pDag); if(pStr == NULL) { @@ -492,6 +482,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQueryNode = NULL; + SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -500,8 +491,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return); + CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); pRequest->code = terrno; } @@ -719,13 +710,17 @@ void* doFetchRow(SRequestObj* pRequest) { return NULL; } - int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; return NULL; } - setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData); + tscDebug("0x%"PRIx64 " fetch results, numOfRows:%d total Rows:%"PRId64", complete:%d, reqId:0x%"PRIx64, pRequest->self, pResInfo->numOfRows, + pResInfo->totalRows, pResInfo->completed, pRequest->requestId); + if (pResultInfo->numOfRows == 0) { return NULL; } @@ -855,7 +850,7 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } -void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { +void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { assert(pResultInfo != NULL && pRsp != NULL); pResultInfo->pRspMsg = (const char*) pRsp; @@ -864,5 +859,6 @@ void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* p pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); + pResultInfo->totalRows += pResultInfo->numOfRows; setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 41f50607cb..26ae44a08f 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -106,7 +106,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); */ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); -int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId); +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId); void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource); int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len); int32_t stringToSubplan(const char* str, SSubplan** subplan); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index dd869d87b3..0038c51c7a 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -261,14 +261,14 @@ static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) { return; } -static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { - SVgroupsInfo* pVgroupList = pTable->pMeta->vgroupList; +static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) { + SVgroupsInfo* pVgroupList = pTableInfo->pMeta->vgroupList; for (int32_t i = 0; i < pVgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); subplan->msgType = TDMT_VND_QUERY; - vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execNode); - subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + vgroupMsgToEpSet(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode); + subplan->pNode = createMultiTableScanNode(pPlanNode, pTableInfo); subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } @@ -384,18 +384,19 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { subplan->msgType = TDMT_VND_QUERY; subplan->pNode = createPhyNode(pCxt, pRoot); - subplan->pDataSink = createDataDispatcher(pCxt, pRoot, subplan->pNode); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot, subplan->pNode); } // todo deal subquery } -int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId) { +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId) { TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, - .nextId = {.queryId = requestId}, + //The unsigned Id starting from 1 would be better + .nextId = {.queryId = requestId, .subplanId = 1, .templateId = 1}, }; *pDag = context.pDag; @@ -408,6 +409,17 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD terrno = code; return TSDB_CODE_FAILED; } END_TRY + + // traverse the dag again to acquire the execution node. + if (pNodeList != NULL) { + SArray** pSubLevel = taosArrayGetLast((*pDag)->pSubplans); + size_t num = taosArrayGetSize(*pSubLevel); + for (int32_t j = 0; j < num; ++j) { + SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); + taosArrayPush(pNodeList, &pPlan->execNode); + } + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 9b32213ad7..c93569a6c1 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -16,6 +16,8 @@ #include "parser.h" #include "plannerInt.h" +static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols); + static void destroyDataSinkNode(SDataSink* pSinkNode) { if (pSinkNode == NULL) { return; @@ -56,7 +58,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId) { SQueryPlanNode* pLogicPlan; int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { @@ -76,17 +78,31 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return code; } - code = createDag(pLogicPlan, NULL, pDag, requestId); + code = createDag(pLogicPlan, NULL, pDag, pNodeList, requestId); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(pLogicPlan); qDestroyQueryDag(*pDag); return code; } + extractResSchema(pDag, pResSchema, numOfCols); + destroyQueryPlan(pLogicPlan); return TSDB_CODE_SUCCESS; } +void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, + int32_t* numOfCols) { // extract the final result schema + SArray* pTopSubplan = taosArrayGetP((*pDag)->pSubplans, 0); + + SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0); + SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); + + *numOfCols = pDataBlockSchema->numOfCols; + *pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema)); + memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema)); +} + void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { setSubplanExecutionNode(subplan, templateId, pSource); } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 6d9e08e829..edf5fa5a81 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -61,7 +61,7 @@ protected: int32_t run() { SQueryDag* dag = nullptr; uint64_t requestId = 20; - int32_t code = createDag(logicPlan_.get(), nullptr, &dag, requestId); + int32_t code = createDag(logicPlan_.get(), nullptr, &dag, NULL, requestId); dag_.reset(dag); return code; } @@ -78,9 +78,9 @@ protected: SQueryDag* dag = nullptr; uint64_t requestId = 20; SSchema *schema = NULL; - uint32_t numOfOutput = 0; + int32_t numOfOutput = 0; - code = qCreateQueryDag(query, &dag, requestId); + code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, nullptr, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 5f9b33f7e3..3a8f34e831 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -171,17 +171,17 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_ELOG_E(param) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 6b047eb96e..2ab519424b 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -146,12 +146,12 @@ typedef struct SSchJob { #define SCH_SET_JOB_TYPE(pAttr, type) (pAttr)->queryJob = ((type) != QUERY_TYPE_MODIFY) #define SCH_JOB_NEED_FETCH(pAttr) ((pAttr)->queryJob) -#define SCH_JOB_ELOG(param, ...) qError("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) -#define SCH_JOB_DLOG(param, ...) qDebug("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_ELOG(param, ...) qError("QID:0x%"PRIx64" " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%"PRIx64" " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) qError("QID:0x%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) qDebug("QID:0x%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) qWarn("QID:0x%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ddfa73f0a5..7b9396bcb9 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -650,15 +650,11 @@ _return: SCH_RET(code); } - int32_t schProcessOnDataFetched(SSchJob *job) { atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); - tsem_post(&job->rspSem); } - - // Note: no more error processing, handled in function internal int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { bool needRetry = false; @@ -882,7 +878,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } SCH_ERR_JRET(schProcessOnDataFetched(pJob)); - break; } case TDMT_VND_DROP_TASK: { @@ -892,7 +887,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -935,8 +929,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in } pTask = *task; - - SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode); + SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode)); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); @@ -1562,8 +1555,8 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0; atomic_add_fetch_32(&pJob->ref, 1); int8_t status = SCH_GET_JOB_STATUS(pJob); @@ -1609,7 +1602,6 @@ _return: while (true) { *pData = atomic_load_ptr(&pJob->res); - if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { continue; } @@ -1628,8 +1620,7 @@ _return: atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - SCH_JOB_DLOG("fetch done, code:%x", code); - + SCH_JOB_DLOG("fetch done, code:%s", tstrerror(code)); atomic_sub_fetch_32(&pJob->ref, 1); SCH_RET(code); From 79ce13e88db2c7da4b841a3dab6380afe0e0b995 Mon Sep 17 00:00:00 2001 From: dapan Date: Sat, 22 Jan 2022 10:36:03 +0800 Subject: [PATCH 057/118] feature/qnode --- source/libs/qworker/src/qworker.c | 18 +++++++++++------ source/libs/qworker/test/qworkerTests.cpp | 24 ++++++++++++----------- source/libs/scheduler/src/scheduler.c | 2 +- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index bcfbafd4c9..021bd642bd 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -629,7 +629,9 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it - locked = false; + locked = false; + + break; } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); @@ -639,6 +641,8 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_SET_RSP_CODE(ctx, output->rspCode); cancelConnection = ctx->cancelConnection; + + break; } if (ctx->rspCode) { @@ -1215,8 +1219,6 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); - - ctx->dropConnection = qwMsg->connection; } else if (ctx->phase > 0) { QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); @@ -1225,11 +1227,11 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { locked = false; needRsp = true; - - QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - if (!needRsp) { + if (!needRsp) { + ctx->dropConnection = qwMsg->connection; + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } @@ -1239,6 +1241,10 @@ _return: QW_UPDATE_RSP_CODE(ctx, code); } + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + if (ctx) { qwReleaseTaskCtx(mgmt, ctx); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index a94af4f69b..5720c2d47c 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -137,7 +137,7 @@ void qwtBuildStatusReqMsg(SSchTasksStatusReq *statusMsg, SRpcMsg *statusRpc) { } int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { - *subplan = 0x1; + *subplan = (SSubplan *)0x1; return 0; } @@ -192,10 +192,10 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { if (0 == pRsp->code) { qwtBuildReadyReqMsg(&qwtreadyMsg, &qwtreadyRpc); - qwtPutReqToFetchQueue(0x1, &qwtreadyRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtreadyRpc); } else { qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); - qwtPutReqToFetchQueue(0x1, &qwtdropRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); } break; @@ -205,10 +205,10 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { if (0 == pRsp->code) { qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); - qwtPutReqToFetchQueue(0x1, &qwtfetchRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc); } else { qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); - qwtPutReqToFetchQueue(0x1, &qwtdropRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); } break; } @@ -217,12 +217,12 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { if (0 == pRsp->code && 0 == rsp->completed) { qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); - qwtPutReqToFetchQueue(0x1, &qwtfetchRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc); return; } qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); - qwtPutReqToFetchQueue(0x1, &qwtdropRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); break; } @@ -352,7 +352,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { qwtTestSinkBlockNum--; pOutput->numOfRows = rand() % 10 + 1; pOutput->compressed = 1; - pOutput->pData = malloc(pOutput->numOfRows); + pOutput->pData = (char *)malloc(pOutput->numOfRows); pOutput->queryEnd = qwtTestSinkQueryEnd; if (qwtTestSinkBlockNum == 0) { pOutput->bufStatus = DS_BUF_EMPTY; @@ -648,7 +648,7 @@ void *clientThread(void *param) { qwtTestCaseFinished = false; qwtBuildQueryReqMsg(&queryRpc); - qwtPutReqToQueue(0x1, &queryRpc); + qwtPutReqToQueue((void *)0x1, &queryRpc); while (!qwtTestCaseFinished) { usleep(1); @@ -692,15 +692,16 @@ void *queryQueueThread(void *param) { taosWUnLockLatch(&qwtTestQueryQueueLock); if (TDMT_VND_QUERY == queryRpc->msgType) { - qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc); } else if (TDMT_VND_QUERY_CONTINUE == queryRpc->msgType) { - qWorkerProcessCQueryMsg(mockPointer, mgmt, &queryRpc) + qWorkerProcessCQueryMsg(mockPointer, mgmt, queryRpc); } else { printf("unknown msg in query queue, type:%d\n", queryRpc->msgType); assert(0); } } + return NULL; } void *fetchQueueThread(void *param) { @@ -743,6 +744,7 @@ void *fetchQueueThread(void *param) { } } + return NULL; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 366cc62a7e..0ad51d0b57 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -275,7 +275,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } -int32_t schRecordTaskSucceedNode(SSchTask *pTask) { +int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { int32_t idx = atomic_load_8(&pTask->candidateIdx); SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, idx); if (NULL == addr) { From a4aa98dafec75d4754e182724e062313e21d91f5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Jan 2022 11:19:31 +0800 Subject: [PATCH 058/118] [td-11818] add table filter uid. --- source/libs/executor/src/executorimpl.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d9b3f5f4a9..e963c49c86 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5408,7 +5408,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } -SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, uint64_t uid, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5428,6 +5428,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); + tqReadHandleSetTbUid(streamReadHandle, uid); pInfo->readerHandle = streamReadHandle; @@ -7719,7 +7720,8 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); } else if (pPhyNode->info.type == OP_StreamScan) { size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pTaskInfo); + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. + return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo); } } From e4338377c4d6881ba59a53736c3689170e387c6d Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 22 Jan 2022 11:54:57 +0800 Subject: [PATCH 059/118] feature/qnode --- source/libs/qworker/test/qworkerTests.cpp | 130 ++++++++-------------- 1 file changed, 49 insertions(+), 81 deletions(-) diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 5720c2d47c..362156ebcd 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -40,8 +40,9 @@ namespace { #define qwtTestQueryQueueSize 1000 #define qwtTestFetchQueueSize 1000 -#define qwtTestMaxExecTaskUsec 1000000 +#define qwtTestMaxExecTaskUsec 2 +uint64_t qwtTestQueryId = 0; bool qwtTestEnableSleep = true; bool qwtTestStop = false; bool qwtTestDeadLoop = true; @@ -70,14 +71,17 @@ int32_t qwtTestSinkBlockNum = 0; int32_t qwtTestSinkMaxBlockNum = 0; bool qwtTestSinkQueryEnd = false; SRWLatch qwtTestSinkLock = 0; +int32_t qwtTestSinkLastLen = 0; +SSubQueryMsg qwtqueryMsg = {0}; SRpcMsg qwtfetchRpc = {0}; SResFetchReq qwtfetchMsg = {0}; SRpcMsg qwtreadyRpc = {0}; SResReadyReq qwtreadyMsg = {0}; SRpcMsg qwtdropRpc = {0}; STaskDropReq qwtdropMsg = {0}; +SSchTasksStatusReq qwtstatusMsg = {0}; void qwtInitLogFile() { @@ -96,18 +100,17 @@ void qwtInitLogFile() { } void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc->pCont = queryMsg; + qwtqueryMsg.queryId = htobe64(atomic_add_fetch_64(&qwtTestQueryId, 1)); + qwtqueryMsg.sId = htobe64(1); + qwtqueryMsg.taskId = htobe64(1); + qwtqueryMsg.contentLen = htonl(100); + queryRpc->pCont = &qwtqueryMsg; queryRpc->contLen = sizeof(SSubQueryMsg) + 100; } void qwtBuildReadyReqMsg(SResReadyReq *readyMsg, SRpcMsg *readyRpc) { readyMsg->sId = htobe64(1); - readyMsg->queryId = htobe64(1); + readyMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); readyMsg->taskId = htobe64(1); readyRpc->pCont = readyMsg; readyRpc->contLen = sizeof(SResReadyReq); @@ -115,7 +118,7 @@ void qwtBuildReadyReqMsg(SResReadyReq *readyMsg, SRpcMsg *readyRpc) { void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { fetchMsg->sId = htobe64(1); - fetchMsg->queryId = htobe64(1); + fetchMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); fetchMsg->taskId = htobe64(1); fetchRpc->pCont = fetchMsg; fetchRpc->contLen = sizeof(SResFetchReq); @@ -123,7 +126,7 @@ void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { dropMsg->sId = htobe64(1); - dropMsg->queryId = htobe64(1); + dropMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); dropMsg->taskId = htobe64(1); dropRpc->pCont = dropMsg; dropRpc->contLen = sizeof(STaskDropReq); @@ -273,7 +276,8 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (endExec) { usleep(rand() % qwtTestMaxExecTaskUsec); - *pRes = (SSDataBlock*)0x1; + *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); + (*pRes)->info.rows = rand() % 1000; } else { *pRes = NULL; usleep(rand() % qwtTestMaxExecTaskUsec); @@ -297,6 +301,8 @@ int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* p assert(0); } + free((void *)pInput->pData); + taosWLockLatch(&qwtTestSinkLock); qwtTestSinkBlockNum++; @@ -339,6 +345,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { } else { *pLen = 0; } + qwtTestSinkLastLen = *pLen; taosWUnLockLatch(&qwtTestSinkLock); *pQueryEnd = qwtTestSinkQueryEnd; @@ -348,11 +355,23 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { taosWLockLatch(&qwtTestSinkLock); - if (qwtTestSinkBlockNum > 0) { - qwtTestSinkBlockNum--; + if (qwtTestSinkLastLen > 0) { pOutput->numOfRows = rand() % 10 + 1; pOutput->compressed = 1; - pOutput->pData = (char *)malloc(pOutput->numOfRows); + pOutput->queryEnd = qwtTestSinkQueryEnd; + if (qwtTestSinkBlockNum == 0) { + pOutput->bufStatus = DS_BUF_EMPTY; + } else if (qwtTestSinkBlockNum <= qwtTestSinkMaxBlockNum*0.5) { + pOutput->bufStatus = DS_BUF_LOW; + } else { + pOutput->bufStatus = DS_BUF_FULL; + } + pOutput->useconds = rand() % 10 + 1; + pOutput->precision = 1; + } else if (qwtTestSinkLastLen == 0) { + pOutput->numOfRows = 0; + pOutput->compressed = 1; + pOutput->pData = NULL; pOutput->queryEnd = qwtTestSinkQueryEnd; if (qwtTestSinkBlockNum == 0) { pOutput->bufStatus = DS_BUF_EMPTY; @@ -534,7 +553,6 @@ void *queryThread(void *param) { while (!qwtTestStop) { qwtBuildQueryReqMsg(&queryRpc); qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - free(queryRpc.pCont); if (qwtTestEnableSleep) { usleep(rand()%5); } @@ -654,8 +672,6 @@ void *clientThread(void *param) { usleep(1); } - free(queryRpc.pCont); - if (qwtTestEnableSleep) { usleep(rand()%5); } @@ -763,41 +779,11 @@ TEST(seqTest, normalCase) { SRpcMsg statusRpc = {0}; qwtInitLogFile(); - - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - SResReadyReq readyMsg = {0}; - readyMsg.sId = htobe64(1); - readyMsg.queryId = htobe64(1); - readyMsg.taskId = htobe64(1); - readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyReq); - - SResFetchReq fetchMsg = {0}; - fetchMsg.sId = htobe64(1); - fetchMsg.queryId = htobe64(1); - fetchMsg.taskId = htobe64(1); - fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchReq); - - STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - - SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; + qwtBuildQueryReqMsg(&queryRpc); + qwtBuildReadyReqMsg(&qwtreadyMsg, &readyRpc); + qwtBuildFetchReqMsg(&qwtfetchMsg, &fetchRpc); + qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); stubSetStringToPlan(); stubSetRpcSendResponse(); @@ -814,35 +800,35 @@ TEST(seqTest, normalCase) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); @@ -859,26 +845,9 @@ TEST(seqTest, cancelFirst) { qwtInitLogFile(); - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - - STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - - SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; + qwtBuildQueryReqMsg(&queryRpc); + qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); stubSetStringToPlan(); stubSetRpcSendResponse(); @@ -886,21 +855,21 @@ TEST(seqTest, cancelFirst) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - ASSERT_EQ(code, TSDB_CODE_QRY_TASK_DROPPED); + ASSERT_TRUE(0 != code); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); @@ -941,7 +910,6 @@ TEST(seqTest, randCase) { printf("Query,%d\n", t++); qwtBuildQueryReqMsg(&queryRpc); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - free(queryRpc.pCont); } else if (r >= maxr/5 && r < maxr * 2/5) { printf("Ready,%d\n", t++); qwtBuildReadyReqMsg(&readyMsg, &readyRpc); From bf7c5baa8946c79530ee878e2620796954722124 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 22 Jan 2022 12:03:25 +0800 Subject: [PATCH 060/118] minor changes --- tests/script/sh/deploy.sh | 2 ++ tests/test/c/create_table.c | 44 +++++++++++++++++-------------------- tests/tsim/src/simSystem.c | 3 ++- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index e00363b28f..3e8f87d665 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -115,6 +115,8 @@ elif [ $NODE -eq 8 ]; then NODE=7800 fi +HOSTNAME=localhost + echo " " >> $TAOS_CFG echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 74dd3f4535..fddf3e7f2a 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -35,8 +35,7 @@ int32_t numOfVgroups = 2; int32_t showTablesFlag = 0; int32_t queryFlag = 0; -int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 - +int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 typedef struct { int64_t tableBeginIndex; @@ -89,7 +88,7 @@ void createDbAndStb() { pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } taos_free_result(pRes); @@ -182,18 +181,14 @@ void *threadFunc(void *param) { exit(1); } - pPrint("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", - pInfo->threadIndex, - pInfo->tableBeginIndex, - pInfo->tableEndIndex); + pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + pInfo->tableEndIndex); pInfo->tableBeginIndex += startOffset; - pInfo->tableEndIndex += startOffset; + pInfo->tableEndIndex += startOffset; - pPrint("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", - pInfo->threadIndex, - pInfo->tableBeginIndex, - pInfo->tableEndIndex); + pPrint("====after thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); @@ -221,7 +216,7 @@ void *threadFunc(void *param) { int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); - if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + if (code != 0) { pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); } taos_free_result(pRes); @@ -251,20 +246,20 @@ void *threadFunc(void *param) { // batch = MIN(batch, batchNum); int32_t len = sprintf(qstr, "insert into "); - + for (int32_t i = 0; i < batchNumOfTbl;) { - int64_t ts = startTimestamp; + int64_t ts = startTimestamp; len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); for (int32_t j = 0; j < batchNumOfRow; j++) { - len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); } - + t++; i++; if (t > pInfo->tableEndIndex) { break; } - } + } int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); @@ -360,7 +355,7 @@ void parseArgument(int32_t argc, char *argv[]) { } else if (strcmp(argv[i], "-q") == 0) { queryFlag = atoi(argv[++i]); } else { - pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); } } @@ -390,15 +385,16 @@ int32_t main(int32_t argc, char *argv[]) { } if (queryFlag) { - //selectRowsFromTable(); + // selectRowsFromTable(); return 0; } if (createTable) { createDbAndStb(); } - - pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); + + pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, + startOffset); pthread_attr_t thattr; pthread_attr_init(&thattr); @@ -456,8 +452,8 @@ int32_t main(int32_t argc, char *argv[]) { if (createTable) { pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 - "us %s", - GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); + "us %s", + GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); } if (insertData) { diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 016b6500ed..6b6392c1be 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -42,7 +42,8 @@ char *simParseArbitratorName(char *varName) { char *simParseHostName(char *varName) { static char hostName[140]; - sprintf(hostName, "%s", simHostName); + //sprintf(hostName, "%s", simHostName); + sprintf(hostName, "%s", "localhost"); return hostName; } From 046794047b9fea16ab465ca6eb88d0aa1413fcc9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 22 Jan 2022 12:09:34 +0800 Subject: [PATCH 061/118] refactor uv code --- source/libs/transport/src/transCli.c | 102 +++++++++++++++------------ source/libs/transport/src/transSrv.c | 82 ++++++++++++++------- 2 files changed, 115 insertions(+), 69 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bf395d62e5..ffd8d35bfc 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -44,7 +44,7 @@ typedef struct SCliThrdObj { uv_loop_t* loop; uv_async_t* cliAsync; // uv_timer_t* pTimer; - void* cache; // conn pool + void* pool; // conn pool queue msg; pthread_mutex_t msgMtx; uint64_t nextTimeout; // next timeout @@ -65,10 +65,10 @@ typedef struct SConnList { // conn pool // add expire timeout and capacity limit -static void* connCacheCreate(int size); -static void* connCacheDestroy(void* cache); -static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); -static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); +static void* connPoolCreate(int size); +static void* connPoolDestroy(void* pool); +static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port); +static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn); // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); @@ -90,8 +90,14 @@ static void clientConnDestroy(SCliConn* pConn); static void clientMsgDestroy(SCliMsg* pMsg); +// thread obj +static SCliThrdObj* createThrdObj(); +static void destroyThrdObj(SCliThrdObj* pThrd); +// thread static void* clientThread(void* arg); +static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); + static void clientProcessData(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; SRpcInfo* pRpc = pCtx->pRpc; @@ -103,7 +109,7 @@ static void clientProcessData(SCliConn* conn) { (pRpc->cfp)(NULL, &rpcMsg, NULL); SCliThrdObj* pThrd = conn->hostThrd; - addConnToCache(pThrd->cache, pCtx->ip, pCtx->port, conn); + addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } @@ -111,15 +117,14 @@ static void clientProcessData(SCliConn* conn) { free(pCtx); // impl } -static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; - tDebug("timeout, try to remove expire conn from connCache"); + tDebug("timeout, try to remove expire conn from conn pool"); - SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); + SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); while (p != NULL) { while (!QUEUE_IS_EMPTY(&p->conn)) { queue* h = QUEUE_HEAD(&p->conn); @@ -131,18 +136,18 @@ static void clientTimeoutCb(uv_timer_t* handle) { break; } } - p = taosHashIterate((SHashObj*)pThrd->cache, p); + p = taosHashIterate((SHashObj*)pThrd->pool, p); } pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); uv_timer_start(handle, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } -static void* connCacheCreate(int size) { - SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - return cache; +static void* connPoolCreate(int size) { + SHashObj* pool = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + return pool; } -static void* connCacheDestroy(void* cache) { - SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); +static void* connPoolDestroy(void* pool) { + SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { queue* h = QUEUE_HEAD(&connList->conn); @@ -150,22 +155,22 @@ static void* connCacheDestroy(void* cache) { SCliConn* c = QUEUE_DATA(h, SCliConn, conn); clientConnDestroy(c); } - connList = taosHashIterate((SHashObj*)cache, connList); + connList = taosHashIterate((SHashObj*)pool, connList); } - taosHashClear(cache); + taosHashClear(pool); } -static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { +static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { char key[128] = {0}; tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); - SHashObj* pCache = cache; - SConnList* plist = taosHashGet(pCache, key, strlen(key)); + SHashObj* pPool = pool; + SConnList* plist = taosHashGet(pPool, key, strlen(key)); if (plist == NULL) { SConnList list; - taosHashPut(pCache, key, strlen(key), (void*)&list, sizeof(list)); - plist = taosHashGet(pCache, key, strlen(key)); + taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list)); + plist = taosHashGet(pPool, key, strlen(key)); QUEUE_INIT(&plist->conn); } @@ -176,14 +181,14 @@ static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { QUEUE_REMOVE(h); return QUEUE_DATA(h, SCliConn, conn); } -static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { +static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { char key[128] = {0}; tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); - SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); + SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -327,7 +332,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { et = taosGetTimestampUs(); STransConnCtx* pCtx = pMsg->ctx; - SCliConn* conn = getConnFromCache(pThrd->cache, pCtx->ip, pCtx->port); + SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later conn->data = pMsg; @@ -378,9 +383,9 @@ static void clientAsyncCb(uv_async_t* handle) { SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); clientHandleReq(pMsg, pThrd); count++; - if (count >= 2) { - tDebug("send batch size: %d", count); - } + } + if (count >= 2) { + tDebug("already process batch size: %d", count); } } @@ -398,24 +403,8 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); - - QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); - - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pThrd->loop); - - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; - - pThrd->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pThrd->loop, pThrd->pTimer); - pThrd->pTimer->data = pThrd; - + SCliThrdObj* pThrd = createThrdObj(); pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); - pThrd->cache = connCacheCreate(1); pThrd->pTransInst = shandle; int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); @@ -430,7 +419,30 @@ static void clientMsgDestroy(SCliMsg* pMsg) { // impl later free(pMsg); } +static SCliThrdObj* createThrdObj() { + SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + + QUEUE_INIT(&pThrd->msg); + pthread_mutex_init(&pThrd->msgMtx, NULL); + + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); + + pThrd->cliAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); + pThrd->cliAsync->data = pThrd; + + pThrd->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pThrd->pTimer); + pThrd->pTimer->data = pThrd; + + pThrd->pool = connPoolCreate(1); + return pThrd; +} static void destroyThrdObj(SCliThrdObj* pThrd) { + if (pThrd == NULL) { + return; + } pthread_join(pThrd->thread, NULL); pthread_mutex_destroy(&pThrd->msgMtx); free(pThrd->cliAsync); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 77b5f635f4..dafb809e2a 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -91,10 +91,14 @@ static SConn* connCreate(); static void connDestroy(SConn* conn); static void uvConnDestroy(uv_handle_t* handle); -// server worke thread +// server and worker thread static void* workerThread(void* arg); static void* acceptThread(void* arg); +// add handle loop +static bool addHandleToWorkloop(void* arg); +static bool addHandleToAcceptloop(void* arg); + void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { /* * formate of data buffer: @@ -268,7 +272,7 @@ static void uvProcessData(SConn* pConn) { rpcMsg.handle = pConn; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); - uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); + // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth // validate msg type } @@ -451,22 +455,14 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { void* acceptThread(void* arg) { // opt SServerObj* srv = (SServerObj*)arg; - uv_tcp_init(srv->loop, &srv->server); - - struct sockaddr_in bind_addr; - - uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); - uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); - int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { - tError("Listen error %s\n", uv_err_name(err)); - return NULL; - } uv_run(srv->loop, UV_RUN_DEFAULT); } -static void initWorkThrdObj(SWorkThrdObj* pThrd) { +static bool addHandleToWorkloop(void* arg) { + SWorkThrdObj* pThrd = arg; pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pThrd->loop); + if (0 != uv_loop_init(pThrd->loop)) { + return false; + } // SRpcInfo* pRpc = pThrd->shandle; uv_pipe_init(pThrd->loop, pThrd->pipe, 1); @@ -482,6 +478,31 @@ static void initWorkThrdObj(SWorkThrdObj* pThrd) { pThrd->workerAsync->data = pThrd; uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + return true; +} + +static bool addHandleToAcceptloop(void* arg) { + // impl later + SServerObj* srv = arg; + + int err = 0; + if ((err = uv_tcp_init(srv->loop, &srv->server)) != 0) { + tError("failed to init accept server: %s", uv_err_name(err)); + return false; + } + + struct sockaddr_in bind_addr; + + uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); + if ((err = uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0)) != 0) { + tError("failed to bind: %s", uv_err_name(err)); + return false; + } + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + tError("failed to listen: %s", uv_err_name(err)); + return false; + } + return true; } void* workerThread(void* arg) { SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; @@ -546,11 +567,12 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { - return NULL; + goto End; } uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write @@ -559,7 +581,9 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, thrd->fd = fds[0]; thrd->pipe = &(srv->pipe[i][1]); // init read - initWorkThrdObj(thrd); + if (false == addHandleToWorkloop(thrd)) { + goto End; + } int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { tDebug("sucess to create worker-thread %d", i); @@ -568,9 +592,10 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, // TODO: clear all other resource later tError("failed to create worker-thread %d", i); } - srv->pThreadObj[i] = thrd; } - + if (false == addHandleToAcceptloop(srv)) { + goto End; + } int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); if (err == 0) { tDebug("success to create accept-thread"); @@ -579,16 +604,25 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } return srv; +End: + taosCloseServer(srv); + return NULL; +} + +void destroyWorkThrd(SWorkThrdObj* pThrd) { + if (pThrd == NULL) { + return; + } + pthread_join(pThrd->thread, NULL); + // free(srv->pipe[i]); + free(pThrd->loop); + free(pThrd); } void taosCloseServer(void* arg) { // impl later SServerObj* srv = arg; for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* pThrd = srv->pThreadObj[i]; - pthread_join(pThrd->thread, NULL); - free(srv->pipe[i]); - free(pThrd->loop); - free(pThrd); + destroyWorkThrd(srv->pThreadObj[i]); } free(srv->loop); free(srv->pipe); From cc04f06bd533c8e7cfc62be673db09044e1c1ff4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 22 Jan 2022 12:43:46 +0800 Subject: [PATCH 062/118] minor changes --- source/dnode/vnode/src/vnd/vnodeWrite.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index dcdc03adf6..a6c3f25c6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -96,7 +96,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } - vDebug("vgId:%d process create %"PRIzu" tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); + vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); break; @@ -131,6 +131,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } + return 0; } From 3ba31879121cc5e713d6639ab273a1c62e168848 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 22 Jan 2022 13:03:49 +0800 Subject: [PATCH 063/118] minor changes --- source/libs/scheduler/src/scheduler.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 46450d78ea..fdeab3f138 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -738,7 +738,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); if (taskDone < pTask->level->taskNum) { - SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); + SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { From 9ef605faa00f679a88d16142b2c83ee81f3cee39 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Jan 2022 14:24:09 +0800 Subject: [PATCH 064/118] [td-11818]fix memory leak. --- source/dnode/vnode/src/vnd/vnodeQuery.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 9390e19523..4012914723 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -156,6 +156,11 @@ _exit: return 0; } +static void freeItemHelper(void* pItem) { + char* p = *(char**)pItem; + free(p); +} + /** * @param pVnode * @param pMsg @@ -169,17 +174,20 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { int32_t totalLen = 0; int32_t numOfTables = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { - if (numOfTables < 1000) { // TODO: temp get tables of vnode, and should del when show tables commad ok. + if (numOfTables < 10000) { // TODO: temp get tables of vnode, and should del when show tables commad ok. taosArrayPush(pArray, &name); totalLen += strlen(name); + } else { + tfree(name); } + numOfTables++; } // TODO: temp debug, and should del when show tables command ok - vError("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); - if (numOfTables > 1000) { - numOfTables = 1000; + vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); + if (numOfTables > 10000) { + numOfTables = 10000; } metaCloseTbCursor(pCur); @@ -214,6 +222,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); - taosArrayDestroy(pArray); + taosArrayDestroyEx(pArray, freeItemHelper); return 0; } From eb712702b9b747ffc65a5481f19d4120a31b7e19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Jan 2022 15:42:37 +0800 Subject: [PATCH 065/118] [td-11818]fix bug in select * from super_table. --- source/libs/executor/inc/executorimpl.h | 5 +- source/libs/executor/src/executorimpl.c | 151 +++++++++++++++--------- 2 files changed, 98 insertions(+), 58 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 73a30a62f5..ebf3d83a1a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -372,11 +372,14 @@ typedef struct STaskParam { typedef struct SExchangeInfo { SArray *pSources; - uint64_t bytes; // total load bytes from remote tsem_t ready; void *pTransporter; SRetrieveTableRsp *pRsp; SSDataBlock *pResult; + int32_t current; + uint64_t rowsOfCurrentSource; + uint64_t bytes; // total load bytes from remote + uint64_t totalRows; } SExchangeInfo; typedef struct STableScanInfo { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e963c49c86..02827ec32c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5144,72 +5144,110 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; *newgroup = false; - if (pExchangeInfo->pRsp != NULL && pExchangeInfo->pRsp->completed == 1) { + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + if (pExchangeInfo->current >= totalSources) { return NULL; } - SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { // todo handle malloc error - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _error; - } + SResFetchReq* pMsg = NULL; + SMsgSendInfo* pMsgSendInfo = NULL; - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, 0); - SEpSet epSet = {0}; - - epSet.numOfEps = pSource->addr.numOfEps; - epSet.port[0] = pSource->addr.epAddr[0].port; - tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("QID:%"PRIx64" calloc %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _error; - } - - pMsgSendInfo->param = pExchangeInfo; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = TDMT_VND_FETCH; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pExchangeInfo->ready); - - if (pExchangeInfo->pRsp->numOfRows == 0) { - return NULL; - } - - SSDataBlock* pRes = pExchangeInfo->pResult; - char* pData = pExchangeInfo->pRsp->data; - - for(int32_t i = 0; i < pOperator->numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); - char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pExchangeInfo->pRsp->numOfRows); - if (tmp == NULL) { + while(1) { + pMsg = calloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { // todo handle malloc error + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _error; } - size_t len = pExchangeInfo->pRsp->numOfRows * pColInfoData->info.bytes; - memcpy(tmp, pData, len); + SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + SEpSet epSet = {0}; - pColInfoData->pData = tmp; - pData += len; + epSet.numOfEps = pSource->addr.numOfEps; + epSet.port[0] = pSource->addr.epAddr[0].port; + tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); + + qDebug("QID:0x%" PRIx64 " build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + + // send the fetch remote task result reques + pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("QID:0x%" PRIx64 " prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _error; + } + + pMsgSendInfo->param = pExchangeInfo; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = TDMT_VND_FETCH; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pExchangeInfo->ready); + + SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; + if (pRsp->numOfRows == 0) { + if (pExchangeInfo->current >= taosArrayGetSize(pExchangeInfo->pSources)) { + return NULL; + } + + qDebug("QID:0x%"PRIx64" vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, + pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows); + + pExchangeInfo->rowsOfCurrentSource = 0; + pExchangeInfo->current += 1; + continue; + } + + SSDataBlock* pRes = pExchangeInfo->pResult; + char* pData = pRsp->data; + + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); + char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows); + if (tmp == NULL) { + goto _error; + } + + size_t len = pRsp->numOfRows * pColInfoData->info.bytes; + memcpy(tmp, pData, len); + + pColInfoData->pData = tmp; + pData += len; + } + + pRes->info.numOfCols = pOperator->numOfOutput; + pRes->info.rows = pRsp->numOfRows; + + pExchangeInfo->totalRows += pRsp->numOfRows; + pExchangeInfo->bytes += pRsp->compLen; + pExchangeInfo->rowsOfCurrentSource += pRsp->numOfRows; + + if (pRsp->completed == 1) { + qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->bytes, + pExchangeInfo->current + 1, totalSources); + + pExchangeInfo->rowsOfCurrentSource = 0; + pExchangeInfo->current += 1; + } else { + qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->bytes); + } + + return pExchangeInfo->pResult; } - pRes->info.numOfCols = pOperator->numOfOutput; - pRes->info.rows = pExchangeInfo->pRsp->numOfRows; - - return pExchangeInfo->pResult; - _error: tfree(pMsg); tfree(pMsgSendInfo); @@ -7719,7 +7757,6 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); } else if (pPhyNode->info.type == OP_StreamScan) { - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo); } From 3698980392921aa9f2e152f26bab635e5906a769 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 22 Jan 2022 18:45:03 +0800 Subject: [PATCH 066/118] refactor code and test reading half packet --- source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 73 ++++++++++++++++------------ source/libs/transport/src/transSrv.c | 50 ++++++++++--------- source/libs/transport/test/rclient.c | 8 +-- source/libs/transport/test/rserver.c | 2 +- 5 files changed, 78 insertions(+), 56 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 4b490936cc..bc9a9de318 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -32,6 +32,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->connType = pInit->connType; + pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ffd8d35bfc..bfadfe56ef 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,7 +17,7 @@ #include "transComm.h" -#define CONN_PERSIST_TIME(para) (para * 1000 * 100) +#define CONN_PERSIST_TIME(para) (para * 1000 * 10) typedef struct SCliConn { uv_connect_t connReq; @@ -65,15 +65,15 @@ typedef struct SConnList { // conn pool // add expire timeout and capacity limit -static void* connPoolCreate(int size); -static void* connPoolDestroy(void* pool); +static void* creatConnPool(int size); +static void* destroyConnPool(void* pool); static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port); static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn); // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); // process data read from server, auth/decompress etc later -static void clientProcessData(SCliConn* conn); +static void clientHandleResp(SCliConn* conn); // check whether already read complete packet from server static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read @@ -86,9 +86,11 @@ static void clientWriteCb(uv_write_t* req, int status); static void clientConnCb(uv_connect_t* req, int status); static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); -static void clientConnDestroy(SCliConn* pConn); +static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void clientMsgDestroy(SCliMsg* pMsg); +// handle req from app +static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); // thread obj static SCliThrdObj* createThrdObj(); @@ -96,9 +98,7 @@ static void destroyThrdObj(SCliThrdObj* pThrd); // thread static void* clientThread(void* arg); -static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); - -static void clientProcessData(SCliConn* conn) { +static void clientHandleResp(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; SRpcInfo* pRpc = pCtx->pRpc; SRpcMsg rpcMsg; @@ -131,7 +131,9 @@ static void clientTimeoutCb(uv_timer_t* handle) { SCliConn* c = QUEUE_DATA(h, SCliConn, conn); if (c->expireTime < currentTime) { QUEUE_REMOVE(h); - clientConnDestroy(c); + // uv_stream_t stm = *(c->stream); + // uv_close((uv_handle_t*)&stm, clientDestroy); + clientConnDestroy(c, true); } else { break; } @@ -142,18 +144,18 @@ static void clientTimeoutCb(uv_timer_t* handle) { pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); uv_timer_start(handle, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } -static void* connPoolCreate(int size) { - SHashObj* pool = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - return pool; +static void* creatConnPool(int size) { + // thread local, no lock + return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); } -static void* connPoolDestroy(void* pool) { +static void* destroyConnPool(void* pool) { SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { queue* h = QUEUE_HEAD(&connList->conn); QUEUE_REMOVE(h); SCliConn* c = QUEUE_DATA(h, SCliConn, conn); - clientConnDestroy(c); + clientConnDestroy(c, true); } connList = taosHashIterate((SHashObj*)pool, connList); } @@ -245,28 +247,37 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf pBuf->len += nread; if (clientReadComplete(pBuf)) { tDebug("alread read complete"); - clientProcessData(conn); + clientHandleResp(conn); } else { - tDebug("read halp packet, continue to read"); + tDebug("read half packet, continue to read"); } return; } - - if (nread != UV_EOF) { - tDebug("Read error %s\n", uv_err_name(nread)); + assert(nread <= 0); + if (nread == 0) { + return; } - // - uv_close((uv_handle_t*)handle, clientDestroy); + if (nread != UV_EOF) { + tDebug("read error %s", uv_err_name(nread)); + } + // tDebug("Read error %s\n", uv_err_name(nread)); + // uv_close((uv_handle_t*)handle, clientDestroy); } -static void clientConnDestroy(SCliConn* conn) { - // impl later - // +static void clientConnDestroy(SCliConn* conn, bool clear) { + tDebug("conn %p destroy", conn); + if (clear) { + uv_close((uv_handle_t*)conn->stream, NULL); + } + free(conn->stream); + free(conn->readBuf.buf); + free(conn->writeReq); + free(conn); } static void clientDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - QUEUE_REMOVE(&conn->conn); - clientConnDestroy(conn); + // QUEUE_REMOVE(&conn->conn); + clientConnDestroy(conn, false); } static void clientWriteCb(uv_write_t* req, int status) { @@ -274,7 +285,8 @@ static void clientWriteCb(uv_write_t* req, int status) { if (status == 0) { tDebug("data already was written on stream"); } else { - uv_close((uv_handle_t*)pConn->stream, clientDestroy); + tError("failed to write: %s", uv_err_name(status)); + clientConnDestroy(pConn, true); return; } SCliThrdObj* pThrd = pConn->hostThrd; @@ -317,7 +329,9 @@ static void clientConnCb(uv_connect_t* req, int status) { rpcMsg.ahandle = pCtx->ahandle; // SRpcInfo* pRpc = pMsg->ctx->pRpc; (pRpc->cfp)(NULL, &rpcMsg, NULL); - uv_close((uv_handle_t*)req->handle, clientDestroy); + + clientConnDestroy(pConn, true); + // uv_close((uv_handle_t*)req->handle, clientDestroy); return; } @@ -421,7 +435,6 @@ static void clientMsgDestroy(SCliMsg* pMsg) { } static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); - QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); @@ -436,7 +449,7 @@ static SCliThrdObj* createThrdObj() { uv_timer_init(pThrd->loop, pThrd->pTimer); pThrd->pTimer->data = pThrd; - pThrd->pool = connPoolCreate(1); + pThrd->pool = creatConnPool(1); return pThrd; } static void destroyThrdObj(SCliThrdObj* pThrd) { diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index dafb809e2a..b519a35f24 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -14,8 +14,8 @@ */ #ifdef USE_UV -#include "transComm.h" +#include "transComm.h" typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -84,12 +84,12 @@ static void uvWorkerAsyncCb(uv_async_t* handle); static void uvPrepareSendData(SConn* conn, uv_buf_t* wb); -// already read complete packet -static bool readComplete(SConnBuffer* buf); +// check whether already read complete packet +static bool readComplete(SConnBuffer* buf); +static SConn* createConn(); +static void destroyConn(SConn* conn, bool clear /*clear handle or not*/); -static SConn* connCreate(); -static void connDestroy(SConn* conn); -static void uvConnDestroy(uv_handle_t* handle); +static void uvDestroyConn(uv_handle_t* handle); // server and worker thread static void* workerThread(void* arg); @@ -283,6 +283,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; + tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread); if (readComplete(pBuf)) { tDebug("alread read complete packet"); uvProcessData(conn); @@ -291,11 +292,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } - if (nread != UV_EOF) { - tDebug("Read error %s", uv_err_name(nread)); + if (nread == 0) { + return; + } + if (nread != UV_EOF) { + tDebug("read error %s", uv_err_name(nread)); } - tDebug("read error %s", uv_err_name(nread)); - uv_close((uv_handle_t*)cli, uvConnDestroy); } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->base = malloc(sizeof(char)); @@ -318,7 +320,7 @@ void uvOnWriteCb(uv_write_t* req, int status) { tDebug("data already was written on stream"); } else { tDebug("failed to write data, %s", uv_err_name(status)); - connDestroy(conn); + destroyConn(conn, true); } // opt } @@ -331,7 +333,8 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { - // impl later + // impl later; + tDebug("prepare to send back"); SRpcMsg* pMsg = &conn->sendMsg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); @@ -394,6 +397,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); + free(cli); } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { @@ -403,7 +407,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tError("read error %s", uv_err_name(nread)); } // TODO(log other failure reason) - uv_close((uv_handle_t*)q, NULL); + // uv_close((uv_handle_t*)q, NULL); return; } // free memory allocated by @@ -422,7 +426,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConn* pConn = connCreate(); + SConn* pConn = createConn(); pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); @@ -448,7 +452,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { tDebug("failed to create new connection"); - connDestroy(pConn); + destroyConn(pConn, true); } } @@ -509,7 +513,7 @@ void* workerThread(void* arg) { uv_run(pThrd->loop, UV_RUN_DEFAULT); } -static SConn* connCreate() { +static SConn* createConn() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); return pConn; } @@ -517,22 +521,24 @@ static void connCloseCb(uv_handle_t* handle) { // impl later // } -static void connDestroy(SConn* conn) { +static void destroyConn(SConn* conn, bool clear) { if (conn == NULL) { return; } + if (clear) { + uv_handle_t handle = *((uv_handle_t*)conn->pTcp); + uv_close(&handle, NULL); + } uv_timer_stop(conn->pTimer); free(conn->pTimer); - // uv_close((uv_handle_t*)conn->pTcp, connCloseCb); free(conn->pTcp); free(conn->connBuf.buf); free(conn->pWriter); - // free(conn); - // handle + free(conn); } -static void uvConnDestroy(uv_handle_t* handle) { +static void uvDestroyConn(uv_handle_t* handle) { SConn* conn = handle->data; - connDestroy(conn); + destroyConn(conn, false); } static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) { STransMsgHead* pHead = (STransMsgHead*)msg; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 4ccbb60cc2..fd3496cc17 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -34,8 +34,8 @@ typedef struct { static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - // tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - // pMsg->code); + tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -63,6 +63,8 @@ static void *sendRequest(void *param) { if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); + tDebug("recv response"); + // usleep(100000000); } tDebug("thread:%d, it is over", pInfo->index); @@ -98,7 +100,7 @@ int main(int argc, char *argv[]) { rpcInit.numOfThreads = 1; rpcInit.cfp = processResponse; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = 100; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 2e32aa57ca..12d8a01819 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -122,7 +122,7 @@ int main(int argc, char *argv[]) { rpcInit.numOfThreads = 1; rpcInit.cfp = processRequestMsg; rpcInit.sessions = 1000; - rpcInit.idleTime = tsShellActivityTimer * 1500; + rpcInit.idleTime = 2 * 1500; rpcInit.afp = retrieveAuthInfo; for (int i = 1; i < argc; ++i) { From e6f05d3889c592ab3a8d48cb16ce34ecbc053ed6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 22 Jan 2022 22:50:00 +0800 Subject: [PATCH 067/118] feature/qnode --- source/libs/qworker/inc/qworkerInt.h | 1 + source/libs/qworker/src/qworker.c | 82 +++-- source/libs/qworker/src/qworkerMsg.c | 7 +- source/libs/qworker/test/qworkerTests.cpp | 375 +++++++++++++++++++--- source/libs/scheduler/src/scheduler.c | 2 +- 5 files changed, 409 insertions(+), 58 deletions(-) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 9ecce3f5f9..2765d7d5d7 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -111,6 +111,7 @@ typedef struct SQWTaskCtx { void *cancelConnection; bool emptyRes; + bool multiExec; int8_t queryContinue; int8_t queryInQueue; int32_t rspCode; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 021bd642bd..f1fd8aa6fb 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -53,6 +53,12 @@ int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { break; case JOB_TASK_STATUS_SUCCEED: + if (newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; case JOB_TASK_STATUS_FAILED: case JOB_TASK_STATUS_CANCELLING: if (newStatus != JOB_TASK_STATUS_CANCELLED) { @@ -249,7 +255,7 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { *ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { //QW_UNLOCK(rwType, &mgmt->ctxLock); - QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); + QW_TASK_DLOG_E("task ctx not exist, may be dropped"); QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } @@ -262,7 +268,7 @@ int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { - QW_TASK_ELOG("ctx not in ctxHash, ctxHashSize:%d", taosHashGetSize(mgmt->ctxHash)); + QW_TASK_DLOG_E("task ctx not exist, may be dropped"); QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } @@ -548,6 +554,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void return TSDB_CODE_SUCCESS; } + + pOutput->bufStatus = DS_BUF_EMPTY; QW_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd); @@ -605,10 +613,10 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_LOCK(QW_WRITE, &ctx->lock); locked = true; - atomic_store_32(&ctx->phase, phase); - switch (phase) { case QW_PHASE_PRE_QUERY: { + atomic_store_8(&ctx->phase, phase); + atomic_store_8(&ctx->taskType, input->taskType); if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { @@ -706,6 +714,8 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu break; } case QW_PHASE_PRE_CQUERY: { + atomic_store_8(&ctx->phase, phase); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); output->needStop = true; @@ -721,17 +731,33 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + QW_SET_RSP_CODE(ctx, output->rspCode); + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + + QW_ERR_JRET(output->rspCode); } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + qwFreeTask(QW_FPARAMS(), ctx); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_SET_RSP_CODE(ctx, output->rspCode); + cancelConnection = ctx->cancelConnection; + + QW_ERR_JRET(output->rspCode); } + if (ctx->rspCode) { QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); output->needStop = true; @@ -874,7 +900,9 @@ _return: QW_UPDATE_RSP_CODE(ctx, output->rspCode); } - atomic_store_32(&ctx->phase, phase); + if (QW_PHASE_POST_FETCH != phase) { + atomic_store_8(&ctx->phase, phase); + } if (locked) { QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -1063,6 +1091,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); atomic_store_8(&ctx->queryInQueue, 0); + atomic_store_8(&ctx->queryContinue, 0); DataSinkHandle sinkHandle = ctx->sinkHandle; @@ -1078,6 +1107,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { // RC WARNING atomic_store_8(&ctx->queryContinue, 1); } + + if (sOutput.queryEnd) { + needStop = true; + } if (rsp) { qwBuildFetchRsp(rsp, &sOutput, dataLen); @@ -1093,6 +1126,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { _return: + if (NULL == ctx) { + break; + } + if (code && QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); @@ -1101,12 +1138,18 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, 0); } - input.code = code; - qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); + QW_LOCK(QW_WRITE, &ctx->lock); + if (needStop || code || 0 == atomic_load_8(&ctx->queryContinue)) { + atomic_store_8(&ctx->phase, 0); + QW_UNLOCK(QW_WRITE,&ctx->lock); + break; + } + + QW_UNLOCK(QW_WRITE,&ctx->lock); + } while (true); - needStop = output.needStop; - code = output.rspCode; - } while ((!needStop) && (0 == code) && atomic_val_compare_exchange_8(&ctx->queryContinue, 1, 0)); + input.code = code; + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); QW_RET(code); } @@ -1159,7 +1202,10 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (QW_IS_QUERY_RUNNING(ctx)) { atomic_store_8(&ctx->queryContinue, 1); } else if (0 == atomic_load_8(&ctx->queryInQueue)) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + if (!ctx->multiExec) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + ctx->multiExec = true; + } atomic_store_8(&ctx->queryInQueue, 1); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index baa4ad2a04..d11bee6dce 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -50,6 +50,7 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_QUERY_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -68,6 +69,7 @@ int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_RES_READY_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -98,7 +100,7 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) { } SRpcMsg rpcRsp = { - .msgType = pMsg->msgType + 1, + .msgType = TDMT_VND_TASKS_STATUS_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -121,6 +123,7 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_ } SRpcMsg rpcRsp = { + .msgType = TDMT_VND_FETCH_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -138,6 +141,7 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_CANCEL_TASK_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -155,6 +159,7 @@ int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_DROP_TASK_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 362156ebcd..5812719c51 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -38,21 +38,25 @@ namespace { -#define qwtTestQueryQueueSize 1000 -#define qwtTestFetchQueueSize 1000 -#define qwtTestMaxExecTaskUsec 2 +#define qwtTestQueryQueueSize 1000000 +#define qwtTestFetchQueueSize 1000000 + +int32_t qwtTestMaxExecTaskUsec = 2; +int32_t qwtTestReqMaxDelayUsec = 2; uint64_t qwtTestQueryId = 0; bool qwtTestEnableSleep = true; bool qwtTestStop = false; -bool qwtTestDeadLoop = true; -int32_t qwtTestMTRunSec = 10; +bool qwtTestDeadLoop = false; +int32_t qwtTestMTRunSec = 60; int32_t qwtTestPrintNum = 100000; int32_t qwtTestCaseIdx = 0; int32_t qwtTestCaseNum = 4; bool qwtTestCaseFinished = false; tsem_t qwtTestQuerySem; tsem_t qwtTestFetchSem; +int32_t qwtTestQuitThreadNum = 0; + int32_t qwtTestQueryQueueRIdx = 0; int32_t qwtTestQueryQueueWIdx = 0; @@ -104,6 +108,7 @@ void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { qwtqueryMsg.sId = htobe64(1); qwtqueryMsg.taskId = htobe64(1); qwtqueryMsg.contentLen = htonl(100); + queryRpc->msgType = TDMT_VND_QUERY; queryRpc->pCont = &qwtqueryMsg; queryRpc->contLen = sizeof(SSubQueryMsg) + 100; } @@ -112,6 +117,7 @@ void qwtBuildReadyReqMsg(SResReadyReq *readyMsg, SRpcMsg *readyRpc) { readyMsg->sId = htobe64(1); readyMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); readyMsg->taskId = htobe64(1); + readyRpc->msgType = TDMT_VND_RES_READY; readyRpc->pCont = readyMsg; readyRpc->contLen = sizeof(SResReadyReq); } @@ -120,6 +126,7 @@ void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { fetchMsg->sId = htobe64(1); fetchMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); fetchMsg->taskId = htobe64(1); + fetchRpc->msgType = TDMT_VND_FETCH; fetchRpc->pCont = fetchMsg; fetchRpc->contLen = sizeof(SResFetchReq); } @@ -128,6 +135,7 @@ void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { dropMsg->sId = htobe64(1); dropMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); dropMsg->taskId = htobe64(1); + dropRpc->msgType = TDMT_VND_DROP_TASK; dropRpc->pCont = dropMsg; dropRpc->contLen = sizeof(STaskDropReq); } @@ -146,7 +154,9 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestFetchQueueLock); - qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = pMsg; + struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { qwtTestFetchQueueWIdx = 0; } @@ -167,7 +177,9 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestQueryQueueLock); - qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = pMsg; + struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { qwtTestQueryQueueWIdx = 0; } @@ -201,6 +213,7 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); } + rpcFreeCont(rsp); break; } case TDMT_VND_RES_READY_RSP: { @@ -213,6 +226,7 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); } + rpcFreeCont(rsp); break; } case TDMT_VND_FETCH_RSP: { @@ -226,16 +240,19 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); + rpcFreeCont(rsp); break; } - case TDMT_VND_DROP_TASK: { + case TDMT_VND_DROP_TASK_RSP: { STaskDropRsp *rsp = (STaskDropRsp *)pRsp->pCont; + rpcFreeCont(rsp); qwtTestCaseFinished = true; break; } } + return; } @@ -271,16 +288,30 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { *pRes = NULL; *useconds = 0; } else { + if (qwtTestSinkQueryEnd) { + *pRes = NULL; + *useconds = rand() % 10; + return 0; + } + endExec = rand() % 5; - if (endExec) { - usleep(rand() % qwtTestMaxExecTaskUsec); + int32_t runTime = 0; + if (qwtTestEnableSleep && qwtTestMaxExecTaskUsec > 0) { + runTime = rand() % qwtTestMaxExecTaskUsec; + } + + if (qwtTestEnableSleep) { + if (runTime) { + usleep(runTime); + } + } + if (endExec) { *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); (*pRes)->info.rows = rand() % 1000; } else { *pRes = NULL; - usleep(rand() % qwtTestMaxExecTaskUsec); *useconds = rand() % 10; } } @@ -308,9 +339,9 @@ int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* p qwtTestSinkBlockNum++; if (qwtTestSinkBlockNum >= qwtTestSinkMaxBlockNum) { - *pContinue = true; - } else { *pContinue = false; + } else { + *pContinue = true; } taosWUnLockLatch(&qwtTestSinkLock); @@ -653,7 +684,7 @@ void *statusThread(void *param) { } -void *clientThread(void *param) { +void *qwtclientThread(void *param) { int32_t code = 0; uint32_t n = 0; void *mgmt = param; @@ -672,15 +703,14 @@ void *clientThread(void *param) { usleep(1); } - if (qwtTestEnableSleep) { - usleep(rand()%5); - } if (++n % qwtTestPrintNum == 0) { - printf("query:%d\n", n); + printf("case run:%d\n", n); } } + atomic_add_fetch_32(&qwtTestQuitThreadNum, 1); + return NULL; } @@ -689,9 +719,13 @@ void *queryQueueThread(void *param) { SRpcMsg *queryRpc = NULL; void *mgmt = param; - while (!qwtTestStop) { + while (true) { tsem_wait(&qwtTestQuerySem); + if (qwtTestStop && qwtTestQueryQueueNum <= 0) { + break; + } + taosWLockLatch(&qwtTestQueryQueueLock); if (qwtTestQueryQueueNum <= 0 || qwtTestQueryQueueRIdx == qwtTestQueryQueueWIdx) { printf("query queue is empty\n"); @@ -707,6 +741,15 @@ void *queryQueueThread(void *param) { qwtTestQueryQueueNum--; taosWUnLockLatch(&qwtTestQueryQueueLock); + + if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { + int32_t delay = rand() % qwtTestReqMaxDelayUsec; + + if (delay) { + usleep(delay); + } + } + if (TDMT_VND_QUERY == queryRpc->msgType) { qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc); } else if (TDMT_VND_QUERY_CONTINUE == queryRpc->msgType) { @@ -715,8 +758,16 @@ void *queryQueueThread(void *param) { printf("unknown msg in query queue, type:%d\n", queryRpc->msgType); assert(0); } + + free(queryRpc); + + if (qwtTestStop && qwtTestQueryQueueNum <= 0) { + break; + } } + atomic_add_fetch_32(&qwtTestQuitThreadNum, 1); + return NULL; } @@ -725,7 +776,7 @@ void *fetchQueueThread(void *param) { SRpcMsg *fetchRpc = NULL; void *mgmt = param; - while (!qwtTestStop) { + while (true) { tsem_wait(&qwtTestFetchSem); taosWLockLatch(&qwtTestFetchQueueLock); @@ -743,23 +794,45 @@ void *fetchQueueThread(void *param) { qwtTestFetchQueueNum--; taosWUnLockLatch(&qwtTestFetchQueueLock); + if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { + int32_t delay = rand() % qwtTestReqMaxDelayUsec; + + if (delay) { + usleep(delay); + } + } + switch (fetchRpc->msgType) { case TDMT_VND_FETCH: qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc); + break; case TDMT_VND_RES_READY: qWorkerProcessReadyMsg(mockPointer, mgmt, fetchRpc); + break; case TDMT_VND_TASKS_STATUS: qWorkerProcessStatusMsg(mockPointer, mgmt, fetchRpc); + break; case TDMT_VND_CANCEL_TASK: qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc); + break; case TDMT_VND_DROP_TASK: qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc); + break; default: printf("unknown msg type:%d in fetch queue", fetchRpc->msgType); assert(0); + break; } + + free(fetchRpc); + + if (qwtTestStop && qwtTestFetchQueueNum <= 0) { + break; + } } + atomic_add_fetch_32(&qwtTestQuitThreadNum, 1); + return NULL; } @@ -767,6 +840,7 @@ void *fetchQueueThread(void *param) { } +#if 0 TEST(seqTest, normalCase) { void *mgmt = NULL; @@ -800,31 +874,15 @@ TEST(seqTest, normalCase) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); - qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); ASSERT_EQ(code, 0); - qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); ASSERT_EQ(code, 0); - qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); ASSERT_EQ(code, 0); - qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); @@ -914,19 +972,31 @@ TEST(seqTest, randCase) { printf("Ready,%d\n", t++); qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + if (qwtTestEnableSleep) { + usleep(1); + } } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); + if (qwtTestEnableSleep) { + usleep(1); + } } else if (r >= maxr * 3/5 && r < maxr * 4/5) { printf("Drop,%d\n", t++); qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); + if (qwtTestEnableSleep) { + usleep(1); + } } else if (r >= maxr * 4/5 && r < maxr-1) { printf("Status,%d\n", t++); qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); + if (qwtTestEnableSleep) { + usleep(1); + } } else { printf("QUIT RAND NOW"); break; @@ -976,7 +1046,236 @@ TEST(seqTest, multithreadRand) { qWorkerDestroy(&mgmt); } -TEST(rcTest, multithread) { +#endif + +TEST(rcTest, shortExecshortDelay) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + qwtTestStop = false; + qwtTestQuitThreadNum = 0; + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + qwtTestMaxExecTaskUsec = 0; + qwtTestReqMaxDelayUsec = 0; + + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, qwtclientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + + while (true) { + if (qwtTestQuitThreadNum == 3) { + break; + } + + sleep(3); + + tsem_post(&qwtTestQuerySem); + usleep(10); + } + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; + + qWorkerDestroy(&mgmt); +} + +TEST(rcTest, longExecshortDelay) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + qwtTestStop = false; + qwtTestQuitThreadNum = 0; + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + qwtTestMaxExecTaskUsec = 1000000; + qwtTestReqMaxDelayUsec = 0; + + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, qwtclientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + + + while (true) { + if (qwtTestQuitThreadNum == 3) { + break; + } + + sleep(3); + + tsem_post(&qwtTestQuerySem); + usleep(10); + } + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; + + qWorkerDestroy(&mgmt); +} + + +TEST(rcTest, shortExeclongDelay) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + qwtTestStop = false; + qwtTestQuitThreadNum = 0; + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + qwtTestMaxExecTaskUsec = 0; + qwtTestReqMaxDelayUsec = 1000000; + + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, qwtclientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + + + while (true) { + if (qwtTestQuitThreadNum == 3) { + break; + } + + sleep(3); + + tsem_post(&qwtTestQuerySem); + usleep(10); + } + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; + + qWorkerDestroy(&mgmt); +} + + +#if 0 +TEST(rcTest, dropTest) { void *mgmt = NULL; int32_t code = 0; void *mockPointer = (void *)0x1; @@ -1025,7 +1324,7 @@ TEST(rcTest, multithread) { qWorkerDestroy(&mgmt); } - +#endif int main(int argc, char** argv) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 0ad51d0b57..d5f7b85b1b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -891,7 +891,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } - case TDMT_VND_DROP_TASK: { + case TDMT_VND_DROP_TASK_RSP: { // SHOULD NEVER REACH HERE SCH_TASK_ELOG("invalid status to handle drop task rsp, ref:%d", atomic_load_32(&pJob->ref)); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); From f5916bcf478fe841acc68bcfa58bfb6008d542c0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Jan 2022 22:51:48 +0800 Subject: [PATCH 068/118] [td-11818] fix bug in query. --- source/libs/executor/src/executorMain.c | 7 ------- source/libs/executor/src/executorimpl.c | 20 +++++++++++++------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index c5b57ee1a5..e5d56aca15 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -164,13 +164,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { return TSDB_CODE_SUCCESS; } - // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; - // if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { - // qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); - // setTaskStatus(pTaskInfo, TASK_COMPLETED); - // return doBuildResCheck(pTaskInfo); - // } - // error occurs, record the error code and return to client int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 02827ec32c..4ce652f5fd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4958,6 +4958,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + if (pTableScanInfo->pTsdbReadHandle == NULL) { + return NULL; + } + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -5161,8 +5165,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { } SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - SEpSet epSet = {0}; + SEpSet epSet = {0}; epSet.numOfEps = pSource->addr.numOfEps; epSet.port[0] = pSource->addr.epAddr[0].port; tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); @@ -5195,17 +5199,18 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; if (pRsp->numOfRows == 0) { - if (pExchangeInfo->current >= taosArrayGetSize(pExchangeInfo->pSources)) { - return NULL; - } - qDebug("QID:0x%"PRIx64" vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows); pExchangeInfo->rowsOfCurrentSource = 0; pExchangeInfo->current += 1; - continue; + + if (pExchangeInfo->current >= totalSources) { + return NULL; + } else { + continue; + } } SSDataBlock* pRes = pExchangeInfo->pResult; @@ -7823,11 +7828,12 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, if (groupInfo.numOfTables == 0) { code = 0; - // qDebug("no table qualified for query, reqId:0x%"PRIx64, (*pTask)->id.queryId); + qDebug("no table qualified for query, reqId:0x%"PRIx64, queryId); goto _error; } return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId); + _error: terrno = code; return NULL; From cf4b5594d43e45813388d253bf0271950a9f6c59 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Jan 2022 22:59:19 +0800 Subject: [PATCH 069/118] [td-11818] fix crash in show. --- source/client/test/clientTests.cpp | 104 ++++++++++++------------- source/libs/parser/src/dCDAstProcess.c | 8 +- 2 files changed, 58 insertions(+), 54 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 854f18bd79..ee23567705 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -567,58 +567,58 @@ TEST(testCase, show_table_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// -//TEST(testCase, projection_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -//// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// for(int32_t i = 0; i < 100; ++i) { -//// char sql[512] = {0}; -//// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); -//// TAOS_RES* p = taos_query(pConn, sql); -//// if (taos_errno(p) != 0) { -//// printf("failed to insert data, reason:%s\n", taos_errstr(p)); -//// } -//// -//// taos_free_result(p); -//// } -// -// pRes = taos_query(pConn, "select * from tu"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, projection_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 100000; ++i) { + char sql[512] = {0}; + sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } + + pRes = taos_query(pConn, "select * from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 994875c0a3..06a9a3d16e 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -26,7 +26,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out const char* msg4 = "pattern is invalid"; const char* msg5 = "database name is empty"; const char* msg6 = "pattern string is empty"; - const char* msg7 = "db is not specified"; + const char* msg7 = "database not specified"; /* * database prefix in pInfo->pMiscInfo->a[0] * wildcard in like clause in pInfo->pMiscInfo->a[1] @@ -50,7 +50,11 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); - catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); From 24ecc277fabdd134d20ba8b81289d8c963fd7779 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 23 Jan 2022 12:45:54 +0800 Subject: [PATCH 070/118] check password --- source/dnode/mgmt/impl/src/dndTransport.c | 10 +++++++--- source/dnode/mnode/impl/src/mndAuth.c | 20 +++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 104e702afb..cd78c33bfe 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -179,9 +179,13 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; - rpcInit.secret = INTERNAL_SECRET; + rpcInit.spi = 1; rpcInit.parent = pDnode; + char pass[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + pMgmt->clientRpc = rpcOpen(&rpcInit); if (pMgmt->clientRpc == NULL) { dError("failed to init rpc client"); @@ -260,7 +264,7 @@ static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *e char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; return 0; @@ -269,7 +273,7 @@ static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *e char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; return 0; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 3e9ab99a45..15d2734841 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAuth.h" +#include "mndUser.h" static int32_t mndProcessAuthReq(SMnodeMsg *pReq); @@ -25,7 +26,24 @@ int32_t mndInitAuth(SMnode *pMnode) { void mndCleanupAuth(SMnode *pMnode) {} -int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } +int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SUserObj *pUser = mndAcquireUser(pMnode, user); + if (pUser == NULL) { + *secret = 0; + mError("user:%s, failed to auth user since %s", user, terrstr()); + return -1; + } + + *spi = 0; + *encrypt = 0; + *ckey = 0; + + memcpy(secret, pUser->pass, TSDB_PASSWORD_LEN); + mndReleaseUser(pMnode, pUser); + + mDebug("user:%s, auth info is returned", user); + return 0; +} static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { SAuthReq *pAuth = pReq->rpcMsg.pCont; From 09062a9ce70da532cdc1f07d9923068f73a82ed6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 23 Jan 2022 19:50:18 +0800 Subject: [PATCH 071/118] check user password --- source/client/src/clientEnv.c | 2 +- source/dnode/mgmt/impl/src/dndTransport.c | 14 ++++++-------- source/dnode/mgmt/impl/test/sut/src/client.cpp | 6 +++--- source/dnode/mnode/impl/src/mndAuth.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 11 ++++++----- source/dnode/mnode/sdb/src/sdbHash.c | 1 + source/libs/executor/src/executorimpl.c | 2 +- tools/shell/src/backup/tnettest.c | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d2696fb355..0780593e7e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -110,7 +110,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; -// rpcInit.spi = 1; + rpcInit.spi = 1; rpcInit.secret = (char *)auth; void* pDnodeConn = rpcOpen(&rpcInit); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index cd78c33bfe..79464347f0 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -182,8 +182,8 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.spi = 1; rpcInit.parent = pDnode; - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); rpcInit.secret = pass; pMgmt->clientRpc = rpcOpen(&rpcInit); @@ -260,18 +260,16 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { - // A simple temporary implementation - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 1; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - // A simple temporary implementation - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 1; *encrypt = 0; diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index fd5dcd77d9..3d61db8268 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -27,8 +27,8 @@ void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; tsem_t* TestClient::GetSem() { return &sem; } bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { - char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); @@ -42,7 +42,7 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint rpcInit.ckey = (char*)"key"; rpcInit.parent = this; rpcInit.secret = (char*)secretEncrypt; - // rpcInit.spi = 1; + rpcInit.spi = 1; clientRpc = rpcOpen(&rpcInit); ASSERT(clientRpc); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 15d2734841..44a5836bd6 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -34,7 +34,7 @@ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, cha return -1; } - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ad378953eb..c36e128893 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -60,9 +60,9 @@ void mndCleanupUser(SMnode *pMnode) {} static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; @@ -202,7 +202,7 @@ SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + terrno = TSDB_CODE_MND_USER_NOT_EXIST; } return pUser; } @@ -214,9 +214,9 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) { SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; userObj.superUser = 0; @@ -351,8 +351,9 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { SUserObj newUser = {0}; memcpy(&newUser, pUser, sizeof(SUserObj)); - memset(pUser->pass, 0, sizeof(pUser->pass)); - taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)pAlter->pass, strlen(pAlter->pass), pass); + memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN); newUser.updateTime = taosGetTimestampMs(); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index a9267b0ea3..62472c8c8e 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -152,6 +152,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); if (code != 0) { + code = terrno; taosWLockLatch(pLock); taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4ce652f5fd..2645b3eac8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5302,7 +5302,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.user = (char *)"root"; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; -// rpcInit.spi = 1; + rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; pInfo->pTransporter = rpcOpen(&rpcInit); diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index 7b5dbd2405..be8714387f 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -319,7 +319,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { char user[] = "nettestinternal"; char pass[] = "nettestinternal"; - taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; @@ -344,7 +344,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p SRpcMsg rspMsg; void * pRpcConn; - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; pRpcConn = taosNetInitRpc(secretEncrypt, spi); if (NULL == pRpcConn) { From 8364c8078b88b329606a348aa21342583a98865f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 23 Jan 2022 19:58:53 +0800 Subject: [PATCH 072/118] minor changes --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7d5199dfaf..9a0a6a11e9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -562,7 +562,7 @@ STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uin tsem_wait(&pRequest->body.rspSem); if (pRequest->code != TSDB_CODE_SUCCESS) { - const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); printf("failed to connect to server, reason: %s\n\n", errorMsg); destroyRequest(pRequest); From 2b135f6eefede22f8505d60e6e9dc034752420de Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 20:27:00 +0800 Subject: [PATCH 073/118] minor changes --- source/common/src/tglobal.c | 2 +- tests/script/sh/deploy.sh | 2 ++ tests/script/tmp/prepare.sim | 42 ++++++++++++++++++++++++++++++++---- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 40e278f54a..7a0966df57 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -44,7 +44,7 @@ int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 5000; +int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 3e8f87d665..ea1ad65a5b 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,6 +113,8 @@ elif [ $NODE -eq 7 ]; then NODE=7700 elif [ $NODE -eq 8 ]; then NODE=7800 +elif [ $NODE -eq 9 ]; then + NODE=7900 fi HOSTNAME=localhost diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 28675be808..3b43656a41 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -5,7 +5,41 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 -system sh/cfg.sh -n dnode4 -c walLevel -v 2 +return + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/deploy.sh -n dnode7 -i 7 +system sh/deploy.sh -n dnode8 -i 8 +system sh/deploy.sh -n dnode9 -i 9 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +system sh/exec.sh -n dnode7 -s start +system sh/exec.sh -n dnode8 -s start +system sh/exec.sh -n dnode9 -s start + +sleep 2000 + +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 +sql create dnode $hostname port 7700 +sql create dnode $hostname port 7800 +sql create dnode $hostname port 7900 + +sql show dnodes; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 \ No newline at end of file From eb4cf744b90be4b835b1174eac94509472c8afdb Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 20:28:26 +0800 Subject: [PATCH 074/118] rename files --- tests/script/{sim => tsim}/db/basic1.sim | 0 tests/script/{sim => tsim}/db/basic6.sim | 0 tests/script/{sim => tsim}/db/error1.sim | 0 tests/script/{sim => tsim}/dnode/basic1.sim | 0 tests/script/{sim => tsim}/table/basic1.sim | 0 tests/script/{sim => tsim}/user/basic1.sim | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename tests/script/{sim => tsim}/db/basic1.sim (100%) rename tests/script/{sim => tsim}/db/basic6.sim (100%) rename tests/script/{sim => tsim}/db/error1.sim (100%) rename tests/script/{sim => tsim}/dnode/basic1.sim (100%) rename tests/script/{sim => tsim}/table/basic1.sim (100%) rename tests/script/{sim => tsim}/user/basic1.sim (100%) diff --git a/tests/script/sim/db/basic1.sim b/tests/script/tsim/db/basic1.sim similarity index 100% rename from tests/script/sim/db/basic1.sim rename to tests/script/tsim/db/basic1.sim diff --git a/tests/script/sim/db/basic6.sim b/tests/script/tsim/db/basic6.sim similarity index 100% rename from tests/script/sim/db/basic6.sim rename to tests/script/tsim/db/basic6.sim diff --git a/tests/script/sim/db/error1.sim b/tests/script/tsim/db/error1.sim similarity index 100% rename from tests/script/sim/db/error1.sim rename to tests/script/tsim/db/error1.sim diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim similarity index 100% rename from tests/script/sim/dnode/basic1.sim rename to tests/script/tsim/dnode/basic1.sim diff --git a/tests/script/sim/table/basic1.sim b/tests/script/tsim/table/basic1.sim similarity index 100% rename from tests/script/sim/table/basic1.sim rename to tests/script/tsim/table/basic1.sim diff --git a/tests/script/sim/user/basic1.sim b/tests/script/tsim/user/basic1.sim similarity index 100% rename from tests/script/sim/user/basic1.sim rename to tests/script/tsim/user/basic1.sim From b72b6895bc2e9753f6d28bf9b909f1b89b3e96c6 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 20:47:13 +0800 Subject: [PATCH 075/118] minor changes --- tests/script/jenkins/basic.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bb6569deb0..a27f5326c9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,17 +2,17 @@ #======================b1-start=============== # ---- user -./test.sh -f sim/user/basic1.sim +./test.sh -f tsim/user/basic1.sim # ---- db -./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/basic6.sim -./test.sh -f sim/db/error1.sim +./test.sh -f tsim/db/basic1.sim +./test.sh -f tsim/db/basic6.sim +./test.sh -f tsim/db/error1.sim # ---- table -./test.sh -f sim/table/basic1.sim +./test.sh -f tsim/table/basic1.sim # ---- dnode -./test.sh -f sim/dnode/basic1.sim +./test.sh -f tsim/dnode/basic1.sim #======================b1-end=============== From 54e59090ab52db3505fadfcbd1047cd07b5610e3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Jan 2022 09:43:27 +0800 Subject: [PATCH 076/118] feature/qnode --- source/libs/qworker/inc/qworkerInt.h | 2 - source/libs/qworker/src/qworker.c | 37 ++++++++++------- source/libs/qworker/test/qworkerTests.cpp | 49 +++++++++++++++++------ 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 2765d7d5d7..46e46d323e 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -86,8 +86,6 @@ typedef struct SQWPhaseInput { int8_t taskStatus; int8_t taskType; int32_t code; - qTaskInfo_t taskHandle; - DataSinkHandle sinkHandle; } SQWPhaseInput; typedef struct SQWPhaseOutput { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index f1fd8aa6fb..63ae1e71f0 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -456,13 +456,15 @@ _return: QW_RET(code); } -int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType, bool shortRun) { +int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; uint64_t useconds = 0; int32_t i = 0; - int32_t leftRun = QW_DEFAULT_SHORT_RUN_TIMES; + int32_t execNum = 0; + qTaskInfo_t *taskHandle = &ctx->taskHandle; + DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); @@ -473,12 +475,14 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH QW_ERR_JRET(code); } + ++execNum; + if (NULL == pRes) { QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); dsEndPut(sinkHandle, useconds); - if (TASK_TYPE_TEMP == taskType) { + if (TASK_TYPE_TEMP == ctx->taskType) { qwFreeTaskHandle(QW_FPARAMS(), taskHandle); } @@ -498,7 +502,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH break; } - if (shortRun && ((--leftRun) <= 0)) { + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY) && execNum >= QW_DEFAULT_SHORT_RUN_TIMES) { + break; + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { break; } } @@ -617,8 +625,6 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu case QW_PHASE_PRE_QUERY: { atomic_store_8(&ctx->phase, phase); - atomic_store_8(&ctx->taskType, input->taskType); - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_ELOG("task already cancelled/dropped at wrong phase, phase:%d", phase); @@ -842,9 +848,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp } if (QW_PHASE_POST_QUERY == phase) { - ctx->taskHandle = input->taskHandle; - ctx->sinkHandle = input->sinkHandle; - if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { ctx->emptyRes = true; } @@ -949,8 +952,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { SQWPhaseOutput output = {0}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; - - input.taskType = taskType; + SQWTaskCtx *ctx = NULL; QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); @@ -961,6 +963,10 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_QUERY); QW_ERR_JRET(code); } + + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + atomic_store_8(&ctx->taskType, taskType); code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { @@ -986,8 +992,11 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { queryRsped = true; + atomic_store_ptr(&ctx->taskHandle, pTaskInfo); + atomic_store_ptr(&ctx->sinkHandle, sinkHandle); + if (pTaskInfo && sinkHandle) { - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &pTaskInfo, sinkHandle, taskType, true)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx)); } _return: @@ -1002,8 +1011,6 @@ _return: } input.code = rspCode; - input.taskHandle = pTaskInfo; - input.sinkHandle = sinkHandle; input.taskStatus = rspCode ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_PARTIAL_SUCCEED; QW_ERR_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); @@ -1095,7 +1102,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx->taskHandle, sinkHandle, ctx->taskType, QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH))); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 5812719c51..23a742c6ec 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -722,7 +722,7 @@ void *queryQueueThread(void *param) { while (true) { tsem_wait(&qwtTestQuerySem); - if (qwtTestStop && qwtTestQueryQueueNum <= 0) { + if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { break; } @@ -761,7 +761,7 @@ void *queryQueueThread(void *param) { free(queryRpc); - if (qwtTestStop && qwtTestQueryQueueNum <= 0) { + if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { break; } } @@ -779,6 +779,10 @@ void *fetchQueueThread(void *param) { while (true) { tsem_wait(&qwtTestFetchSem); + if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { + break; + } + taosWLockLatch(&qwtTestFetchQueueLock); if (qwtTestFetchQueueNum <= 0 || qwtTestFetchQueueRIdx == qwtTestFetchQueueWIdx) { printf("Fetch queue is empty\n"); @@ -826,7 +830,7 @@ void *fetchQueueThread(void *param) { free(fetchRpc); - if (qwtTestStop && qwtTestFetchQueueNum <= 0) { + if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { break; } } @@ -1104,10 +1108,17 @@ TEST(rcTest, shortExecshortDelay) { break; } - sleep(3); + sleep(1); + + if (qwtTestCaseFinished) { + if (qwtTestQuitThreadNum < 3) { + tsem_post(&qwtTestQuerySem); + tsem_post(&qwtTestFetchSem); + + usleep(10); + } + } - tsem_post(&qwtTestQuerySem); - usleep(10); } qwtTestQueryQueueNum = 0; @@ -1179,10 +1190,17 @@ TEST(rcTest, longExecshortDelay) { break; } - sleep(3); + sleep(1); + + if (qwtTestCaseFinished) { + if (qwtTestQuitThreadNum < 3) { + tsem_post(&qwtTestQuerySem); + tsem_post(&qwtTestFetchSem); + + usleep(10); + } + } - tsem_post(&qwtTestQuerySem); - usleep(10); } qwtTestQueryQueueNum = 0; @@ -1255,10 +1273,17 @@ TEST(rcTest, shortExeclongDelay) { break; } - sleep(3); + sleep(1); + + if (qwtTestCaseFinished) { + if (qwtTestQuitThreadNum < 3) { + tsem_post(&qwtTestQuerySem); + tsem_post(&qwtTestFetchSem); + + usleep(10); + } + } - tsem_post(&qwtTestQuerySem); - usleep(10); } qwtTestQueryQueueNum = 0; From 09568ff6552ea216e3fe5dbed24114913a33371a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Jan 2022 10:06:11 +0800 Subject: [PATCH 077/118] feature/qnode --- source/libs/scheduler/src/scheduler.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 15ef6b3d56..46851d722f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1743,3 +1743,4 @@ void schedulerDestroy(void) { } } + \ No newline at end of file From c7c702daabc9698346550ae3e507c0128905bae5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 10:07:35 +0800 Subject: [PATCH 078/118] [td-11818] update log. --- source/client/src/clientEnv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d2696fb355..1f2048608a 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -67,7 +67,9 @@ static void deregisterRequest(SRequestObj* pRequest) { int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + int64_t duration = taosGetTimestampMs() - pRequest->metric.start; + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, + pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } From a00a2b27bf0d3b8548d703769e77a0a23cb11783 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 10:16:45 +0800 Subject: [PATCH 079/118] [td-11818] update log. --- source/libs/catalog/src/catalog.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 43b8ae53a4..2197fdfd62 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -86,7 +86,7 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for use db, code:%x, db:%s", rpcRsp.code, input->db); + ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -258,7 +258,7 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport return TSDB_CODE_SUCCESS; } - ctgError("error rsp for stablemeta from mnode, code:%x, tbName:%s", rpcRsp.code, tbFullName); + ctgError("error rsp for stablemeta from mnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tbFullName); CTG_ERR_RET(rpcRsp.code); } @@ -320,18 +320,17 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, return TSDB_CODE_SUCCESS; } - ctgError("error rsp for table meta from vnode, code:%x, tbName:%s", rpcRsp.code, tNameGetTableName(pTableName)); + ctgError("error rsp for table meta from vnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, tbName:%s", tstrerror(code), tNameGetTableName(pTableName)); CTG_ERR_RET(code); } ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); - return TSDB_CODE_SUCCESS; } From 3b77216826072d3dd027904dbde17c348509f679 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 18:25:02 -0800 Subject: [PATCH 080/118] TD-12785 --- include/common/tmsg.h | 2 +- include/common/tmsgdef.h | 6 +- source/dnode/mgmt/impl/src/dndTransport.c | 6 +- source/dnode/mnode/impl/src/mndFunc.c | 68 +++++++++++------------ source/dnode/mnode/impl/src/mndShow.c | 2 +- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/sql.c | 2 +- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 857de3671f..4821e48785 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -100,7 +100,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, - TSDB_MGMT_TABLE_FUNCTION, + TSDB_MGMT_TABLE_FUNC, TSDB_MGMT_TABLE_MAX, } EShowType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2ff4d58e0b..26d8bc81eb 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -119,9 +119,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "mnode-alter-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "mnode-sync-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "mnode-compact-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNCTION, "mnode-create-function", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNCTION, "mnode-retrieve-function", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNCTION, "mnode-drop-function", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "mnode-create-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNC, "mnode-retrieve-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNC, "mnode-drop-func", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 79464347f0..d8b45b74eb 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -90,9 +90,9 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNCTION)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNCTION)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNCTION)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d406247bc1..ae3661f5d8 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -26,13 +26,13 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate); -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); -static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); -static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg); -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); +static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq); +static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -44,13 +44,13 @@ int32_t mndInitFunc(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndFuncActionUpdate, .deleteFp = (SdbDeleteFp)mndFuncActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNCTION, mndProcessCreateFuncMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNCTION, mndProcessDropFuncMsg); - mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNC, mndProcessCreateFuncReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNC, mndProcessDropFuncReq); + mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNC, mndProcessRetrieveFuncReq); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndGetFuncMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndRetrieveFuncs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndCancelGetNextFunc); return sdbSetTable(pMnode->pSdb, table); } @@ -156,7 +156,7 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNe return 0; } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate) { +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); pFunc->createdTime = taosGetTimestampMs(); pFunc->funcType = pCreate->funcType; @@ -172,7 +172,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pC pFunc->pCode = pFunc->pData + pCreate->commentSize; memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { free(pFunc); mError("func:%s, failed to create since %s", pCreate->name, terrstr()); @@ -219,8 +219,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pC return 0; } -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); return -1; @@ -261,10 +261,10 @@ static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { return 0; } -static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; - SCreateFuncReq *pCreate = pMsg->rpcMsg.pCont; + SCreateFuncReq *pCreate = pReq->rpcMsg.pCont; pCreate->outputLen = htonl(pCreate->outputLen); pCreate->bufSize = htonl(pCreate->bufSize); pCreate->sigature = htobe64(pCreate->sigature); @@ -311,7 +311,7 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateFunc(pMnode, pMsg, pCreate); + int32_t code = mndCreateFunc(pMnode, pReq, pCreate); if (code != 0) { mError("func:%s, failed to create since %s", pCreate->name, terrstr()); @@ -321,9 +321,9 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropFuncReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SDropFuncReq *pDrop = pReq->rpcMsg.pCont; mDebug("func:%s, start to drop", pDrop->name); @@ -340,7 +340,7 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropFunc(pMnode, pMsg, pFunc); + int32_t code = mndDropFunc(pMnode, pReq, pFunc); if (code != 0) { mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); @@ -350,10 +350,10 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; - SRetrieveFuncReq *pRetrieve = pMsg->rpcMsg.pCont; + SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; @@ -389,14 +389,14 @@ static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; } - pMsg->pCont = pRetrieveRsp; - pMsg->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + pReq->pCont = pRetrieveRsp; + pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); return 0; } -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -477,8 +477,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 28fe0551c2..e2ddcee0e9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -296,7 +296,7 @@ char *mndShowStr(int32_t showType) { return "show streamtables"; case TSDB_MGMT_TABLE_TP: return "show topics"; - case TSDB_MGMT_TABLE_FUNCTION: + case TSDB_MGMT_TABLE_FUNC: return "show functions"; default: return "undefined"; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 6e91ad997c..410e4fb187 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -65,7 +65,7 @@ program ::= cmd. {} //////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} +cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index a6537998f7..6f63feb3c4 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2232,7 +2232,7 @@ static void yy_reduce( { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} break; case 3: /* cmd ::= SHOW FUNCTIONS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} break; case 4: /* cmd ::= SHOW MNODES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} From 989d35c79cee181a0dc5f0ae5fb340ac1e90165b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Jan 2022 10:29:11 +0800 Subject: [PATCH 081/118] feature/qnode --- source/libs/scheduler/src/scheduler.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 46851d722f..f031d37743 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -279,7 +279,7 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { int32_t idx = atomic_load_8(&pTask->candidateIdx); SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, idx); if (NULL == addr) { - SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", idx, taosArrayGetSize(pTask->candidateAddrs)); + SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", idx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -1743,4 +1743,3 @@ void schedulerDestroy(void) { } } - \ No newline at end of file From af406ab31a54f7ff2356da383de3f1ffeac1fd34 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 02:31:19 +0000 Subject: [PATCH 082/118] no refact --- cmake/cmake.options | 2 +- source/libs/scheduler/src/scheduler.c | 577 +++++++++++++------------- 2 files changed, 298 insertions(+), 281 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 4883b6ee8e..faa45256fb 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -47,7 +47,7 @@ option( option( BUILD_WITH_UV "If build with libuv" - ON + OFF ) option( diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a6b499aca8..f31a27cb42 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -13,18 +13,20 @@ * along with this program. If not, see . */ -#include "catalog.h" -#include "query.h" #include "schedulerInt.h" #include "tmsg.h" +#include "query.h" +#include "catalog.h" static SSchedulerMgmt schMgmt = {0}; -uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } +uint64_t schGenTaskId(void) { + return atomic_add_fetch_64(&schMgmt.taskId, 1); +} uint64_t schGenUUID(void) { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64]; @@ -36,17 +38,18 @@ uint64_t schGenUUID(void) { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; } -int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) { - pTask->plan = pPlan; - pTask->level = pLevel; + +int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { + pTask->plan = pPlan; + pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); @@ -58,7 +61,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * return TSDB_CODE_SUCCESS; } -void schFreeTask(SSchTask *pTask) { +void schFreeTask(SSchTask* pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -78,9 +81,10 @@ void schFreeTask(SSchTask *pTask) { } } + int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = atomic_load_32(&pTask->lastMsgType); - + switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: case TDMT_VND_SUBMIT_RSP: @@ -93,23 +97,22 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && - SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), - msgType); + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } break; default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); - + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } return TSDB_CODE_SUCCESS; } + int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; @@ -121,34 +124,37 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_NOT_START) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_NOT_START: if (newStatus != JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && - newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED && - newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_CANCELLING + && newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_SUCCEED && - newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_SUCCEED + && newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_SUCCEED: case JOB_TASK_STATUS_FAILED: @@ -156,13 +162,13 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { if (newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); break; - + default: SCH_JOB_ELOG("invalid job status:%d", oriStatus); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); @@ -185,22 +191,23 @@ _return: SCH_ERR_RET(code); } + int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { for (int32_t i = 0; i < pJob->levelNum; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); - + for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SSubplan *pPlan = pTask->plan; - int32_t childNum = pPlan->pChildren ? (int32_t)taosArrayGetSize(pPlan->pChildren) : 0; - int32_t parentNum = pPlan->pParents ? (int32_t)taosArrayGetSize(pPlan->pParents) : 0; + int32_t childNum = pPlan->pChildren ? (int32_t)taosArrayGetSize(pPlan->pChildren) : 0; + int32_t parentNum = pPlan->pParents ? (int32_t)taosArrayGetSize(pPlan->pParents) : 0; if (childNum > 0) { if (pJob->levelIdx == pLevel->level) { SCH_JOB_ELOG("invalid query plan, lowest level, childNum:%d", childNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pTask->children = taosArrayInit(childNum, POINTER_BYTES); if (NULL == pTask->children) { SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); @@ -227,7 +234,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("invalid task info, level:0, parentNum:%d", parentNum); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); if (NULL == pTask->parents) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); @@ -252,7 +259,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - } + } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); } @@ -267,6 +274,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { return TSDB_CODE_SUCCESS; } + int32_t schRecordTaskSucceedNode(SSchTask *pTask) { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); @@ -277,6 +285,7 @@ int32_t schRecordTaskSucceedNode(SSchTask *pTask) { return TSDB_CODE_SUCCESS; } + int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { if (NULL == taosArrayPush(pTask->execAddrs, addr)) { SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); @@ -286,25 +295,23 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad return TSDB_CODE_SUCCESS; } + int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t code = 0; pJob->queryId = pDag->queryId; - + if (pDag->numOfSubplans <= 0) { SCH_JOB_ELOG("invalid subplan num:%d", pDag->numOfSubplans); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + int32_t levelNum = (int32_t)taosArrayGetSize(pDag->pSubplans); if (levelNum <= 0) { SCH_JOB_ELOG("invalid level num:%d", levelNum); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SHashObj *planToTask = taosHashInit( - SCHEDULE_DEFAULT_TASK_NUMBER, - taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, - HASH_NO_LOCK); + SHashObj *planToTask = taosHashInit(SCHEDULE_DEFAULT_TASK_NUMBER, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_TASK_NUMBER); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -321,9 +328,9 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { pJob->subPlans = pDag->pSubplans; - SSchLevel level = {0}; - SArray * plans = NULL; - int32_t taskNum = 0; + SSchLevel level = {0}; + SArray *plans = NULL; + int32_t taskNum = 0; SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -335,9 +342,9 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { } pLevel = taosArrayGet(pJob->levels, i); - + pLevel->level = i; - + plans = taosArrayGetP(pDag->pSubplans, i); if (NULL == plans) { SCH_JOB_ELOG("empty level plan, level:%d", i); @@ -351,13 +358,13 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { } pLevel->taskNum = taskNum; - + pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + for (int32_t n = 0; n < taskNum; ++n) { SSubplan *plan = taosArrayGetP(plans, n); @@ -365,15 +372,15 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - + SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); - + void *p = taosArrayPush(pLevel->subTasks, &task); if (NULL == p) { SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -422,10 +429,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { int32_t nodeNum = 0; if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - + for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); - + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -440,14 +447,14 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_QRY_INVALID_INPUT; } - /* - for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { - strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); - epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; - - ++epSet->numOfEps; - } - */ +/* + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; + } +*/ return TSDB_CODE_SUCCESS; } @@ -459,7 +466,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("task already in execTask list, code:%x", code); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -478,11 +485,11 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in succTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -490,13 +497,13 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks)); - + return TSDB_CODE_SUCCESS; } int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = false; - + if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); } @@ -505,11 +512,11 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_WLOG("task already in failTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -517,10 +524,11 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks)); - + return TSDB_CODE_SUCCESS; } + int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); @@ -530,11 +538,11 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -542,10 +550,11 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { *moved = true; SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); - + return TSDB_CODE_SUCCESS; } + int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info @@ -560,7 +569,7 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) { // if already FAILED, no more processing SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, status)); - + if (errCode) { atomic_store_32(&pJob->errCode, errCode); } @@ -574,6 +583,8 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod assert(0); } + + // Note: no more error processing, handled in function internal int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_FAILED, errCode)); @@ -584,10 +595,11 @@ int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_DROPPING, errCode)); } + // Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; - + if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) { SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch)); return TSDB_CODE_SUCCESS; @@ -604,7 +616,7 @@ int32_t schFetchFromRemote(SSchJob *pJob) { SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); return TSDB_CODE_SUCCESS; - + _return: atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); @@ -614,16 +626,17 @@ _return: return code; } + // Note: no more error processing, handled in function internal int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t code = 0; - + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); if ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } - + if (atomic_load_8(&pJob->userFetch)) { SCH_ERR_JRET(schFetchFromRemote(pJob)); } @@ -644,15 +657,15 @@ int32_t schProcessOnDataFetched(SSchJob *job) { // Note: no more error processing, handled in function internal int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { - bool needRetry = false; - bool moved = false; + bool needRetry = false; + bool moved = false; int32_t taskDone = 0; int32_t code = 0; SCH_TASK_DLOG("taskOnFailure, code:%s", tstrerror(errCode)); SCH_ERR_JRET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry)); - + if (!needRetry) { SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode)); @@ -664,7 +677,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); - + if (SCH_TASK_NEED_WAIT_ALL(pTask)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskFailed++; @@ -672,7 +685,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); atomic_store_32(&pJob->errCode, errCode); - + if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("not all tasks done, done:%d, all:%d", taskDone, pTask->level->taskNum); SCH_ERR_RET(errCode); @@ -681,7 +694,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } else { // Note: no more error processing, already handled SCH_ERR_RET(schLaunchTask(pJob, pTask)); - + return TSDB_CODE_SUCCESS; } @@ -692,14 +705,15 @@ _return: SCH_ERR_RET(errCode); } + // Note: no more error processing, handled in function internal int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { - bool moved = false; - int32_t code = 0; + bool moved = false; + int32_t code = 0; SSchTask *pErrTask = pTask; SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); - + code = schMoveTaskToSuccList(pJob, pTask, &moved); if (code && moved) { SCH_ERR_RET(code); @@ -708,20 +722,20 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); - + int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { int32_t taskDone = 0; - + if (SCH_TASK_NEED_WAIT_ALL(pTask)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskSucceed++; taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); - + if (taskDone < pTask->level->taskNum) { - SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); - + SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); + return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { assert(0); @@ -742,32 +756,32 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (code && moved) { SCH_ERR_RET(code); } - + SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); return TSDB_CODE_SUCCESS; } - /* - if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { - strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); - job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; +/* + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { + strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); + job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; - ++job->dataSrcEps.numOfEps; - } - */ + ++job->dataSrcEps.numOfEps; + } +*/ for (int32_t i = 0; i < parentNum; ++i) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); pErrTask = par; - + atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - + if (SCH_TASK_READY_TO_LUNCH(par)) { SCH_ERR_RET(schLaunchTask(pJob, par)); } @@ -782,24 +796,23 @@ _return: SCH_ERR_RET(code); } -int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, - int32_t rspCode) { +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { int32_t code = 0; SCH_ERR_JRET(schValidateTaskReceivedMsgType(pJob, pTask, msgType)); switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: { - if (rspCode != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + } + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + + break; } - - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; - } case TDMT_VND_SUBMIT_RSP: { -#if 0 // TODO OPEN THIS + #if 0 //TODO OPEN THIS SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { @@ -807,69 +820,71 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } pJob->resNumOfRows += rsp->affectedRows; -#else - if (rspCode != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + #else + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + } + + SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg; + if (rsp) { + pJob->resNumOfRows += rsp->affectedRows; + } + #endif + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + + break; } - - SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg; - if (rsp) { - pJob->resNumOfRows += rsp->affectedRows; - } -#endif - - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; - } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + SQueryTableRsp *rsp = (SQueryTableRsp *)msg; + + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + } + + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + + break; } - - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); - - break; - } case TDMT_VND_RES_READY_RSP: { - SResReadyRsp *rsp = (SResReadyRsp *)msg; - - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); + SResReadyRsp *rsp = (SResReadyRsp *)msg; + + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + } + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + + break; } - - SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); - - break; - } case TDMT_VND_FETCH_RSP: { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); - } + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + } - if (pJob->res) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res); - tfree(rsp); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); - } + if (pJob->res) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res); + tfree(rsp); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + atomic_store_ptr(&pJob->res, rsp); + atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); + + if (rsp->completed) { + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + } SCH_ERR_JRET(schProcessOnDataFetched(pJob)); break; } - - SCH_ERR_JRET(schProcessOnDataFetched(pJob)); - - break; - } case TDMT_VND_DROP_TASK: { - // SHOULD NEVER REACH HERE - assert(0); - break; - } + // SHOULD NEVER REACH HERE + assert(0); + break; + } default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -880,19 +895,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch _return: SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); - + SCH_RET(code); } -int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { - int32_t code = 0; - SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchJob * pJob = NULL; - SSchTask * pTask = NULL; +int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { + int32_t code = 0; + SSchCallbackParam *pParam = (SSchCallbackParam *)param; + SSchJob *pJob = NULL; + SSchTask *pTask = NULL; + SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); if (NULL == job || NULL == (*job)) { - qError("QID:%" PRIx64 " taosHashGet queryId not exist, may be dropped", pParam->queryId); + qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -902,13 +918,13 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in int32_t s = taosHashGetSize(pJob->execTasks); if (s <= 0) { - qError("QID:%" PRIx64 ",TID:%" PRId64 " no task in execTask list", pParam->queryId, pParam->taskId); + qError("QID:%"PRIx64",TID:%"PRId64" no task in execTask list", pParam->queryId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { - qError("QID:%" PRIx64 ",TID:%" PRId64 " taosHashGet taskId not exist", pParam->queryId, pParam->taskId); + qError("QID:%"PRIx64",TID:%"PRId64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -927,29 +943,29 @@ _return: SCH_RET(code); } -int32_t schHandleSubmitCallback(void *param, const SDataBuf *pMsg, int32_t code) { +int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code); } -int32_t schHandleCreateTableCallback(void *param, const SDataBuf *pMsg, int32_t code) { +int32_t schHandleCreateTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code); } -int32_t schHandleQueryCallback(void *param, const SDataBuf *pMsg, int32_t code) { +int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code); } -int32_t schHandleFetchCallback(void *param, const SDataBuf *pMsg, int32_t code) { +int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code); } -int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) { +int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) { return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code); } -int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { +int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) { SSchCallbackParam *pParam = (SSchCallbackParam *)param; - qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); + qDebug("QID:%"PRIx64",TID:%"PRIx64" drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); } int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { @@ -957,16 +973,16 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_CREATE_TABLE: *fp = schHandleCreateTableCallback; break; - case TDMT_VND_SUBMIT: + case TDMT_VND_SUBMIT: *fp = schHandleSubmitCallback; break; - case TDMT_VND_QUERY: + case TDMT_VND_QUERY: *fp = schHandleQueryCallback; break; - case TDMT_VND_RES_READY: + case TDMT_VND_RES_READY: *fp = schHandleReadyCallback; break; - case TDMT_VND_FETCH: + case TDMT_VND_FETCH: *fp = schHandleFetchCallback; break; case TDMT_VND_DROP_TASK: @@ -980,18 +996,18 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { return TSDB_CODE_SUCCESS; } -int32_t schAsyncSendMsg(void *transport, SEpSet *epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, - uint32_t msgSize) { - int32_t code = 0; - SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + +int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, uint32_t msgSize) { + int32_t code = 0; + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - qError("QID:%" PRIx64 ",TID:%" PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SMsgSendInfo)); + qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); if (NULL == param) { - qError("QID:%" PRIx64 ",TID:%" PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SSchCallbackParam)); + qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", qId, tId, (int32_t)sizeof(SSchCallbackParam)); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1006,19 +1022,19 @@ int32_t schAsyncSendMsg(void *transport, SEpSet *epSet, uint64_t qId, uint64_t t pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - - int64_t transporterId = 0; - + + int64_t transporterId = 0; + code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo); if (code) { - qError("QID:%" PRIx64 ",TID:%" PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); + qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); SCH_ERR_JRET(code); } - + return TSDB_CODE_SUCCESS; _return: - + tfree(param); tfree(pMsgSendInfo); @@ -1028,7 +1044,7 @@ _return: void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { epSet->inUse = addr->inUse; epSet->numOfEps = addr->numOfEps; - + for (int8_t i = 0; i < epSet->numOfEps; ++i) { strncpy(epSet->fqdn[i], addr->epAddr[i].fqdn, sizeof(addr->epAddr[i].fqdn)); epSet->port[i] = addr->epAddr[i].port; @@ -1037,19 +1053,19 @@ void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void * msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; - SEpSet epSet; + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; + SEpSet epSet; if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); - + isCandidateAddr = true; } schConvertAddrToEpSet(addr, &epSet); - + switch (msgType) { case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { @@ -1075,7 +1091,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SSubQueryMsg *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); - + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); pMsg->taskId = htobe64(pTask->taskId); @@ -1094,12 +1110,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SResReadyReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } case TDMT_VND_FETCH: { @@ -1109,31 +1125,31 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SResFetchReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } - case TDMT_VND_DROP_TASK: { + case TDMT_VND_DROP_TASK:{ msgSize = sizeof(STaskDropReq); msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + STaskDropReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); + + pMsg->header.vgId = htonl(addr->nodeId); + + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskId = htobe64(pTask->taskId); break; } default: @@ -1149,7 +1165,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (isCandidateAddr) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); } - + return TSDB_CODE_SUCCESS; _return: @@ -1166,24 +1182,25 @@ static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { *pStatus = status; } - return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || - status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING); + return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED + || status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING); } + // Note: no more error processing, handled in function internal int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { - int8_t status = 0; + int8_t status = 0; int32_t code = 0; - + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_ELOG("no need to launch task cause of job status, job status:%d", status); - + code = atomic_load_32(&pJob->errCode); SCH_ERR_RET(code); - + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SSubplan *plan = pTask->plan; if (NULL == pTask->msg) { @@ -1195,7 +1212,7 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { // printf("physical plan:%s\n", pTask->msg); } - + SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); // NOTE: race condition: the task should be put into the hash table before send msg to server @@ -1206,13 +1223,13 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); - + return TSDB_CODE_SUCCESS; _return: SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); - + SCH_RET(code); } @@ -1220,12 +1237,12 @@ int32_t schLaunchJob(SSchJob *pJob) { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); - + for (int32_t i = 0; i < level->taskNum; ++i) { SSchTask *pTask = taosArrayGet(level->subTasks, i); SCH_ERR_RET(schLaunchTask(pJob, pTask)); } - + return TSDB_CODE_SUCCESS; } @@ -1236,7 +1253,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { } int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); - + if (size <= 0) { SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); return; @@ -1260,9 +1277,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { if (!SCH_TASK_NO_NEED_DROP(pTask)) { schDropTaskOnExecutedNode(pJob, pTask); } - + pIter = taosHashIterate(list, pIter); - } + } } void schDropJobAllTasks(SSchJob *pJob) { @@ -1271,15 +1288,15 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag *pDag, struct SSchJob **job, bool syncSchedule) { +int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { if (nodeList && taosArrayGetSize(nodeList) <= 0) { - qInfo("QID:%" PRIx64 " input nodeList is empty", pDag->queryId); + qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); } - int32_t code = 0; + int32_t code = 0; SSchJob *pJob = calloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:%"PRIx64" calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1289,22 +1306,19 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag *pDag, struc SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); - pJob->execTasks = - taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->execTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->succTasks = - taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->succTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->succTasks) { SCH_JOB_ELOG("taosHashInit %d succTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->failTasks = - taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + pJob->failTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->failTasks) { SCH_JOB_ELOG("taosHashInit %d failTasks failed", pDag->numOfSubplans); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1324,11 +1338,11 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag *pDag, struc } pJob->status = JOB_TASK_STATUS_NOT_START; - + SCH_ERR_JRET(schLaunchJob(pJob)); *(SSchJob **)job = pJob; - + if (syncSchedule) { SCH_JOB_DLOG("will wait for rsp now, job status:%d", SCH_GET_JOB_STATUS(pJob)); tsem_wait(&pJob->rspSem); @@ -1341,18 +1355,20 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag *pDag, struc _return: *(SSchJob **)job = NULL; - + scheduleFreeJob(pJob); - + SCH_RET(code); } int32_t schCancelJob(SSchJob *pJob) { - // TODO + //TODO + + //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST - // TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST } + int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobs) { qError("scheduler already initialized"); @@ -1361,7 +1377,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { if (cfg) { schMgmt.cfg = *cfg; - + if (schMgmt.cfg.maxJobNum == 0) { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } @@ -1369,8 +1385,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } - schMgmt.jobs = - taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + schMgmt.jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == schMgmt.jobs) { qError("init schduler jobs failed, num:%u", schMgmt.cfg.maxJobNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1381,12 +1396,12 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } - qInfo("scheduler %" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); - + qInfo("scheduler %"PRIx64" initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + return TSDB_CODE_SUCCESS; } -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag *pDag, struct SSchJob **pJob, SQueryResult *pRes) { +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1399,11 +1414,11 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag *pDag, stru pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - + return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag *pDag, struct SSchJob **pJob) { +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1412,7 +1427,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag *pDag, return TSDB_CODE_SUCCESS; } -int32_t schedulerConvertDagToTaskList(SQueryDag *pDag, SArray **pTasks) { +int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { if (NULL == pDag || pDag->numOfSubplans <= 0 || taosArrayGetSize(pDag->pSubplans) <= 0) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1437,10 +1452,10 @@ int32_t schedulerConvertDagToTaskList(SQueryDag *pDag, SArray **pTasks) { } STaskInfo tInfo = {0}; - char * msg = NULL; - int32_t msgLen = 0; - int32_t code = 0; - + char *msg = NULL; + int32_t msgLen = 0; + int32_t code = 0; + for (int32_t i = 0; i < taskNum; ++i) { SSubplan *plan = taosArrayGetP(plans, i); @@ -1481,7 +1496,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag *pDag, SArray **pTasks) { *pTasks = info; info = NULL; - + _return: schedulerFreeTaskList(info); @@ -1502,7 +1517,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgSize = src->msg->contentLen + sizeof(*src->msg); + int32_t msgSize = src->msg->contentLen + sizeof(*src->msg); STaskInfo info = {0}; info.addr = src->addr; @@ -1535,7 +1550,8 @@ _return: SCH_RET(code); } -int32_t scheduleFetchRows(SSchJob *pJob, void **pData) { + +int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1577,7 +1593,7 @@ int32_t scheduleFetchRows(SSchJob *pJob, void **pData) { SCH_JOB_ELOG("job failed or dropping, status:%d", status); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } - + if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } @@ -1629,7 +1645,7 @@ void scheduleFreeJob(void *job) { SSchJob *pJob = job; uint64_t queryId = pJob->queryId; - bool setJobFree = false; + bool setJobFree = false; if (SCH_GET_JOB_STATUS(pJob) > 0) { if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { @@ -1664,33 +1680,33 @@ void scheduleFreeJob(void *job) { schDropJobAllTasks(pJob); } - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans - + pJob->subPlans = NULL; // it is a reference to pDag->pSubplans + int32_t numOfLevels = taosArrayGetSize(pJob->levels); - for (int32_t i = 0; i < numOfLevels; ++i) { + for(int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); - for (int32_t j = 0; j < numOfTasks; ++j) { - SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + for(int32_t j = 0; j < numOfTasks; ++j) { + SSchTask* pTask = taosArrayGet(pLevel->subTasks, j); schFreeTask(pTask); } taosArrayDestroy(pLevel->subTasks); } - + taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->succTasks); - + taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); tfree(pJob->res); - + tfree(pJob); - qDebug("QID:%" PRIx64 " job freed", queryId); + qDebug("QID:%"PRIx64" job freed", queryId); } void schedulerFreeTaskList(SArray *taskList) { @@ -1706,10 +1722,11 @@ void schedulerFreeTaskList(SArray *taskList) { taosArrayDestroy(taskList); } - + void schedulerDestroy(void) { if (schMgmt.jobs) { - taosHashCleanup(schMgmt.jobs); // TODO + taosHashCleanup(schMgmt.jobs); //TODO schMgmt.jobs = NULL; } } + From e67e7486ec6a3b499d69e87081f58f9df8f2daa4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 10:32:02 +0800 Subject: [PATCH 083/118] [td-11818]Fix a race condition. --- 2.0/src/client/src/tscSql.c | 6 +- source/client/inc/clientInt.h | 16 +- source/client/src/clientEnv.c | 3 +- source/client/src/clientImpl.c | 20 +- source/client/test/clientTests.cpp | 967 +++++++++++++++-------------- 5 files changed, 510 insertions(+), 502 deletions(-) diff --git a/2.0/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c index ab1fffd5a2..c1f7306e87 100644 --- a/2.0/src/client/src/tscSql.c +++ b/2.0/src/client/src/tscSql.c @@ -51,7 +51,7 @@ static bool validPassword(const char* passwd) { } static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { + void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { if (taos_init()) { return NULL; } @@ -186,7 +186,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, (void **)&pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, syncConnCallback, NULL, (void **)&pObj); if (pSql != NULL) { pSql->fp = syncConnCallback; pSql->param = pSql; @@ -262,7 +262,7 @@ static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, (void **)&pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, asyncConnCallback, param, (void **)&pObj); if (pSql == NULL) { return NULL; } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 296fbba634..c61f3da6bd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,13 +101,13 @@ struct SAppInstInfo { }; typedef struct SAppInfo { - int64_t startTime; - char appName[TSDB_APP_NAME_LEN]; - char *ep; - int32_t pid; - int32_t numOfThreads; - - SHashObj *pInstMap; + int64_t startTime; + char appName[TSDB_APP_NAME_LEN]; + char *ep; + int32_t pid; + int32_t numOfThreads; + SHashObj *pInstMap; + pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { @@ -192,7 +192,7 @@ uint64_t generateRequestId(); void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getConnectionDB(STscObj* pObj); +char *getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 1f2048608a..890c4aa425 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -253,10 +253,11 @@ void taos_init_imp(void) { clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); + pthread_mutex_init(&appInfo.mutex, NULL); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - tscDebug("client is initialized successfully"); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7d5199dfaf..cc86bf2650 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -58,7 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i return strdup(key); } -static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); +static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); static void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { @@ -110,9 +110,11 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } char* key = getClusterKey(user, secretEncrypt, ip, port); + SAppInstInfo** pInst = NULL; - // TODO: race condition here. - SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + pthread_mutex_lock(&appInfo.mutex); + + pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; @@ -123,8 +125,10 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, pInst = &p; } + pthread_mutex_unlock(&appInfo.mutex); + tfree(key); - return taosConnectImpl(user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst); + return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); } int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) { @@ -155,7 +159,7 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { SParseContext cxt = { .requestId = pRequest->requestId, .acctId = pTscObj->acctId, - .db = getConnectionDB(pTscObj), + .db = getDbOfConnection(pTscObj), .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, @@ -541,7 +545,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return 0; } -STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { +STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { STscObj *pTscObj = createTscObj(user, auth, db, pAppInfo); if (NULL == pTscObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -599,7 +603,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) { STscObj *pObj = pRequest->pTscObj; - char* db = getConnectionDB(pObj); + char* db = getDbOfConnection(pObj); if (db != NULL) { tstrncpy(pConnect->db, db, sizeof(pConnect->db)); } @@ -831,7 +835,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t } } -char* getConnectionDB(STscObj* pObj) { +char* getDbOfConnection(STscObj* pObj) { char *p = NULL; pthread_mutex_lock(&pObj->mutex); size_t len = strlen(pObj->db); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ee23567705..552295f69b 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,416 +53,419 @@ TEST(testCase, driverInit_Test) { // taos_init(); } -//TEST(testCase, connect_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// if (pConn == NULL) { -// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); -// } -// taos_close(pConn); -//} -// -//TEST(testCase, create_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show users"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show databases"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, create_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create database abc1 vgroups 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_close(pConn); -//} -// -//TEST(testCase, create_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); -// if (taos_errno(pRes) != 0) { -// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// taos_close(pConn); -//} -// -//TEST(testCase, drop_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// pRes = taos_query(pConn, "drop dnode 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, use_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_close(pConn); -//} -// -// TEST(testCase, drop_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// pRes = taos_query(pConn, "drop stable `123_$^)`"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, create_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); -// ASSERT_EQ(taos_errno(pRes), 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); -// ASSERT_NE(taos_errno(pRes), 0); -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_ctable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_vgroup_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show vgroups"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_multiple_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// taos_close(pConn); -// return; -// } -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// -// for (int32_t i = 0; i < 20; ++i) { -// char sql[512] = {0}; -// snprintf(sql, tListLen(sql), -// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, -// (i + 1) * 30, (i + 2) * 40); -// TAOS_RES* pres = taos_query(pConn, sql); -// if (taos_errno(pres) != 0) { -// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); -// } -// taos_free_result(pres); -// } -// -// taos_close(pConn); -//} +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); + } + taos_close(pConn); +} + +TEST(testCase, create_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_close(pConn); +} + +TEST(testCase, create_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); + if (taos_errno(pRes) != 0) { + printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); + if (taos_errno(pRes) != 0) { + printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + taos_close(pConn); +} + +TEST(testCase, drop_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + pRes = taos_query(pConn, "drop dnode 4"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, use_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + + TEST(testCase, drop_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + showDB(pConn); + + TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + showDB(pConn); + + pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("create to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + pRes = taos_query(pConn, "drop stable `123_$^)`"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + taos_close(pConn); +} + +TEST(testCase, create_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + ASSERT_EQ(taos_errno(pRes), 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + ASSERT_NE(taos_errno(pRes), 0); + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); + + TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); + if (taos_errno(pRes) != 0) { + printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_vgroup_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show vgroups"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + taos_close(pConn); + return; + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + + for (int32_t i = 0; i < 20; ++i) { + char sql[512] = {0}; + snprintf(sql, tListLen(sql), + "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, + (i + 1) * 30, (i + 2) * 40); + TAOS_RES* pres = taos_query(pConn, sql); + if (taos_errno(pres) != 0) { + printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); + } + taos_free_result(pres); + } + + taos_close(pConn); +} TEST(testCase, show_table_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); TAOS_RES* pRes = taos_query(pConn, "show tables"); + ASSERT_NE(taos_errno(pRes), 0); + if (taos_errno(pRes) != 0) { - printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); + printf("expected failed to show tables, reason:%s\n", taos_errstr(pRes)); } + taos_free_result(pRes); + pRes = taos_query(pConn, "show abc1.tables"); if (taos_errno(pRes) != 0) { printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); @@ -485,88 +488,88 @@ TEST(testCase, show_table_Test) { taos_close(pConn); } -//TEST(testCase, drop_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in using db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop stable st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, generated_request_id_test) { -// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); -// -// for (int32_t i = 0; i < 50000; ++i) { -// uint64_t v = generateRequestId(); -// void* result = taosHashGet(phash, &v, sizeof(v)); -// if (result != nullptr) { -// printf("0x%lx, index:%d\n", v, i); -// } -// assert(result == nullptr); -// taosHashPut(phash, &v, sizeof(v), NULL, 0); -// } -// -// taosHashCleanup(phash); -//} -// -//TEST(testCase, create_topic_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == nullptr); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// char* sql = "select * from tu"; -// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, drop_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); -//TEST(testCase, insert_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in creating db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in using db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop stable st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, generated_request_id_test) { + SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + for (int32_t i = 0; i < 50000; ++i) { + uint64_t v = generateRequestId(); + void* result = taosHashGet(phash, &v, sizeof(v)); + if (result != nullptr) { + printf("0x%lx, index:%d\n", v, i); + } + assert(result == nullptr); + taosHashPut(phash, &v, sizeof(v), NULL, 0); + } + + taosHashCleanup(phash); +} + +TEST(testCase, create_topic_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from tu"; + pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, insert_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + taos_close(pConn); +} TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); From cd377e4aa8fbf541aa8080ae6f9b7222d03a2713 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 18:38:49 -0800 Subject: [PATCH 084/118] minor changes --- include/common/tmsg.h | 4 ++-- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndFunc.c | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4821e48785..73fe2b857c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -532,7 +532,7 @@ typedef struct { int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char pCont[]; @@ -555,7 +555,7 @@ typedef struct { int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char pCont[]; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1507e2a30d..ebbc42f277 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -284,7 +284,7 @@ typedef struct { int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char *pComment; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index ae3661f5d8..402c48403b 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -73,7 +73,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_INT8(pRaw, dataPos, pFunc->outputType, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize, FUNC_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pFunc->sigature, FUNC_ENCODE_OVER) + SDB_SET_INT64(pRaw, dataPos, pFunc->signature, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER) @@ -121,7 +121,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pFunc->outputType, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->outputLen, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->bufSize, FUNC_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pFunc->sigature, FUNC_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize, FUNC_DECODE_OVER) @@ -164,7 +164,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pC pFunc->outputType = pCreate->outputType; pFunc->outputLen = pCreate->outputLen; pFunc->bufSize = pCreate->bufSize; - pFunc->sigature = pCreate->sigature; + pFunc->signature = pCreate->signature; pFunc->commentSize = pCreate->commentSize; pFunc->codeSize = pCreate->codeSize; pFunc->pComment = pFunc->pData; @@ -267,7 +267,7 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { SCreateFuncReq *pCreate = pReq->rpcMsg.pCont; pCreate->outputLen = htonl(pCreate->outputLen); pCreate->bufSize = htonl(pCreate->bufSize); - pCreate->sigature = htobe64(pCreate->sigature); + pCreate->signature = htobe64(pCreate->signature); pCreate->commentSize = htonl(pCreate->commentSize); pCreate->codeSize = htonl(pCreate->codeSize); @@ -381,7 +381,7 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { pFuncInfo->outputType = pFunc->outputType; pFuncInfo->outputLen = htonl(pFunc->outputLen); pFuncInfo->bufSize = htonl(pFunc->bufSize); - pFuncInfo->sigature = htobe64(pFunc->sigature); + pFuncInfo->signature = htobe64(pFunc->signature); pFuncInfo->commentSize = htonl(pFunc->commentSize); pFuncInfo->codeSize = htonl(pFunc->codeSize); memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize); From 02e454a0d14f594a76a3f986e610c0e0cf54d4fc Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 24 Jan 2022 10:46:55 +0800 Subject: [PATCH 085/118] expose interfaces of mq --- include/client/taos.h | 29 +++ include/common/tmsg.h | 37 ++-- source/client/src/clientImpl.c | 223 ++++++++++++++++++--- source/client/src/clientMsgHandler.c | 2 +- source/client/test/clientTests.cpp | 73 ++++--- source/dnode/mnode/impl/inc/mndDef.h | 204 +++++++++++-------- source/dnode/mnode/impl/src/mndSubscribe.c | 18 +- source/dnode/vnode/inc/tq.h | 15 +- source/dnode/vnode/src/inc/tqInt.h | 6 +- source/dnode/vnode/src/tq/tq.c | 105 +++++++++- source/dnode/vnode/src/vnd/vnodeWrite.c | 4 +- 11 files changed, 531 insertions(+), 185 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 84f6255710..40772e9d2c 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -192,9 +192,38 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); +typedef struct tmq_t tmq_t; +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; + +typedef struct tmq_message_t tmq_message_t; +typedef struct tmq_message_topic_t tmq_message_topic_t; +typedef struct tmq_message_tb_t tmq_message_tb_t; +typedef struct tmq_tb_iter_t tmq_tb_iter_t; +typedef struct tmq_message_col_t tmq_message_col_t; +typedef struct tmq_col_iter_t tmq_col_iter_t; + +DLL_EXPORT tmq_list_t* tmq_list_new(); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*); + +DLL_EXPORT tmq_conf_t* tmq_conf_new(); + +DLL_EXPORT int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); +DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); + +DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); + +DLL_EXPORT tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time); + +DLL_EXPORT int32_t tmq_topic_num(tmq_message_t* msg); +DLL_EXPORT char* tmq_get_topic(tmq_message_topic_t* msg); +DLL_EXPORT int32_t tmq_get_vgId(tmq_message_topic_t* msg); +DLL_EXPORT tmq_message_tb_t* tmq_get_next_tb(tmq_message_topic_t* msg, tmq_tb_iter_t* iter); +DLL_EXPORT tmq_message_col_t* tmq_get_next_col(tmq_message_tb_t* msg, tmq_col_iter_t* iter); + #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 857de3671f..7630c5f5e5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1519,7 +1519,8 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { typedef struct SMqSetCVgReq { int32_t vgId; - int64_t consumerId; + int64_t oldConsumerId; + int64_t newConsumerId; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; char* sql; @@ -1550,7 +1551,8 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); + tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); + tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); tlen += taosEncodeString(buf, pReq->topicName); tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->sql); @@ -1562,7 +1564,8 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); + buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); + buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); buf = taosDecodeStringTo(buf, pReq->topicName); buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeString(buf, &pReq->sql); @@ -1579,15 +1582,6 @@ typedef struct SMqSetCVgRsp { char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; -typedef struct SMqConsumeReq { - int64_t reqId; - int64_t offset; - int64_t consumerId; - int64_t blockingTime; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CONSUMER_GROUP_LEN]; -} SMqConsumeReq; - typedef struct SMqColData { int16_t colId; int16_t type; @@ -1615,12 +1609,29 @@ typedef struct SMqTopicBlk { typedef struct SMqConsumeRsp { int64_t reqId; - int64_t clientId; + int64_t consumerId; int32_t bodyLen; int32_t numOfTopics; SMqTopicData data[]; } SMqConsumeRsp; +// one req for one vg+topic +typedef struct SMqConsumeReq { + //0: commit only, current offset + //1: consume only, poll next offset + //2: commit current and consume next offset + int32_t reqType; + + int64_t reqId; + int64_t consumerId; + int64_t blockingTime; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; + + int64_t offset; + char topic[TSDB_TOPIC_FNAME_LEN]; +} SMqConsumeReq; + + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7d5199dfaf..f5d7f169de 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -117,7 +117,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - /*p->pAppHbMgr = appHbMgrInit(p);*/ + p->pAppHbMgr = appHbMgrInit(p); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -254,14 +254,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); } -typedef struct tmq_t tmq_t; -typedef struct SMqClientTopic { - // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; +typedef struct SMqClientVg { // statistics int64_t consumeCnt; // offset @@ -270,36 +264,160 @@ typedef struct SMqClientTopic { //connection info int32_t vgId; SEpSet epSet; +} SMqClientVg; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; //SArray } SMqClientTopic; typedef struct tmq_resp_err_t { int32_t code; } tmq_resp_err_t; -typedef struct tmq_topic_vgroup_list_t { - char* topicName; +typedef struct tmq_topic_vgroup_t { + char* topic; int32_t vgId; - int64_t committedOffset; + int64_t commitOffset; +} tmq_topic_vgroup_t; + +typedef struct tmq_topic_vgroup_list_t { + int32_t cnt; + int32_t size; + tmq_topic_vgroup_t* elems; } tmq_topic_vgroup_list_t; typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); -typedef struct tmq_conf_t{ - char* clientId; - char* groupId; +struct tmq_conf_t { + char clientId[256]; + char groupId[256]; char* ip; uint16_t port; tmq_commit_cb* commit_cb; -} tmq_conf_t; +}; + +tmq_conf_t* tmq_conf_new() { + tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + return conf; +} + +int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (strcmp(key, "group.id")) { + strcpy(conf->groupId, value); + } + if (strcmp(key, "client.id")) { + strcpy(conf->clientId, value); + } + return 0; +} struct tmq_t { char groupId[256]; char clientId[256]; + int64_t consumerId; + int64_t status; STscObj* pTscObj; tmq_commit_cb* commit_cb; - SArray* clientTopics; // SArray + int32_t nextTopicIdx; + SArray* clientTopics; //SArray }; +tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + if (pTmq == NULL) { + return NULL; + } + pTmq->pTscObj = (STscObj*)conn; + pTmq->status = 0; + strcpy(pTmq->clientId, conf->clientId); + strcpy(pTmq->groupId, conf->groupId); + pTmq->commit_cb = conf->commit_cb; + pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); + return pTmq; +} + +struct tmq_list_t { + int32_t cnt; + int32_t tot; + char* elems[]; +}; +tmq_list_t* tmq_list_new() { + tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + if (ptr == NULL) { + return ptr; + } + ptr->cnt = 0; + ptr->tot = 8; + return ptr; +} + +int32_t tmq_list_append(tmq_list_t* ptr, char* src) { + if (ptr->cnt >= ptr->tot-1) return -1; + ptr->elems[ptr->cnt] = src; + ptr->cnt++; + return 0; +} + + +TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { + SRequestObj *pRequest = NULL; + tmq->status = 1; + int32_t sz = topic_list->cnt; + tmq->clientTopics = taosArrayInit(sz, sizeof(void*)); + for (int i = 0; i < sz; i++) { + char* topicName = strdup(topic_list->elems[i]); + taosArrayPush(tmq->clientTopics, &topicName); + } + SCMSubscribeReq req; + req.topicNum = taosArrayGetSize(tmq->clientTopics); + req.consumerId = tmq->consumerId; + req.consumerGroup = strdup(tmq->groupId); + req.topicNames = tmq->clientTopics; + + int tlen = tSerializeSCMSubscribeReq(NULL, &req); + void* buf = malloc(tlen); + if(buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMSubscribeReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TSDB_SQL_SELECT); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + } + + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + pRequest->type = TDMT_MND_CREATE_TOPIC; + + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); + + tsem_wait(&pRequest->body.rspSem); + +_return: + if (body != NULL) { + destroySendMsgInfo(body); + } + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } @@ -327,10 +445,10 @@ SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { int sz = taosArrayGetSize(clientTopics); for (int i = 0; i < sz; i++) { SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); - if (pCTopic->vgId == -1) { - pMqHb->status = 1; - break; - } + /*if (pCTopic->vgId == -1) {*/ + /*pMqHb->status = 1;*/ + /*break;*/ + /*}*/ } kv.value = pMqHb; kv.valueLen = sizeof(SMqHbMsg); @@ -451,22 +569,63 @@ _return: return pRequest; } -typedef struct tmq_message_t { - int32_t numOfRows; - char* topicName; - TAOS_ROW row[]; -} tmq_message_t; +/*typedef SMqConsumeRsp tmq_message_t;*/ -tmq_message_t* tmq_consume_poll(tmq_t* mq, int64_t blocking_time) { +struct tmq_message_t { + SMqConsumeRsp rsp; +}; + +tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { + if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) { + return NULL; + } + SRequestObj *pRequest = NULL; + SMqConsumeReq req = {0}; + req.reqType = 1; + req.blockingTime = blocking_time; + req.consumerId = tmq->consumerId; + strcpy(req.cgroup, tmq->groupId); + + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); + tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); + strcpy(req.topic, pTopic->topicName); + int32_t nextVgIdx = pTopic->nextVgIdx; + pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx); + req.offset = pVg->currentOffset; + + pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) }; + pRequest->type = TDMT_VND_CONSUME; + + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, body); + + tsem_wait(&pRequest->body.rspSem); + + return (tmq_message_t*)pRequest->body.resInfo.pData; + + /*tsem_wait(&pRequest->body.rspSem);*/ + + /*if (body != NULL) {*/ + /*destroySendMsgInfo(body);*/ + /*}*/ + + /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ + /*pRequest->code = terrno;*/ + /*}*/ + + /*return pRequest;*/ +} + +tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { + SMqConsumeReq req = {0}; return NULL; } -tmq_resp_err_t* tmq_commit(tmq_t* mq, void* callback, int32_t async) { - return NULL; -} - -void tmq_message_destroy(tmq_message_t* mq_message) { - +void tmq_message_destroy(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 81ea18fe08..ec088eb073 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; - /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/ + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ee23567705..a2a26cf6dc 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -526,29 +526,54 @@ TEST(testCase, show_table_Test) { // taosHashCleanup(phash); //} // -//TEST(testCase, create_topic_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == nullptr); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// char* sql = "select * from tu"; -// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_topic_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from tu"; + pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, tmq_subscribe_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg1"); + tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_topic_1"); + tmq_subscribe(tmq, topic_list); + + while (1) { + tmq_message_t* msg = tmq_consume_poll(tmq, 0); + printf("get msg\n"); + if (msg == NULL) break; + } +} + +TEST(testCase, tmq_consume_Test) { +} + +TEST(testCase, tmq_commit_TEST) { +} //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -681,4 +706,4 @@ TEST(testCase, projection_query_tables) { // taos_close(pConn); //} -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1507e2a30d..78f371133c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -19,14 +19,14 @@ #include "os.h" #include "cJSON.h" +#include "scheduler.h" #include "sync.h" -#include "tmsg.h" #include "thash.h" #include "tlist.h" #include "tlog.h" +#include "tmsg.h" #include "trpc.h" #include "ttimer.h" -#include "scheduler.h" #include "mnode.h" @@ -37,12 +37,42 @@ extern "C" { extern int32_t mDebugFlag; // mnode log function -#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} -#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} -#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} -#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }} -#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} -#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} +#define mFatal(...) \ + { \ + if (mDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define mError(...) \ + { \ + if (mDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define mWarn(...) \ + { \ + if (mDebugFlag & DEBUG_WARN) { \ + taosPrintLog("MND WARN ", 255, __VA_ARGS__); \ + } \ + } +#define mInfo(...) \ + { \ + if (mDebugFlag & DEBUG_INFO) { \ + taosPrintLog("MND ", 255, __VA_ARGS__); \ + } \ + } +#define mDebug(...) \ + { \ + if (mDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \ + } \ + } +#define mTrace(...) \ + { \ + if (mDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \ + } \ + } typedef enum { MND_AUTH_ACCT_START = 0, @@ -96,13 +126,13 @@ typedef struct { ETrnPolicy policy; int32_t code; int32_t failedTimes; - void *rpcHandle; - void *rpcAHandle; - SArray *redoLogs; - SArray *undoLogs; - SArray *commitLogs; - SArray *redoActions; - SArray *undoActions; + void* rpcHandle; + void* rpcAHandle; + SArray* redoLogs; + SArray* undoLogs; + SArray* commitLogs; + SArray* redoActions; + SArray* undoActions; } STrans; typedef struct { @@ -135,28 +165,28 @@ typedef struct { ESyncState role; int32_t roleTerm; int64_t roleTime; - SDnodeObj *pDnode; + SDnodeObj* pDnode; } SMnodeObj; typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; - SDnodeObj *pDnode; + SDnodeObj* pDnode; } SQnodeObj; typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; - SDnodeObj *pDnode; + SDnodeObj* pDnode; } SSnodeObj; typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; - SDnodeObj *pDnode; + SDnodeObj* pDnode; } SBnodeObj; typedef struct { @@ -201,7 +231,7 @@ typedef struct { int64_t updateTime; int8_t superUser; int32_t acctId; - SHashObj *prohibitDbHash; + SHashObj* prohibitDbHash; } SUserObj; typedef struct { @@ -226,15 +256,15 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; uint64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; } SDbObj; typedef struct { @@ -272,7 +302,7 @@ typedef struct { int32_t numOfColumns; int32_t numOfTags; SRWLatch lock; - SSchema *pSchema; + SSchema* pSchema; } SStbObj; typedef struct { @@ -287,8 +317,8 @@ typedef struct { int64_t sigature; int32_t commentSize; int32_t codeSize; - char *pComment; - char *pCode; + char* pComment; + char* pCode; char pData[]; } SFuncObj; @@ -301,8 +331,8 @@ typedef struct { int32_t numOfRows; int32_t numOfReads; int32_t payloadLen; - void *pIter; - SMnode *pMnode; + void* pIter; + SMnode* pMnode; char db[TSDB_DB_FNAME_LEN]; int16_t offset[TSDB_MAX_COLUMNS]; int32_t bytes[TSDB_MAX_COLUMNS]; @@ -327,9 +357,10 @@ typedef struct SMqTopicConsumer { #endif typedef struct SMqConsumerEp { - int32_t vgId; // -1 for unassigned + int32_t vgId; // -1 for unassigned + int32_t status; SEpSet epSet; - int64_t consumerId; // -1 for unassigned + int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; int32_t execLen; @@ -339,6 +370,7 @@ typedef struct SMqConsumerEp { static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); + tlen += taosEncodeFixedI32(buf, pConsumerEp->status); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); @@ -347,6 +379,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); + buf = taosDecodeFixedI32(buf, &pConsumerEp->status); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); @@ -354,16 +387,17 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu return buf; } -//unit for rebalance +// unit for rebalance typedef struct SMqSubscribeObj { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t epoch; - //TODO: replace with priority queue + // TODO: replace with priority queue int32_t nextConsumerIdx; - SArray* availConsumer; // SArray (consumerId) - SArray* assigned; // SArray - SArray* unassignedConsumer; // SArray - SArray* unassignedVg; // SArray + SArray* availConsumer; // SArray (consumerId) + SArray* assigned; // SArray + SArray* idleConsumer; // SArray + SArray* lostConsumer; // SArray + SArray* unassignedVg; // SArray } SMqSubscribeObj; static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { @@ -384,17 +418,17 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { free(pSub); return NULL; } - pSub->unassignedConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); + pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); if (pSub->assigned == NULL) { taosArrayDestroy(pSub->availConsumer); - taosArrayDestroy(pSub->unassignedConsumer); + taosArrayDestroy(pSub->idleConsumer); free(pSub); return NULL; } pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); if (pSub->assigned == NULL) { taosArrayDestroy(pSub->availConsumer); - taosArrayDestroy(pSub->unassignedConsumer); + taosArrayDestroy(pSub->idleConsumer); taosArrayDestroy(pSub->unassignedVg); free(pSub); return NULL; @@ -422,10 +456,10 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb tlen += tEncodeSMqConsumerEp(buf, pCEp); } - sz = taosArrayGetSize(pSub->unassignedConsumer); + sz = taosArrayGetSize(pSub->idleConsumer); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedConsumer, i); + SMqConsumerEp* pCEp = taosArrayGet(pSub->idleConsumer, i); tlen += tEncodeSMqConsumerEp(buf, pCEp); } @@ -457,22 +491,22 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) } buf = taosDecodeFixedI32(buf, &sz); - pSub->unassignedConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); - if (pSub->unassignedConsumer == NULL) { + pSub->idleConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->idleConsumer == NULL) { taosArrayDestroy(pSub->assigned); return NULL; } for (int32_t i = 0; i < sz; i++) { SMqConsumerEp cEp; buf = tDecodeSMqConsumerEp(buf, &cEp); - taosArrayPush(pSub->unassignedConsumer, &cEp); + taosArrayPush(pSub->idleConsumer, &cEp); } buf = taosDecodeFixedI32(buf, &sz); pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); if (pSub->unassignedVg == NULL) { taosArrayDestroy(pSub->assigned); - taosArrayDestroy(pSub->unassignedConsumer); + taosArrayDestroy(pSub->idleConsumer); return NULL; } for (int32_t i = 0; i < sz; i++) { @@ -487,38 +521,37 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) typedef struct SMqCGroup { char name[TSDB_CONSUMER_GROUP_LEN]; int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal - SList *consumerIds; // SList - SList *idleVGroups; // SList + SList* consumerIds; // SList + SList* idleVGroups; // SList } SMqCGroup; typedef struct SMqTopicObj { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - uint64_t uid; - uint64_t dbUid; - int32_t version; - SRWLatch lock; - int32_t sqlLen; - char *sql; - char *logicalPlan; - char *physicalPlan; - //SHashObj *cgroups; // SHashObj - //SHashObj *consumers; // SHashObj + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + uint64_t uid; + uint64_t dbUid; + int32_t version; + SRWLatch lock; + int32_t sqlLen; + char* sql; + char* logicalPlan; + char* physicalPlan; + // SHashObj *cgroups; // SHashObj + // SHashObj *consumers; // SHashObj } SMqTopicObj; // TODO: add cache and change name to id typedef struct SMqConsumerTopic { char name[TSDB_TOPIC_FNAME_LEN]; int32_t epoch; - //TODO: replace with something with ep - //SList *vgroups; // SList - //vg assigned to the consumer on the topic - SArray *pVgInfo; // SArray + // vg assigned to the consumer on the topic + SArray* pVgInfo; // SArray } SMqConsumerTopic; -static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { +static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, + SMqSubscribeObj* pSub) { SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic)); if (pCTopic == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -567,10 +600,11 @@ static FORCE_INLINE void* tDecodeSMqConsumerTopic(void* buf, SMqConsumerTopic* p typedef struct SMqConsumerObj { int64_t consumerId; + int64_t connId; SRWLatch lock; char cgroup[TSDB_CONSUMER_GROUP_LEN]; - SArray *topics; // SArray - //SHashObj *topicHash; //SHashObj + SArray* topics; // SArray + // SHashObj *topicHash; //SHashObj } SMqConsumerObj; static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { @@ -602,12 +636,12 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons typedef struct SMqSubConsumerObj { int64_t consumerUid; // if -1, unassigned - SList *vgId; // SList + SList* vgId; // SList } SMqSubConsumerObj; typedef struct SMqSubCGroupObj { char name[TSDB_CONSUMER_GROUP_LEN]; - SList *consumers; // SList + SList* consumers; // SList } SMqSubCGroupObj; typedef struct SMqSubTopicObj { @@ -620,30 +654,30 @@ typedef struct SMqSubTopicObj { int32_t version; SRWLatch lock; int32_t sqlLen; - char *sql; - char *logicalPlan; - char *physicalPlan; - SList *cgroups; // SList + char* sql; + char* logicalPlan; + char* physicalPlan; + SList* cgroups; // SList } SMqSubTopicObj; typedef struct SMqConsumerSubObj { int64_t topicUid; - SList *vgIds; // SList + SList* vgIds; // SList } SMqConsumerSubObj; typedef struct SMqConsumerHbObj { int64_t consumerId; - SList *consumerSubs; // SList + SList* consumerSubs; // SList } SMqConsumerHbObj; typedef struct SMqVGroupSubObj { int64_t topicUid; - SList *consumerIds; // SList + SList* consumerIds; // SList } SMqVGroupSubObj; typedef struct SMqVGroupHbObj { int64_t vgId; - SList *vgSubs; // SList + SList* vgSubs; // SList } SMqVGroupHbObj; #if 0 @@ -663,11 +697,11 @@ typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; char db[TSDB_DB_FNAME_LEN]; int32_t acctId; - SMnode *pMnode; + SMnode* pMnode; int64_t createdTime; SRpcMsg rpcMsg; int32_t contLen; - void *pCont; + void* pCont; } SMnodeMsg; #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 817ca4f4be..9a573cbe2c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -98,7 +98,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // build msg SMqSetCVgReq req = { .vgId = pCEp->vgId, - .consumerId = consumerId, + .oldConsumerId = -1, + .newConsumerId = consumerId, }; strcpy(req.cgroup, cgroup); strcpy(req.topicName, topic); @@ -152,6 +153,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas //convert dag to msg for (int32_t i = 0; i < sz; i++) { SMqConsumerEp CEp; + CEp.status = 0; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; STaskInfo* pTaskInfo = taosArrayGet(pArray, i); tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); @@ -171,7 +173,8 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SMqSetCVgReq req = { .vgId = vgId, - .consumerId = pConsumer->consumerId, + .oldConsumerId = -1, + .newConsumerId = pConsumer->consumerId, }; strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.topicName, pTopic->name); @@ -451,12 +454,13 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); if (pTopic == NULL) { - /*terrno = */ + mError("topic being subscribed not exist: %s", newTopicName); continue; } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); if (pSub == NULL) { + mDebug("create new subscription, group: %s, topic %s", consumerGroup, newTopicName); pSub = tNewSubscribeObj(); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -464,14 +468,15 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } // set unassigned vg mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); + //TODO: disable alter } taosArrayPush(pSub->availConsumer, &consumerId); - // TODO: no need SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); taosArrayPush(pConsumer->topics, pConsumerTopic); if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { + ASSERT(taosArrayGetSize(pConsumerTopic->pVgInfo) == 1); int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); // send setmsg to vnode if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) { @@ -479,8 +484,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } } - taosArrayDestroy(pConsumerTopic->pVgInfo); - free(pConsumerTopic); + SSdbRaw *pRaw = mndSubActionEncode(pSub); /*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/ mndTransAppendRedolog(pTrans, pRaw); @@ -533,12 +537,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + if (newSub) taosArrayDestroy(newSub); mndTransDrop(pTrans); mndReleaseConsumer(pMnode, pConsumer); return -1; } - // TODO: free memory if (newSub) taosArrayDestroy(newSub); mndTransDrop(pTrans); mndReleaseConsumer(pMnode, pConsumer); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 9cc987d731..0f318dea0b 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -18,7 +18,6 @@ #include "common.h" #include "executor.h" -#include "vnode.h" #include "mallocator.h" #include "meta.h" #include "os.h" @@ -29,6 +28,7 @@ #include "trpc.h" #include "ttimer.h" #include "tutil.h" +#include "vnode.h" #include "wal.h" #ifdef __cplusplus @@ -149,10 +149,11 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - qTaskInfo_t task; + int8_t status; + int64_t offset; + void* dst; + qTaskInfo_t task; + SSubQueryMsg* pQueryMsg; } STqTaskItem; // new version @@ -164,7 +165,6 @@ typedef struct STqBuffer { typedef struct STqTopicHandle { char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_TOPIC_FNAME_LEN]; char* sql; char* logicalPlan; char* physicalPlan; @@ -177,6 +177,7 @@ typedef struct STqTopicHandle { typedef struct STqConsumerHandle { int64_t consumerId; int64_t epoch; + char cgroup[TSDB_TOPIC_FNAME_LEN]; SArray* topics; // SArray } STqConsumerHandle; @@ -318,7 +319,7 @@ int tqSendLaunchQuery(STqMsgItem*, int64_t offset); #endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); -int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq); +int32_t tqProcessSetConnReq(STQ* pTq, char* msg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 2b4200fce5..a9ba825a29 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -44,8 +44,10 @@ extern int32_t tqDebugFlag; // delete persistent storage for meta info // int tqDropTCGroup(STQ*, const char* topic, int cgId); -int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); +//int tqSerializeGroup(const STqGroup*, STqSerializedHead**); +//const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); +int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b18f50cd3f..49bbb77797 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -68,7 +68,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl // TODO: error code of buffer pool } #endif - pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); + pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); if (pTq->tqMeta == NULL) { free(pTq); #if 0 @@ -478,6 +478,59 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { } #endif +int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { + int32_t num = taosArrayGetSize(pConsumer->topics); + int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); + if (sz > (*ppHead)->ssize) { + void* tmpPtr = realloc(*ppHead, sz); + if (tmpPtr == NULL) { + free(*ppHead); + return -1; + } + *ppHead = tmpPtr; + (*ppHead)->ssize = sz; + } + + void* ptr = (*ppHead)->content; + *(int64_t*)ptr = pConsumer->consumerId; + ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); + *(int64_t*)ptr = pConsumer->epoch; + ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); + memcpy(ptr, pConsumer->topics, TSDB_TOPIC_FNAME_LEN); + ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); + *(int32_t*)ptr = num; + ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); + for (int32_t i = 0; i < num; i++) { + STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); + memcpy(ptr, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); + ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); + *(int64_t*)ptr = pTopic->committedOffset; + POINTER_SHIFT(ptr, sizeof(int64_t)); + } + + return 0; +} + +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) { + STqConsumerHandle* pConsumer = *ppConsumer; + const void* ptr = pHead->content; + pConsumer->consumerId = *(int64_t*)ptr; + ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); + pConsumer->epoch = *(int64_t*)ptr; + ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); + memcpy(pConsumer->cgroup, ptr, TSDB_TOPIC_FNAME_LEN); + ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); + int32_t sz = *(int32_t*)ptr; + ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); + pConsumer->topics = taosArrayInit(sz, sizeof(STqTopicHandle)); + for (int32_t i = 0; i < sz; i++) { + /*STqTopicHandle* topicHandle = */ + /*taosArrayPush(pConsumer->topics, );*/ + } + return NULL; +} + +#if 0 int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { // calculate size int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); @@ -608,6 +661,7 @@ int tqItemSSize() { // mainly for executor return 0; } +#endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SMqConsumeReq* pReq = pMsg->pCont; @@ -625,7 +679,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { for (int i = 0; i < sz; i++) { STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); + //TODO: support multiple topic in one req + if (strcmp(pTopic->topicName, pReq->topic) != 0) { + continue; + } + if (fetchOffset == -1) { + fetchOffset = pTopic->committedOffset + 1; + } int8_t pos; int8_t skip = 0; SWalHead* pHead; @@ -670,6 +731,23 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { break; } if (pDataBlock != NULL) { + SMqTbData tbData = { + .uid = pDataBlock->info.uid, + .numOfCols = pDataBlock->info.numOfCols, + .numOfRows = pDataBlock->info.rows, + }; + for (int i = 0; i < pDataBlock->info.numOfCols; i++) { + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, i); + int32_t sz = pColData->info.bytes * pDataBlock->info.rows; + SMqColData colData = { + .bytes = pColData->info.bytes, + .colId = pColData->info.colId, + .type = pColData->info.type, + }; + memcpy(colData.data, pColData->pData, colData.bytes * pDataBlock->info.rows); + memcpy(&tbData.colData[i], &colData, sz); + } + /*pDataBlock->info.*/ taosArrayPush(pRes, pDataBlock); } else { break; @@ -692,29 +770,34 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { pTopic->buffer.lastOffset = pReq->offset; } // put output into rsp + SMqConsumeRsp rsp = { + .consumerId = consumerId, + .numOfTopics = 1 + }; } - // launch query - // get result return 0; } -int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { +int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { + SMqSetCVgReq req; + tDecodeSMqSetCVgReq(msg, &req); STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); if (pConsumer == NULL) { return -1; } + strcpy(pConsumer->cgroup, req.cgroup); + pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); if (pTopic == NULL) { free(pConsumer); return -1; } - strcpy(pTopic->topicName, pReq->topicName); - strcpy(pTopic->cgroup, pReq->cgroup); - strcpy(pTopic->sql, pReq->sql); - strcpy(pTopic->logicalPlan, pReq->logicalPlan); - strcpy(pTopic->physicalPlan, pReq->physicalPlan); + strcpy(pTopic->topicName, req.topicName); + strcpy(pTopic->sql, req.sql); + strcpy(pTopic->logicalPlan, req.logicalPlan); + strcpy(pTopic->physicalPlan, req.physicalPlan); pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; @@ -724,9 +807,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, pReadHandle); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&req.msg, pReadHandle); } - // write mq meta + taosArrayPush(pConsumer->topics, pTopic); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index a6c3f25c6f..ccddfd56d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -112,9 +112,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_SET_CONN: { - SMqSetCVgReq req; - tDecodeSMqSetCVgReq(ptr, &req); - if (tqProcessSetConnReq(pVnode->pTq, &req) < 0) { + if (tqProcessSetConnReq(pVnode->pTq, ptr) < 0) { } } break; default: From 0f9d860d4ec54020150c7da45eb7925a2af7b02d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 10:48:49 +0800 Subject: [PATCH 086/118] [td-11818]Fix invalid read in the unit test. --- source/client/test/clientTests.cpp | 1 - source/libs/scheduler/src/scheduler.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 552295f69b..2699f9869f 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -537,7 +537,6 @@ TEST(testCase, create_topic_Test) { if (taos_errno(pRes) != 0) { printf("error in use db, reason:%s\n", taos_errstr(pRes)); } - taos_free_result(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == nullptr); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 747446a6ba..4c3ddd87ab 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1706,7 +1706,7 @@ void scheduleFreeJob(void *job) { tfree(pJob); - qDebug("QID:%"PRIx64" job freed", queryId); + qDebug("QID:0x%"PRIx64" job freed", queryId); } void schedulerFreeTaskList(SArray *taskList) { From 24ef67a278376dff0203e24657ed9f003f973c3c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Jan 2022 11:33:28 +0800 Subject: [PATCH 087/118] fix mem leak --- source/client/src/clientImpl.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b0b3d031b2..112bf354fc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -215,6 +215,8 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, if (pQueryNode->type == TSDB_SQL_SELECT) { setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); pRequest->type = TDMT_VND_QUERY; + } else { + tfree(pSchema); } return code; @@ -641,7 +643,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQueryNode = NULL; - SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -650,6 +651,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { + SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); pRequest->code = terrno; From c51385570b1ece88048ce78ae4119aa43f6b545f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Jan 2022 11:47:14 +0800 Subject: [PATCH 088/118] fix mem leak issue --- source/client/src/clientImpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 112bf354fc..a17c299b9a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -239,6 +239,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; + taosArrayDestroy(pNodeList); + int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry From 1f16647c97bd483e3a46917278a1672464ce1450 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 11:53:10 +0800 Subject: [PATCH 089/118] [td-11818] Fix memory leak. --- source/libs/planner/src/planner.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index c93569a6c1..2fc4c8ea3e 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -91,16 +91,18 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return TSDB_CODE_SUCCESS; } -void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, - int32_t* numOfCols) { // extract the final result schema +// extract the final result schema +void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols) { SArray* pTopSubplan = taosArrayGetP((*pDag)->pSubplans, 0); - SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0); + SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0); SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); *numOfCols = pDataBlockSchema->numOfCols; - *pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema)); - memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema)); + if (*numOfCols > 0) { + *pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema)); + memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema)); + } } void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { From 92bef71ec7ee4dabb989520f7f476f6032d5680c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jan 2022 12:53:17 +0800 Subject: [PATCH 090/118] rename MAX/MIN/POW2/SWAP name --- 2.0/src/client/src/tscGlobalmerge.c | 4 +-- 2.0/src/client/src/tscParseLineProtocol.c | 4 +-- 2.0/src/client/src/tscSQLParser.c | 10 +++--- 2.0/src/client/src/tscSql.c | 10 +++--- 2.0/src/client/src/tscSubquery.c | 2 +- 2.0/src/query/src/qAggMain.c | 12 +++---- 2.0/src/query/src/qExecutor.c | 34 +++++++++---------- include/os/osMath.h | 14 ++++---- source/client/src/clientImpl.c | 8 ++--- source/common/src/ttypes.c | 14 ++++---- source/dnode/mgmt/impl/src/dndVnodes.c | 8 ++--- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndSync.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 2 +- source/dnode/mnode/sdb/src/sdbHash.c | 8 ++--- source/dnode/vnode/src/tsdb/tsdbCompact.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 32 ++++++++--------- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 ++++---- source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c | 2 +- source/libs/executor/src/executorimpl.c | 34 +++++++++---------- source/libs/function/src/taggfunction.c | 12 +++---- source/libs/function/src/tbinoperator.c | 14 ++++---- source/libs/index/src/index_fst.c | 6 ++-- source/libs/parser/src/astValidate.c | 10 +++--- source/libs/parser/src/insertParser.c | 2 +- source/libs/planner/src/physicalPlan.c | 2 +- source/libs/qcom/src/queryUtil.c | 2 +- source/libs/sync/inc/sync_type.h | 8 ++--- source/libs/sync/src/sync_raft_progress.c | 6 ++-- source/libs/wal/src/walMeta.c | 2 +- source/util/src/thash.c | 2 +- tools/shell/src/shellEngine.c | 30 ++++++++-------- 33 files changed, 158 insertions(+), 158 deletions(-) diff --git a/2.0/src/client/src/tscGlobalmerge.c b/2.0/src/client/src/tscGlobalmerge.c index 6acbfe3e89..c8ff36db51 100644 --- a/2.0/src/client/src/tscGlobalmerge.c +++ b/2.0/src/client/src/tscGlobalmerge.c @@ -701,7 +701,7 @@ SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint // todo remove it int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { - int32_t maxRowSize = MAX(rowSize, finalRowSize); + int32_t maxRowSize = TMAX(rowSize, finalRowSize); char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); size_t size = tscNumOfFields(pQueryInfo); @@ -965,7 +965,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1)); if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) { - SWAP(w->skey, w->ekey, TSKEY); + TSWAP(w->skey, w->ekey, TSKEY); assert(w->skey <= w->ekey); } } diff --git a/2.0/src/client/src/tscParseLineProtocol.c b/2.0/src/client/src/tscParseLineProtocol.c index 20d52fbd3e..2225b2c580 100644 --- a/2.0/src/client/src/tscParseLineProtocol.c +++ b/2.0/src/client/src/tscParseLineProtocol.c @@ -47,7 +47,7 @@ int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; int kvLen1 = (int)strlen(kv1->key); int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { @@ -123,7 +123,7 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra if (code != 0) { return code; } - pField->bytes = MAX(pField->bytes, bytes); + pField->bytes = TMAX(pField->bytes, bytes); } else { SSchema field = {0}; diff --git a/2.0/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c index c68d7cef06..7d2d6f6ebf 100644 --- a/2.0/src/client/src/tscSQLParser.c +++ b/2.0/src/client/src/tscSQLParser.c @@ -1785,7 +1785,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 getNewResColId(pCmd), sizeof(double), false); char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z; - size_t len = MIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); + size_t len = TMIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); tstrncpy(pExpr->base.aliasName, name, len); tExprNode* pNode = NULL; @@ -2246,7 +2246,7 @@ SSchema tGetUserSpecifiedColumnSchema(SVariant* pVal, SStrToken* exprStr, const if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); + size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1); tstrncpy(s.name, exprStr->z, tlen); strdequote(s.name); } @@ -2419,7 +2419,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { char uname[TSDB_COL_NAME_LEN] = {0}; - int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); if (tsKeepOriginalColumnName) { // keep the original column name @@ -2432,7 +2432,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } } else { // use the user-input result column name - int32_t len = MIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->pNode->exprToken.z, len); } } @@ -4380,7 +4380,7 @@ static void exchangeExpr(tSqlExpr* pExpr) { } pExpr->tokenId = optr; - SWAP(pExpr->pLeft, pExpr->pRight, void*); + TSWAP(pExpr->pLeft, pExpr->pRight, void*); } } diff --git a/2.0/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c index ab1fffd5a2..83f9e9b830 100644 --- a/2.0/src/client/src/tscSql.c +++ b/2.0/src/client/src/tscSql.c @@ -120,7 +120,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa pObj->signature = pObj; pObj->pRpcObj = (SRpcObj *)pRpcObj; tstrncpy(pObj->user, user, sizeof(pObj->user)); - secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); + secretEncryptLen = TMIN(secretEncryptLen, sizeof(pObj->pass)); memcpy(pObj->pass, secretEncrypt, secretEncryptLen); if (db) { @@ -245,10 +245,10 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us char userBuf[TSDB_USER_LEN] = {0}; char passBuf[TSDB_KEY_LEN] = {0}; char dbBuf[TSDB_DB_NAME_LEN] = {0}; - strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passBuf, pass, MIN(TSDB_KEY_LEN - 1, passLen)); - strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + strncpy(ipBuf, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userBuf, user, TMIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passBuf, pass, TMIN(TSDB_KEY_LEN - 1, passLen)); + strncpy(dbBuf, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); } diff --git a/2.0/src/client/src/tscSubquery.c b/2.0/src/client/src/tscSubquery.c index 24a6377a73..8ed7eb08e6 100644 --- a/2.0/src/client/src/tscSubquery.c +++ b/2.0/src/client/src/tscSubquery.c @@ -3396,7 +3396,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { } int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); - numOfRes = (int32_t)(MIN(numOfRes, remain)); + numOfRes = (int32_t)(TMIN(numOfRes, remain)); } if (numOfRes == 0) { // no result any more, free all subquery objects diff --git a/2.0/src/query/src/qAggMain.c b/2.0/src/query/src/qAggMain.c index d57532ac8f..7ab3855506 100644 --- a/2.0/src/query/src/qAggMain.c +++ b/2.0/src/query/src/qAggMain.c @@ -373,7 +373,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *interBytes = (int16_t)sizeof(SPercentileInfo); } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; - *bytes = MAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string + *bytes = TMAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string *interBytes = *bytes; } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; @@ -1275,7 +1275,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { continue; \ } \ (num) += 1; \ - (r) += POW2(((type *)d)[i] - (delta)); \ + (r) += TPOW2(((type *)d)[i] - (delta)); \ } static void stddev_function(SQLFunctionCtx *pCtx) { @@ -1314,7 +1314,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -1427,7 +1427,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -4109,8 +4109,8 @@ static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockD pDist->totalRows += pSrc->totalRows; if (pResInfo->hasResult == DATA_SET_FLAG) { - pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); - pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); } else { pDist->maxRows = pSrc->maxRows; pDist->minRows = pSrc->minRows; diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c index a6a9115b2f..fdadf39d4d 100644 --- a/2.0/src/query/src/qExecutor.c +++ b/2.0/src/query/src/qExecutor.c @@ -328,7 +328,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.bytes = pExpr[i].base.resBytes; idata.info.colId = pExpr[i].base.resColId; - int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -2643,7 +2643,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2653,7 +2653,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2664,7 +2664,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->order.order = TSDB_ORDER_ASC; @@ -2677,7 +2677,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2688,7 +2688,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2703,7 +2703,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2714,7 +2714,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2754,8 +2754,8 @@ static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); @@ -3523,7 +3523,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) return; } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; SWITCH_ORDER(pTableQueryInfo->cur.order); @@ -3730,7 +3730,7 @@ static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo } // reverse order time range - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SET_REVERSE_SCAN_FLAG(pRuntimeEnv); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); @@ -4109,8 +4109,8 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { */ STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(win.skey, win.ekey); - TSKEY ek = MAX(win.skey, win.ekey); + TSKEY sk = TMIN(win.skey, win.ekey); + TSKEY ek = TMAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4235,7 +4235,7 @@ static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFuncti } SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); + pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -6945,8 +6945,8 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); pInfo->pFillInfo = diff --git a/include/os/osMath.h b/include/os/osMath.h index 440d3bc787..44a63500ab 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -20,37 +20,37 @@ extern "C" { #endif -#define POW2(x) ((x) * (x)) +#define TPOW2(x) ((x) * (x)) #define ABS(x) ((x) > 0 ? (x) : -(x)) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define SWAP(a, b, c) \ + #define TSWAP(a, b, c) \ do { \ c __tmp = (c)(a); \ (a) = (c)(b); \ (b) = __tmp; \ } while (0) - #define MAX(a, b) (((a) > (b)) ? (a) : (b)) - #define MIN(a, b) (((a) < (b)) ? (a) : (b)) + #define TMAX(a, b) (((a) > (b)) ? (a) : (b)) + #define TMIN(a, b) (((a) < (b)) ? (a) : (b)) #else - #define SWAP(a, b, c) \ + #define TSWAP(a, b, c) \ do { \ typeof(a) __tmp = (a); \ (a) = (b); \ (b) = __tmp; \ } while (0) - #define MAX(a, b) \ + #define TMAX(a, b) \ ({ \ typeof(a) __a = (a); \ typeof(b) __b = (b); \ (__a > __b) ? __a : __b; \ }) - #define MIN(a, b) \ + #define TMIN(a, b) \ ({ \ typeof(a) __a = (a); \ typeof(b) __b = (b); \ diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 34ab1fb05a..cb095ea12f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -694,10 +694,10 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c char userStr[TSDB_USER_LEN] = {0}; char passStr[TSDB_PASSWORD_LEN] = {0}; - strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); - strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + strncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipStr, userStr, passStr, dbStr, port); } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 82f6a8e49a..73f755bf1a 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -630,7 +630,7 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -#define SWAP(a, b, c) \ +#define TSWAP(a, b, c) \ do { \ typeof(a) __tmp = (a); \ (a) = (b); \ @@ -642,35 +642,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf switch (type) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: { - SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); + TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - SWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); + TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); break; } case TSDB_DATA_TYPE_DOUBLE: { - SWAP(*(double *)(pLeft), *(double *)(pRight), double); + TSWAP(*(double *)(pLeft), *(double *)(pRight), double); break; } case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: { - SWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); + TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); break; } case TSDB_DATA_TYPE_FLOAT: { - SWAP(*(float *)(pLeft), *(float *)(pRight), float); + TSWAP(*(float *)(pLeft), *(float *)(pRight), float); break; } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { - SWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); + TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); break; } diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 0d4b9c803d..bcc11c4759 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -910,11 +910,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->env.numOfCores); - int32_t minQueryThreads = MAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); + int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores); + int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = MAX(pDnode->env.numOfCores, 1); - int32_t maxSyncThreads = MAX(pDnode->env.numOfCores / 2, 1); + int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); + int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); SWorkerPool *pPool = &pMgmt->queryPool; pPool->name = "vnode-query"; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 22fdfde2ac..76fabc96ce 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -261,7 +261,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } - pConn->numOfQueries = MIN(QUERY_SAVE_SIZE, numOfQueries); + pConn->numOfQueries = TMIN(QUERY_SAVE_SIZE, numOfQueries); int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); if (saveSize > 0 && pConn->pQueries != NULL) { diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 035c05c286..aacaa0d728 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -56,7 +56,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { int64_t last = walGetLastVer(pWal); mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); - first = MAX(lastSdbVer + 1, first); + first = TMAX(lastSdbVer + 1, first); for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { if (walReadWithHandle(pHandle, ver) < 0) { mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0bed943b60..04fedbb3ce 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -459,7 +459,7 @@ static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void int32_t *pNumOfVgroups = p3; if (pVgroup->dbUid == uid) { - *pReplica = MAX(*pReplica, pVgroup->replica); + *pReplica = TMAX(*pReplica, pVgroup->replica); (*pNumOfVgroups)++; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index a9267b0ea3..02e7222942 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -162,10 +162,10 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { - pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) { - pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } pSdb->tableVer[pRow->type]++; @@ -420,13 +420,13 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { while (ppRow != NULL) { SSdbRow *pRow = *ppRow; int32_t id = *(int32_t *)pRow->pObj; - maxId = MAX(id, maxId); + maxId = TMAX(id, maxId); ppRow = taosHashIterate(hash, ppRow); } taosRUnLockLatch(pLock); - maxId = MAX(maxId, pSdb->maxId[type]); + maxId = TMAX(maxId, pSdb->maxId[type]); return maxId + 1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index 59b680ec15..80b59d8756 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -459,7 +459,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { while (true) { if (pReadh->pDCols[0]->numOfRows - ridx == 0) break; - int rowsToMerge = MIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); + int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 135b81f282..ca2b1069f9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -836,7 +836,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // return -1; // } -// maxBufSize = MAX(maxBufSize, rInfo.size); +// maxBufSize = TMAX(maxBufSize, rInfo.size); // if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) { // tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index b16b3581df..81f0a6736b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -153,8 +153,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (fKey == INT64_MAX && rowKey == INT64_MAX) break; if (fKey < rowKey) { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); filterIter++; if (filterIter >= nFilterKeys) { @@ -170,8 +170,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } @@ -196,12 +196,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsUpdated++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); } } @@ -714,8 +714,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (fKey == INT64_MAX && rowKey == INT64_MAX) break; if (fKey < rowKey) { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); filterIter++; if (filterIter >= nFilterKeys) { @@ -731,8 +731,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } @@ -757,12 +757,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsUpdated++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 15748118d7..68c7f190db 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -504,7 +504,7 @@ void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { if (emptyQueryTimewindow(pTsdbReadHandle)) { if (pCond->order != pTsdbReadHandle->order) { pTsdbReadHandle->order = pCond->order; - SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); + TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); } return; @@ -971,7 +971,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { pHandle->cur.mixBlock = true; if (!ASCENDING_TRAVERSE(pHandle->order)) { - SWAP(win->skey, win->ekey, TSKEY); + TSWAP(win->skey, win->ekey, TSKEY); } return true; @@ -1074,8 +1074,8 @@ static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, i assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } - s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + s = TMIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + e = TMAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); @@ -1240,7 +1240,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - SWAP(cur->win.skey, cur->win.ekey, TSKEY); + TSWAP(cur->win.skey, cur->win.ekey, TSKEY); } cur->mixBlock = true; @@ -1795,7 +1795,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa int32_t end = endPos; if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - SWAP(start, end, int32_t); + TSWAP(start, end, int32_t); } assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); @@ -2020,7 +2020,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - SWAP(cur->win.skey, cur->win.ekey, TSKEY); + TSWAP(cur->win.skey, cur->win.ekey, TSKEY); } moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index f49babb1a5..5844ef0345 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -68,7 +68,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { void * ptr; if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { - uint64_t capacity = MAX(pVMA->ssize, size); + uint64_t capacity = TMAX(pVMA->ssize, size); pNode = vArenaNodeNew(capacity); if (pNode == NULL) { // TODO: handle error diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index dc4d9c7238..e2e5f400f7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -303,7 +303,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.bytes = pExpr[i].base.resSchema.bytes; idata.info.colId = pExpr[i].base.resSchema.colId; - int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -328,7 +328,7 @@ SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { idata.info.bytes = pExpr->base.resSchema.bytes; idata.info.colId = pExpr->base.resSchema.colId; - int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -2678,7 +2678,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2688,7 +2688,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2699,7 +2699,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->order.order = TSDB_ORDER_ASC; @@ -2712,7 +2712,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, // pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2723,7 +2723,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, // pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2738,7 +2738,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, // pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2749,7 +2749,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, // pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2789,8 +2789,8 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); @@ -3473,7 +3473,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) return; } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; SWITCH_ORDER(pTableQueryInfo->cur.order); @@ -4095,8 +4095,8 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { */ STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(win.skey, win.ekey); - TSKEY ek = MAX(win.skey, win.ekey); + TSKEY sk = TMIN(win.skey, win.ekey); + TSKEY ek = TMAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4221,7 +4221,7 @@ static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctio } // SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); -// pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); +// pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -7147,8 +7147,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); pInfo->pFillInfo = diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index cad40b342a..ff2fb95328 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -443,7 +443,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI pInfo->intermediateBytes = (int16_t)sizeof(SPercentileInfo); } else if (functionId == FUNCTION_LEASTSQR) { pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = MAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string + pInfo->bytes = TMAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string pInfo->intermediateBytes = pInfo->bytes; } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { pInfo->type = TSDB_DATA_TYPE_BINARY; @@ -1331,7 +1331,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { continue; \ } \ (num) += 1; \ - (r) += POW2(((type *)d)[i] - (delta)); \ + (r) += TPOW2(((type *)d)[i] - (delta)); \ } static void stddev_function(SQLFunctionCtx *pCtx) { @@ -1370,7 +1370,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -1484,7 +1484,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -4210,8 +4210,8 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock pDist->totalRows += pSrc->totalRows; if (pResInfo->hasResult == DATA_SET_FLAG) { - pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); - pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); } else { pDist->maxRows = pSrc->maxRows; pDist->minRows = pSrc->minRows; diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c index 1803e282bf..4de11cbe71 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/function/src/tbinoperator.c @@ -26,7 +26,7 @@ void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1; int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { @@ -181,7 +181,7 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { } void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -220,7 +220,7 @@ void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int } void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -257,7 +257,7 @@ void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int } } void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -296,7 +296,7 @@ void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out } void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -342,7 +342,7 @@ void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double * output = (double *)out; @@ -416,7 +416,7 @@ void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *ou void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { int32_t len = pLeft->bytes + pRight->bytes; - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; char *output = (char *)out; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 5299a7dc5f..46b4c9d7c6 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -152,7 +152,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, uint64_t addPrefix = 0; uint8_t* data = fstSliceData(s, NULL); if (t && t->inp == data[i]) { - uint64_t commPrefix = MIN(t->out, *out); + uint64_t commPrefix = TMIN(t->out, *out); uint64_t tAddPrefix = t->out - commPrefix; (*out) = (*out) - commPrefix; t->out = commPrefix; @@ -244,8 +244,8 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil bool anyOuts = (node->finalOutput != 0); for (size_t i = 0; i < sz; i++) { FstTransition* t = taosArrayGet(node->trans, i); - tSize = MAX(tSize, packDeltaSize(addr, t->addr)); - oSize = MAX(oSize, packSize(t->out)); + tSize = TMAX(tSize, packDeltaSize(addr, t->addr)); + oSize = TMAX(oSize, packSize(t->out)); anyOuts = anyOuts || (t->out != 0); } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 8327b7c131..46321fc191 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1353,7 +1353,7 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf numOfFillVal = numOfFields; } } else { - numOfFillVal = MIN(num, numOfFields); + numOfFillVal = TMIN(num, numOfFields); } int32_t j = 1; @@ -1928,7 +1928,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { char uname[TSDB_COL_NAME_LEN] = {0}; - int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); if (tsKeepOriginalColumnName) { // keep the original column name @@ -1944,7 +1944,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } } else { // use the user-input result column name - int32_t len = MIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->pNode->exprToken.z, len); } } @@ -2948,7 +2948,7 @@ static SSchema createConstantColumnSchema(SVariant* pVal, const SToken* exprStr, if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); + size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1); tstrncpy(s.name, exprStr->z, tlen); strdequote(s.name); } @@ -3026,7 +3026,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* SSchema colSchema = createConstantColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); char token[TSDB_COL_NAME_LEN] = {0}; - tstrncpy(token, pItem->pNode->exprToken.z, MIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN)); + tstrncpy(token, pItem->pNode->exprToken.z, TMIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN)); STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &colSchema); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 8b1506bad1..a031199992 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -156,7 +156,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; - SWAP(dst->pData, src->pData, char*); + TSWAP(dst->pData, src->pData, char*); buildMsgHeader(dst); taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 1e1a97df1f..a1c1663512 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -135,7 +135,7 @@ static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks, SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter), pRoot); inserter->numOfTables = pBlocks->numOfTables; inserter->size = pBlocks->size; - SWAP(inserter->pData, pBlocks->pData, char*); + TSWAP(inserter->pData, pBlocks->pData, char*); return (SDataSink*)inserter; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 90a4388485..cfe2ed96ad 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,7 +85,7 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); diff --git a/source/libs/sync/inc/sync_type.h b/source/libs/sync/inc/sync_type.h index 9c4bc9e63c..c5c4cc3a76 100644 --- a/source/libs/sync/inc/sync_type.h +++ b/source/libs/sync/inc/sync_type.h @@ -43,12 +43,12 @@ typedef struct SSyncRaftLog SSyncRaftLog; typedef struct SSyncRaftEntry SSyncRaftEntry; #if 0 -#ifndef MIN -#define MIN(x, y) (((x) < (y)) ? (x) : (y)) +#ifndef TMIN +#define TMIN(x, y) (((x) < (y)) ? (x) : (y)) #endif -#ifndef MAX -#define MAX(x, y) (((x) > (y)) ? (x) : (y)) +#ifndef TMAX +#define TMAX(x, y) (((x) > (y)) ? (x) : (y)) #endif #endif diff --git a/source/libs/sync/src/sync_raft_progress.c b/source/libs/sync/src/sync_raft_progress.c index a3ab93c0fc..6577972b29 100644 --- a/source/libs/sync/src/sync_raft_progress.c +++ b/source/libs/sync/src/sync_raft_progress.c @@ -59,7 +59,7 @@ bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastInde probeAcked(progress); } - progress->nextIndex = MAX(progress->nextIndex, lastIndex + 1); + progress->nextIndex = TMAX(progress->nextIndex, lastIndex + 1); return updated; } @@ -100,7 +100,7 @@ bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress, return false; } - progress->nextIndex = MAX(MIN(rejected, matchHint + 1), 1); + progress->nextIndex = TMAX(TMIN(rejected, matchHint + 1), 1); progress->probeSent = false; return true; @@ -166,7 +166,7 @@ void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress) { if (progress->state == PROGRESS_STATE_SNAPSHOT) { SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex; resetProgressState(progress, PROGRESS_STATE_PROBE); - progress->nextIndex = MAX(progress->matchIndex + 1, pendingSnapshotIndex + 1); + progress->nextIndex = TMAX(progress->matchIndex + 1, pendingSnapshotIndex + 1); } else { resetProgressState(progress, PROGRESS_STATE_PROBE); progress->nextIndex = progress->matchIndex + 1; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index a3894ceedd..2d6fb8fc76 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -66,7 +66,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { struct stat statbuf; stat(fnameStr, &statbuf); - int readSize = MIN(WAL_MAX_SIZE + 2, statbuf.st_size); + int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; FileFd fd = taosOpenFileRead(fnameStr); diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 2b013bfdd0..0b0d8c4c58 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -67,7 +67,7 @@ static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = TMIN(length, HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1ad61ee2b0..c283423fbf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -751,56 +751,56 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { switch (field->type) { case TSDB_DATA_TYPE_BOOL: - return MAX(5, width); // 'false' + return TMAX(5, width); // 'false' case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - return MAX(4, width); // '-127' + return TMAX(4, width); // '-127' case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - return MAX(6, width); // '-32767' + return TMAX(6, width); // '-32767' case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: - return MAX(11, width); // '-2147483648' + return TMAX(11, width); // '-2147483648' case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: - return MAX(21, width); // '-9223372036854775807' + return TMAX(21, width); // '-9223372036854775807' case TSDB_DATA_TYPE_FLOAT: - return MAX(20, width); + return TMAX(20, width); case TSDB_DATA_TYPE_DOUBLE: - return MAX(25, width); + return TMAX(25, width); case TSDB_DATA_TYPE_BINARY: if (field->bytes > tsMaxBinaryDisplayWidth) { - return MAX(tsMaxBinaryDisplayWidth, width); + return TMAX(tsMaxBinaryDisplayWidth, width); } else { - return MAX(field->bytes, width); + return TMAX(field->bytes, width); } case TSDB_DATA_TYPE_NCHAR: { int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; if (bytes > tsMaxBinaryDisplayWidth) { - return MAX(tsMaxBinaryDisplayWidth, width); + return TMAX(tsMaxBinaryDisplayWidth, width); } else { - return MAX(bytes, width); + return TMAX(bytes, width); } } case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - return MAX(14, width); + return TMAX(14, width); } if (precision == TSDB_TIME_PRECISION_NANO) { - return MAX(29, width); + return TMAX(29, width); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - return MAX(26, width); // '2020-01-01 00:00:00.000000' + return TMAX(26, width); // '2020-01-01 00:00:00.000000' } else { - return MAX(23, width); // '2020-01-01 00:00:00.000' + return TMAX(23, width); // '2020-01-01 00:00:00.000' } default: From edc118e825b022fbc61f6123e741948ad783955e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jan 2022 12:57:42 +0800 Subject: [PATCH 091/118] rename MAX/MIN/POW2/SWAP/ABS name --- 2.0/src/client/src/tscSQLParser.c | 2 +- include/common/tdataformat.h | 2 +- include/libs/tfs/tfs.h | 2 +- include/os/osMath.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/libs/parser/src/astValidate.c | 2 +- source/os/src/osDir.c | 2 +- source/util/src/tlog.c | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/2.0/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c index 7d2d6f6ebf..1c50a2823e 100644 --- a/2.0/src/client/src/tscSQLParser.c +++ b/2.0/src/client/src/tscSQLParser.c @@ -8089,7 +8089,7 @@ int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); int64_t intervalRange = 0; if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index cc30cd78f5..58ede62b23 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,7 +136,7 @@ typedef uint64_t TKEY; #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) #define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) #define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) -#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) #define MIN_TS_KEY ((TSKEY)0x8000000000000001) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index e1f28092cf..289cfd9b04 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -33,7 +33,7 @@ typedef struct { typedef struct { SDiskID did; - char aname[TSDB_FILENAME_LEN]; // ABS name + char aname[TSDB_FILENAME_LEN]; // TABS name char rname[TSDB_FILENAME_LEN]; // REL name STfs *pTfs; } STfsFile; diff --git a/include/os/osMath.h b/include/os/osMath.h index 44a63500ab..948fbbf665 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -21,7 +21,7 @@ extern "C" { #endif #define TPOW2(x) ((x) * (x)) -#define ABS(x) ((x) > 0 ? (x) : -(x)) +#define TABS(x) ((x) > 0 ? (x) : -(x)) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4bc570c11d..efdacf957b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -234,7 +234,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { } bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { - int64_t interval = ABS(pDnode->lastAccessTime - curMs); + int64_t interval = TABS(pDnode->lastAccessTime - curMs); if (interval > 3500 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 46321fc191..563443d3c3 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1267,7 +1267,7 @@ static int32_t checkFillQueryRange(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) // return buildInvalidOperationMsg(pMsgBuf, msg1); // } - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); int64_t intervalRange = 0; if (!TIME_IS_VAR_DURATION(pQueryInfo->interval.intervalUnit)) { diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 070fe4a6c8..04efbef5b3 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -98,7 +98,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { } if (fileSec <= 100) continue; - int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1); + int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { (void)remove(filename); //printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c284efbcd4..c7b1350591 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -185,7 +185,7 @@ static void taosKeepOldLog(char *oldName) { } } - taosRemoveOldFiles(tsLogDir, ABS(tsLogKeepDays)); + taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); } static void *taosThreadToOpenNewFile(void *param) { From 65c12e7b1543f172c8e8b650aee2d4d365165ad4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jan 2022 13:04:44 +0800 Subject: [PATCH 092/118] rename MAX/MIN/POW2/SWAP/ABS name --- source/util/src/tstrbuild.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index eec21d1835..230bff42f5 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -70,11 +70,11 @@ void taosStringBuilderAppendNull(SStringBuilder* sb) { taosStringBuilderAppendSt void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) { char buf[64]; size_t len = snprintf(buf, sizeof(buf), "%" PRId64, v); - taosStringBuilderAppendStringLen(sb, buf, MIN(len, sizeof(buf))); + taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf))); } void taosStringBuilderAppendDouble(SStringBuilder* sb, double v) { char buf[512]; size_t len = snprintf(buf, sizeof(buf), "%.9lf", v); - taosStringBuilderAppendStringLen(sb, buf, MIN(len, sizeof(buf))); + taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf))); } From b3fadd5c932925be21e2f90b86b4340f1de95317 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 13:24:27 +0800 Subject: [PATCH 093/118] [td-11818]Fix memory leak in creating table. --- include/libs/scheduler/scheduler.h | 4 ++-- source/client/src/clientImpl.c | 1 + source/libs/scheduler/src/scheduler.c | 15 +++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 3262b9437c..53c8bb24e0 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -77,10 +77,10 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru /** * Process the query job, generated according to the query physical plan. * This is a asynchronized API, and is also thread-safety. - * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr + * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); +int32_t scheduleAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob); /** * Fetch query result from the remote query executor diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index cc86bf2650..95ffc91f7c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -501,6 +501,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { } _return: + taosArrayDestroy(pNodeList); qDestroyQuery(pQueryNode); if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 4c3ddd87ab..2e3ba69e8f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1288,9 +1288,9 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { - if (nodeList && taosArrayGetSize(nodeList) <= 0) { - qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); +int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { + if (pNodeList && taosArrayGetSize(pNodeList) <= 0) { + qDebug("QID:%"PRIx64" input exec nodeList is empty", pDag->queryId); } int32_t code = 0; @@ -1302,7 +1302,10 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; - pJob->nodeList = nodeList; + + if (pNodeList != NULL) { + pJob->nodeList = taosArrayDup(pNodeList); + } SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); @@ -1418,12 +1421,12 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t scheduleAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, false)); return TSDB_CODE_SUCCESS; } From 6632640d6701de5ec18cfca33d24046033d95f17 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 13:35:48 +0800 Subject: [PATCH 094/118] more tdb --- source/libs/tdb/src/db/tdb_db.c | 76 +++----------------------------- source/libs/tdb/src/inc/tdb_db.h | 20 ++++----- 2 files changed, 15 insertions(+), 81 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_db.c b/source/libs/tdb/src/db/tdb_db.c index 9fac9887d0..3675844535 100644 --- a/source/libs/tdb/src/db/tdb_db.c +++ b/source/libs/tdb/src/db/tdb_db.c @@ -13,76 +13,14 @@ * along with this program. If not, see . */ -#if 0 -#include "tdbDB.h" -#include "tdb.h" +#include "tdb_db.h" -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) { - TDB* dbp; - int ret; - - dbp = calloc(1, sizeof(*dbp)); - if (dbp == NULL) { - return -1; - } - - dbp->pageSize = TDB_DEFAULT_PGSIZE; - dbp->type = type; - - switch (type) { - case TDB_BTREE_T: - // ret = tdbInitBtreeDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HASH_T: - // ret = tdbInitHashDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HEAP_T: - // ret = tdbInitHeapDB(dbp); - // if (ret < 0) goto _err; - break; - default: - break; - } - - *dbpp = dbp; - return 0; - -_err: - if (dbp) { - free(dbp); - } - *dbpp = NULL; - return 0; -} - -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) { - int ret = 0; - - if ((dbp->fname = strdup(fname)) == NULL) { - ret = -1; - return ret; - } - - // Create the backup file if the file not exists - - // Open the file as a sub-db or a master-db - if (dbname) { - if ((dbp->dbname = strdup(dbname)) == NULL) { - ret = -1; - return ret; - } - // TODO: Open the DB as a SUB-DB in this file - } else { - // TODO: Open the DB as a MASTER-DB in this file - } - - return ret; -} - -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) { // TODO return 0; } -#endif \ No newline at end of file + +int tdbClose(TDB *dbp, uint32_t flags) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_db.h b/source/libs/tdb/src/inc/tdb_db.h index 7607580d3e..5f0529462b 100644 --- a/source/libs/tdb/src/inc/tdb_db.h +++ b/source/libs/tdb/src/inc/tdb_db.h @@ -16,32 +16,28 @@ #ifndef _TD_TDB_DB_H_ #define _TD_TDB_DB_H_ -#include "tdb.h" -#include "tdbBtree.h" +#include "tdb_mpool.h" #ifdef __cplusplus extern "C" { #endif -typedef struct { - int fd; -} TDB_FH; +typedef struct TDB TDB; struct TDB { - pgsize_t pageSize; - tdb_db_t type; - char * fname; - char * dbname; + char * fname; + char * dbname; + TDB_MPFILE *mpf; // union { // TDB_BTREE *btree; // TDB_HASH * hash; // TDB_HEAP * heap; // } dbam; // db access method - - // TDB_FH * fhp; // The backup file handle - // TDB_MPOOL *mph; // The memory pool handle }; +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags); +int tdbClose(TDB *dbp, uint32_t flags); + #ifdef __cplusplus } #endif From f7392620075f486272d36519b52fc668d37b39c0 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 24 Jan 2022 13:38:51 +0800 Subject: [PATCH 095/118] [modify insert] --- tests/test/c/create_table.c | 88 +++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index fddf3e7f2a..f1b6b6b502 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -31,6 +31,7 @@ int32_t createTable = 1; int32_t insertData = 0; int32_t batchNumOfTbl = 100; int32_t batchNumOfRow = 1; +int32_t totalRowsOfPerTbl = 1; int32_t numOfVgroups = 2; int32_t showTablesFlag = 0; int32_t queryFlag = 0; @@ -106,14 +107,14 @@ void printCreateProgress(SThreadInfo *pInfo, int64_t t) { totalTables, seconds, speed); } -void printInsertProgress(SThreadInfo *pInfo, int64_t t) { +void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) { int64_t endMs = taosGetTimestampMs(); - int64_t totalTables = t - pInfo->tableBeginIndex; + //int64_t totalTables = t - pInfo->tableBeginIndex; float seconds = (endMs - pInfo->startMs) / 1000.0; - float speed = totalTables / seconds; + float speed = insertTotalRows / seconds; pInfo->insertDataSpeed = speed; pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex, - totalTables, seconds, speed); + insertTotalRows, seconds, speed); } static int64_t getResult(TAOS_RES *tres) { @@ -181,14 +182,13 @@ void *threadFunc(void *param) { exit(1); } - pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - pInfo->tableEndIndex); + //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + // pInfo->tableEndIndex); pInfo->tableBeginIndex += startOffset; pInfo->tableEndIndex += startOffset; - pPrint("====after thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - pInfo->tableEndIndex); + pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); @@ -237,51 +237,50 @@ void *threadFunc(void *param) { } if (insertData) { + int64_t insertTotalRows = 0; int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); pInfo->startMs = beginMs; int64_t t = pInfo->tableBeginIndex; - for (; t <= pInfo->tableEndIndex;) { - // int64_t batch = (pInfo->tableEndIndex - t); - // batch = MIN(batch, batchNum); - - int32_t len = sprintf(qstr, "insert into "); - - for (int32_t i = 0; i < batchNumOfTbl;) { - int64_t ts = startTimestamp; + for (; t <= pInfo->tableEndIndex; t++) { + //printf("table name: %"PRId64"\n", t); + int64_t ts = startTimestamp; + for (int32_t i = 0; i < totalRowsOfPerTbl;) { + int32_t len = sprintf(qstr, "insert into "); len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); for (int32_t j = 0; j < batchNumOfRow; j++) { len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + i++; + insertTotalRows++; + if (i >= totalRowsOfPerTbl) { + break; + } } - t++; - i++; - if (t > pInfo->tableEndIndex) { - break; + #if 1 + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); } + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + printInsertProgress(pInfo, insertTotalRows); + } + #endif } - - int64_t startTs = taosGetTimestampUs(); - TAOS_RES *pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); - } - taos_free_result(pRes); - int64_t endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); - if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; - if (delay < pInfo->minDelay) pInfo->minDelay = delay; - - curMs = taosGetTimestampMs(); - if (curMs - beginMs > 10000) { - beginMs = curMs; - // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); - printInsertProgress(pInfo, t); - } - } - printInsertProgress(pInfo, t); + } + printInsertProgress(pInfo, insertTotalRows); } taos_close(con); @@ -319,6 +318,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); printf("%s%s\n", indent, "-l"); printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl); exit(EXIT_SUCCESS); } @@ -350,6 +351,8 @@ void parseArgument(int32_t argc, char *argv[]) { batchNumOfTbl = atoi(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { batchNumOfRow = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + totalRowsOfPerTbl = atoi(argv[++i]); } else if (strcmp(argv[i], "-w") == 0) { showTablesFlag = atoi(argv[++i]); } else if (strcmp(argv[i], "-q") == 0) { @@ -370,6 +373,7 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); + pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC); pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); From 02a3655780cd1b34da23375d1ed82348ea17f6d1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 13:46:55 +0800 Subject: [PATCH 096/118] [td-11818] merge 3.0 --- contrib/test/CMakeLists.txt | 2 +- source/client/src/clientImpl.c | 5 +---- source/libs/scheduler/src/scheduler.c | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 0c71113056..eacaeb9524 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -20,7 +20,7 @@ if(${BUILD_WITH_CRAFT}) endif(${BUILD_WITH_CRAFT}) if(${BUILD_WITH_TRAFT}) - add_subdirectory(traft) + # add_subdirectory(traft) endif(${BUILD_WITH_TRAFT}) add_subdirectory(tdev) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8d83146811..6014042e11 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -242,9 +242,6 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - - taosArrayDestroy(pNodeList); - int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry @@ -649,6 +646,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQueryNode = NULL; + SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -657,7 +655,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { - SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 06afda94f8..f90fdd6e11 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1432,7 +1432,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } From fde22bca3677758354609893a1f354fe22d1e816 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 21:59:09 -0800 Subject: [PATCH 097/118] manage func obj --- include/common/tmsg.h | 5 +- source/dnode/mnode/impl/inc/mndDef.h | 1 - source/dnode/mnode/impl/src/mndFunc.c | 193 +++++++++++++------------ source/dnode/mnode/impl/src/mndStb.c | 11 +- source/dnode/mnode/impl/src/mndTrans.c | 8 +- 5 files changed, 114 insertions(+), 104 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 73fe2b857c..d528f8dfb2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -526,9 +526,9 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t igExists; int8_t funcType; int8_t scriptType; - int8_t align; int8_t outputType; int32_t outputLen; int32_t bufSize; @@ -540,6 +540,7 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t igNotExists; } SDropFuncReq; typedef struct { @@ -549,9 +550,9 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t align; int8_t funcType; int8_t scriptType; - int8_t align; int8_t outputType; int32_t outputLen; int32_t bufSize; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ebbc42f277..d6b2e13fe0 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -289,7 +289,6 @@ typedef struct { int32_t codeSize; char *pComment; char *pCode; - char pData[]; } SFuncObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 402c48403b..15d5ef2d7a 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -25,7 +25,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); -static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew); static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); @@ -104,13 +104,11 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { goto FUNC_DECODE_OVER; } - int32_t size = sizeof(SFuncObj) + TSDB_FUNC_COMMENT_LEN + TSDB_FUNC_CODE_LEN; - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj)); if (pRow == NULL) goto FUNC_DECODE_OVER; SFuncObj *pFunc = sdbGetRowObj(pRow); if (pFunc == NULL) goto FUNC_DECODE_OVER; - char *tmp = (char *)pFunc + sizeof(SFuncObj); int32_t dataPos = 0; SDB_GET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, FUNC_DECODE_OVER) @@ -124,9 +122,15 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize, FUNC_DECODE_OVER) - pFunc->pComment = pFunc->pData; - pFunc->pCode = (pFunc->pData + pFunc->commentSize); + + pFunc->pComment = calloc(1, pFunc->commentSize); + pFunc->pCode = calloc(1, pFunc->codeSize); + if (pFunc->pComment == NULL || pFunc->pCode == NULL) { + goto FUNC_DECODE_OVER; + } + + SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER) terrno = 0; @@ -151,114 +155,104 @@ static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { return 0; } -static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) { - mTrace("func:%s, perform update action, old row:%p new row:%p", pOldFunc->name, pOldFunc, pNewFunc); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { + mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); return 0; } +static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) { + SSdb *pSdb = pMnode->pSdb; + SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName); + if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + } + return pFunc; +} + +static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pFunc); +} + static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { - SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); - pFunc->createdTime = taosGetTimestampMs(); - pFunc->funcType = pCreate->funcType; - pFunc->scriptType = pCreate->scriptType; - pFunc->outputType = pCreate->outputType; - pFunc->outputLen = pCreate->outputLen; - pFunc->bufSize = pCreate->bufSize; - pFunc->signature = pCreate->signature; - pFunc->commentSize = pCreate->commentSize; - pFunc->codeSize = pCreate->codeSize; - pFunc->pComment = pFunc->pData; - memcpy(pFunc->pComment, pCreate->pCont, pCreate->commentSize); - pFunc->pCode = pFunc->pData + pCreate->commentSize; - memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); + int32_t code = -1; + STrans *pTrans = NULL; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); - if (pTrans == NULL) { - free(pFunc); - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + SFuncObj func = {0}; + memcpy(func.name, pCreate->name, TSDB_FUNC_NAME_LEN); + func.createdTime = taosGetTimestampMs(); + func.funcType = pCreate->funcType; + func.scriptType = pCreate->scriptType; + func.outputType = pCreate->outputType; + func.outputLen = pCreate->outputLen; + func.bufSize = pCreate->bufSize; + func.signature = pCreate->signature; + func.commentSize = pCreate->commentSize; + func.codeSize = pCreate->codeSize; + func.pComment = malloc(func.commentSize); + func.pCode = malloc(func.codeSize); + if (func.pCode == NULL || func.pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto CREATE_FUNC_OVER; } + memcpy(func.pComment, pCreate->pCont, pCreate->commentSize); + memcpy(func.pCode, pCreate->pCont + pCreate->commentSize, func.codeSize); + + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) goto CREATE_FUNC_OVER; + mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto CREATE_FUNC_OVER; - SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto CREATE_FUNC_OVER; - SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto CREATE_FUNC_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_FUNC_OVER; - free(pFunc); + code = 0; + +CREATE_FUNC_OVER: + free(func.pCode); + free(func.pComment); mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); - if (pTrans == NULL) { - mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); - return -1; - } + if (pTrans == NULL) goto DROP_FUNC_OVER; + mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_FUNC_OVER; + code = 0; + +DROP_FUNC_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { @@ -273,11 +267,19 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { mDebug("func:%s, start to create", pCreate->name); - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pCreate->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode->pSdb, pCreate->name); if (pFunc != NULL) { - sdbRelease(pMnode->pSdb, pFunc); - terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + mndReleaseFunc(pMnode->pSdb, pFunc); + if (pCreate->igExists) { + mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); + return 0; + } else { + terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } else if (terrno != TSDB_CODE_MND_FUNC_ALREADY_EXIST) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -312,7 +314,6 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { } int32_t code = mndCreateFunc(pMnode, pReq, pCreate); - if (code != 0) { mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -333,15 +334,19 @@ static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { return -1; } - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pDrop->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode->pSdb, pDrop->name); if (pFunc == NULL) { - terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + if (pDrop->igNotExists) { + mDebug("func:%s, not exist, ignore not exist is set", pDrop->name); + return 0; + } else { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } } int32_t code = mndDropFunc(pMnode, pReq, pFunc); - if (code != 0) { mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 5ed801e3ea..cd1b69d023 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -123,8 +123,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { goto STB_DECODE_OVER; } - int32_t size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj)); if (pRow == NULL) goto STB_DECODE_OVER; SStbObj *pStb = sdbGetRowObj(pRow); @@ -143,6 +142,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; pStb->pSchema = calloc(totalCols, sizeof(SSchema)); + if (pStb->pSchema == NULL) { + goto STB_DECODE_OVER; + } for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pStb->pSchema[i]; @@ -448,7 +450,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCr stbObj.pSchema[i].colId = i + 1; } - int32_t code = 0; + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_STB_OVER; @@ -481,7 +483,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); if (pStb != NULL) { - sdbRelease(pMnode->pSdb, pStb); + mndReleaseStb(pMnode->pSdb, pStb); if (pCreate->igExists) { mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); return 0; @@ -492,6 +494,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; } // topic should have different name with stb diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 2301df65d7..03226c7400 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -390,9 +390,11 @@ static void mndTransDropActions(SArray *pArray) { } void mndTransDrop(STrans *pTrans) { - mndTransDropData(pTrans); - mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); - tfree(pTrans); + if (pTrans != NULL) { + mndTransDropData(pTrans); + mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); + tfree(pTrans); + } } static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { From 4c95270f582c329b68f1a1828970d7329ad61811 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 22:18:36 -0800 Subject: [PATCH 098/118] minor changes --- source/dnode/mnode/impl/src/mndFunc.c | 8 +++++--- source/dnode/mnode/impl/src/mndStb.c | 5 ++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 15d5ef2d7a..7aada921f3 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -267,9 +267,9 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { mDebug("func:%s, start to create", pCreate->name); - SFuncObj *pFunc = mndAcquireFunc(pMnode->pSdb, pCreate->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode, pCreate->name); if (pFunc != NULL) { - mndReleaseFunc(pMnode->pSdb, pFunc); + mndReleaseFunc(pMnode, pFunc); if (pCreate->igExists) { mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); return 0; @@ -334,7 +334,7 @@ static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { return -1; } - SFuncObj *pFunc = mndAcquireFunc(pMnode->pSdb, pDrop->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode, pDrop->name); if (pFunc == NULL) { if (pDrop->igNotExists) { mDebug("func:%s, not exist, ignore not exist is set", pDrop->name); @@ -347,6 +347,8 @@ static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { } int32_t code = mndDropFunc(pMnode, pReq, pFunc); + mndReleaseFunc(pMnode, pFunc); + if (code != 0) { mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index cd1b69d023..45b8428663 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -483,7 +483,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); if (pStb != NULL) { - mndReleaseStb(pMnode->pSdb, pStb); + mndReleaseStb(pMnode, pStb); if (pCreate->igExists) { mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); return 0; @@ -643,7 +643,7 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { DROP_STB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { @@ -668,7 +668,6 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { mndReleaseStb(pMnode, pStb); if (code != 0) { - terrno = code; mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); return -1; } From c19e7779df118091db8221b91eb75edd6f878376 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 06:27:28 +0000 Subject: [PATCH 099/118] fix memory leak --- source/dnode/vnode/src/meta/metaBDBImpl.c | 2 ++ source/dnode/vnode/src/vnd/vnodeBufferPool.c | 1 + source/dnode/vnode/src/vnd/vnodeMgr.c | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 3 +++ source/dnode/vnode/src/vnd/vnodeWrite.c | 9 ++++++--- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index d8ee48ff23..b9f7649913 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -588,6 +588,8 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { continue; + } else if (tbCfg.type == META_CHILD_TABLE) { + kvRowFree(tbCfg.ctbCfg.pTag) } return tbCfg.name; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 8df6b42566..434498eef5 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -71,6 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { + tfree(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 730155b75a..6829d778f0 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -119,6 +119,7 @@ static void* loop(void* arg) { pthread_mutex_unlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); + free(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 888520dc56..436dbab357 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -145,6 +145,9 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { _exit: + free(pSW); + free(pTbCfg->name); + free(pTbCfg); rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.pCont = pTbMetaMsg; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ccddfd56d8..169dc32c1c 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void *ptr = vnodeMalloc(pVnode, pMsg->contLen); + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -76,6 +76,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the requst struct + free(vCreateTbReq.stbCfg.pSchema); + free(vCreateTbReq.stbCfg.pTagSchema); + free(vCreateTbReq.name); break; case TDMT_VND_CREATE_TABLE: tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); @@ -129,7 +132,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } - + return 0; } From 03c19aa887676bfd4a81ab69ac7c904c23f4cee6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 06:44:19 +0000 Subject: [PATCH 100/118] fix memory leak --- source/dnode/vnode/src/meta/metaBDBImpl.c | 3 ++- source/dnode/vnode/src/tq/tq.c | 3 +++ source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c | 1 - source/dnode/vnode/src/vnd/vnodeQuery.c | 19 +++++++++++++++---- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index b9f7649913..c8c5759bd2 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -587,9 +587,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { + free(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { - kvRowFree(tbCfg.ctbCfg.pTag) + kvRowFree(tbCfg.ctbCfg.pTag); } return tbCfg.name; } else { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 49bbb77797..243b8d1b66 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -81,6 +81,9 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl } void tqClose(STQ* pTq) { + if (pTq) { + free(pTq); + } // TODO } diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index 5844ef0345..a2e3a0dc56 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -114,4 +114,3 @@ static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { free(pNode); } -} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 436dbab357..be30df6e3c 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -145,9 +145,15 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { _exit: + free(pSW->pSchema); free(pSW); free(pTbCfg->name); free(pTbCfg); + if (pTbCfg->type == META_SUPER_TABLE) { + free(pTbCfg->stbCfg.pTagSchema); + } else if (pTbCfg->type == META_SUPER_TABLE) { + kvRowFree(pTbCfg->ctbCfg.pTag); + } rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.pCont = pTbMetaMsg; @@ -159,8 +165,8 @@ _exit: return 0; } -static void freeItemHelper(void* pItem) { - char* p = *(char**)pItem; +static void freeItemHelper(void *pItem) { + char *p = *(char **)pItem; free(p); } @@ -190,14 +196,14 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { // TODO: temp debug, and should del when show tables command ok vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); if (numOfTables > 10000) { - numOfTables = 10000; + numOfTables = 10000; } metaCloseTbCursor(pCur); int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); + // int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; @@ -225,6 +231,11 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); + for (int i = 0; i < taosArrayGetSize(pArray); i++) { + name = *(char **)taosArrayGet(pArray, i); + free(name); + } + taosArrayDestroyEx(pArray, freeItemHelper); return 0; } From 748df84238d5f06e23253f95c68e3244db7a494f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 06:57:00 +0000 Subject: [PATCH 101/118] fix memory leakage --- source/dnode/vnode/src/meta/metaBDBImpl.c | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 5 +---- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index c8c5759bd2..dfdc144750 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -587,6 +587,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { + free(tbCfg.name); free(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 243b8d1b66..f3ebd5b284 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -82,6 +82,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl void tqClose(STQ* pTq) { if (pTq) { + tfree(pTq->path); free(pTq); } // TODO diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index a2e3a0dc56..14b9a5124f 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -114,3 +114,4 @@ static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { free(pNode); } +} diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index be30df6e3c..9c89e36903 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -217,6 +217,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); + free(n); } pFetchRsp->numOfRows = htonl(numOfTables); @@ -231,10 +232,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); - for (int i = 0; i < taosArrayGetSize(pArray); i++) { - name = *(char **)taosArrayGet(pArray, i); - free(name); - } taosArrayDestroyEx(pArray, freeItemHelper); return 0; From 76ce87e2d309aff428529b3fa707c189bc59cb59 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Sun, 23 Jan 2022 23:00:27 -0800 Subject: [PATCH 102/118] minor changes --- source/dnode/mnode/impl/src/mndFunc.c | 30 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 7aada921f3..087f82cd37 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -358,14 +358,21 @@ static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { } static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { + int32_t code = -1; SMnode *pMnode = pReq->pMnode; SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); - int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; + int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN; + int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs; SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); + if (pRetrieveRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FUNC_RETRIEVE_OVER; + } + pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); char *pOutput = pRetrieveRsp->pFuncInfos; @@ -373,16 +380,15 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { char funcName[TSDB_FUNC_NAME_LEN] = {0}; memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, funcName); + SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC; mError("func:%s, failed to retrieve since %s", funcName, terrstr()); - return -1; + goto FUNC_RETRIEVE_OVER; } SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; - - strncpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); + memcpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); pFuncInfo->funcType = pFunc->funcType; pFuncInfo->scriptType = pFunc->scriptType; pFuncInfo->outputType = pFunc->outputType; @@ -391,15 +397,21 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { pFuncInfo->signature = htobe64(pFunc->signature); pFuncInfo->commentSize = htonl(pFunc->commentSize); pFuncInfo->codeSize = htonl(pFunc->codeSize); - memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize); - + memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize); + memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize); pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; + mndReleaseFunc(pMnode, pFunc); } pReq->pCont = pRetrieveRsp; pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); - return 0; + code = 0; + +FUNC_RETRIEVE_OVER: + if (code != 0) rpcFreeCont(pRetrieveRsp); + + return code; } static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { @@ -461,7 +473,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, "show funcs"); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } From 2530f0863070a5193485ac5da10081c91daae185 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 24 Jan 2022 15:03:57 +0800 Subject: [PATCH 103/118] [add docker scripts] --- .../sh/dockerbuilder/addDnodeToCluster.sh | 36 +++++ .../script/sh/dockerbuilder/addFqdnToHost.sh | 46 ++++++ .../sh/dockerbuilder/clientCfg/taos.cfg | 4 + .../sh/dockerbuilder/createComposeYml.sh | 136 ++++++++++++++++++ .../sh/dockerbuilder/createDnodeStorage.sh | 90 ++++++++++++ .../sh/dockerbuilder/tdserver/Dockerfile | 42 ++++++ tests/script/sh/dockerbuilder/tmux.conf | 115 +++++++++++++++ 7 files changed, 469 insertions(+) create mode 100755 tests/script/sh/dockerbuilder/addDnodeToCluster.sh create mode 100755 tests/script/sh/dockerbuilder/addFqdnToHost.sh create mode 100644 tests/script/sh/dockerbuilder/clientCfg/taos.cfg create mode 100755 tests/script/sh/dockerbuilder/createComposeYml.sh create mode 100755 tests/script/sh/dockerbuilder/createDnodeStorage.sh create mode 100644 tests/script/sh/dockerbuilder/tdserver/Dockerfile create mode 100644 tests/script/sh/dockerbuilder/tmux.conf diff --git a/tests/script/sh/dockerbuilder/addDnodeToCluster.sh b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh new file mode 100755 index 0000000000..394a663b09 --- /dev/null +++ b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +dnodeNumber=1 + +while getopts "hn:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +serverPort=6030 + +for ((i=2; i<=${dnodeNumber}; i++)); do + taos -s "create dnode node${i} port ${serverPort};" ||: + echo "create dnode node${i} port ${serverPort};" +done + + + + diff --git a/tests/script/sh/dockerbuilder/addFqdnToHost.sh b/tests/script/sh/dockerbuilder/addFqdnToHost.sh new file mode 100755 index 0000000000..4d613d1dd0 --- /dev/null +++ b/tests/script/sh/dockerbuilder/addFqdnToHost.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +dnodeNumber=1 +subnet="172.33.0.0/16" + +while getopts "hn:s:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + s) + subnet=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -s [ subnet] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +addFqdnToHosts() { + index=$1 + ipPrefix=$2 + let ipIndex=index+1 + echo "${ipPrefix}.${ipIndex} node${i}" >> /etc/hosts +} + +ipPrefix=${subnet%.*} +for ((i=1; i<=${dnodeNumber}; i++)); do + addFqdnToHosts ${i} ${ipPrefix} +done + + + + diff --git a/tests/script/sh/dockerbuilder/clientCfg/taos.cfg b/tests/script/sh/dockerbuilder/clientCfg/taos.cfg new file mode 100644 index 0000000000..491216d5f6 --- /dev/null +++ b/tests/script/sh/dockerbuilder/clientCfg/taos.cfg @@ -0,0 +1,4 @@ +debugFlag 131 +firstEp node1:6030 +asyncLog 0 +telemetryReporting 0 diff --git a/tests/script/sh/dockerbuilder/createComposeYml.sh b/tests/script/sh/dockerbuilder/createComposeYml.sh new file mode 100755 index 0000000000..494e79318b --- /dev/null +++ b/tests/script/sh/dockerbuilder/createComposeYml.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# create docker-compose.yml + +set -e +#set -x + +# set parameters by default value +composeYmlFile="./docker-compose.yml" +dnodeNumber=1 +subnet="172.33.0.0/16" + +while getopts "hn:f:s:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + composeYmlFile=$(echo $OPTARG) + ;; + s) + subnet=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -f [ yml file] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "dnodeNumber=${dnodeNumber} composeYmlFile=${composeYmlFile}" + +createFirstSection() { + ymlFile=$1 + + echo "version: '3.7'" > ${ymlFile} + echo "" >> ${ymlFile} + echo "x-node: &x-node" >> ${ymlFile} + echo " build:" >> ${ymlFile} + echo " context: ." >> ${ymlFile} + echo " dockerfile: ./tdserver/Dockerfile" >> ${ymlFile} + echo " args:" >> ${ymlFile} + echo " - PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz" >> ${ymlFile} + echo " - EXTRACTDIR=TDengine-server-3.0.0.0" >> ${ymlFile} + echo " image: 'tdengine:3.0.0.0'" >> ${ymlFile} + echo " container_name: 'node1'" >> ${ymlFile} + echo " privileged: true" >> ${ymlFile} + echo " cap_add:" >> ${ymlFile} + echo " - ALL" >> ${ymlFile} + echo " stdin_open: true" >> ${ymlFile} + echo " tty: true" >> ${ymlFile} + echo " environment:" >> ${ymlFile} + echo ' TZ: "Asia/Shanghai"' >> ${ymlFile} + echo ' command: >' >> ${ymlFile} + echo ' sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&' >> ${ymlFile} + echo ' echo $TZ > /etc/timezone &&' >> ${ymlFile} + echo ' exec sysctl -w kernel.core_pattern=/corefile/core-%e-%p"' >> ${ymlFile} + echo " restart: always" >> ${ymlFile} + echo " hostname: node1" >> ${ymlFile} + echo " command: taosd" >> ${ymlFile} + echo " deploy:" >> ${ymlFile} + echo " resources:" >> ${ymlFile} + echo " limits:" >> ${ymlFile} + echo ' cpus: "2.00"' >> ${ymlFile} + echo " memory: 4G" >> ${ymlFile} + echo " reservations:" >> ${ymlFile} + echo ' cpus: "1.00"' >> ${ymlFile} + echo " memory: 500M" >> ${ymlFile} + echo " volumes:" >> ${ymlFile} + echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile} + echo ' - $PWD:/work' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/data:/var/lib/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/log:/var/log/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/cfg:/etc/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/core:/corefile' >> ${ymlFile} + echo "" >> ${ymlFile} + echo "networks:" >> ${ymlFile} + echo " tdnet:" >> ${ymlFile} + echo " ipam:" >> ${ymlFile} + echo " driver: default" >> ${ymlFile} + echo " config:" >> ${ymlFile} + echo " - subnet: ${subnet}" >> ${ymlFile} + echo "" >> ${ymlFile} + echo "services:" >> ${ymlFile} +} + +createSingleDnodesCfg() { + ymlFile=$1 + index=$2 + + ipPrefix=${subnet%.*} + + let ipIndex=index+1 + + echo " node${index}:" >> ${ymlFile} + echo " <<: *x-node" >> ${ymlFile} + echo " container_name: 'node${index}'" >> ${ymlFile} + echo " hostname: node${index}" >> ${ymlFile} + echo " networks:" >> ${ymlFile} + echo " tdnet:" >> ${ymlFile} + echo " ipv4_address: ${ipPrefix}.${ipIndex}" >> ${ymlFile} + echo " volumes:" >> ${ymlFile} + echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile} + echo " - \$PWD:/work" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/data:/var/lib/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/log:/var/log/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/cfg:/etc/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/core:/corefile" >> ${ymlFile} + echo "" >> ${ymlFile} +} + +createDnodesOfDockerComposeYml() { + ymlFile=$1 + dnodeNumber=$2 + + for ((i=1; i<=${dnodeNumber}; i++)); do + createSingleDnodesCfg ${ymlFile} ${i} + done +} + +######################################################################################## +############################### main process ########################################## + +createFirstSection ${composeYmlFile} +createDnodesOfDockerComposeYml ${composeYmlFile} ${dnodeNumber} + +echo "====create docker-compose.yml end====" +echo " " + + diff --git a/tests/script/sh/dockerbuilder/createDnodeStorage.sh b/tests/script/sh/dockerbuilder/createDnodeStorage.sh new file mode 100755 index 0000000000..df02c721b4 --- /dev/null +++ b/tests/script/sh/dockerbuilder/createDnodeStorage.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# setup test environment + +set -e +#set -x + +# set parameters by default value +dataRootDir="/data/dockerbuilder/storage" +dnodeNumber=1 +firstEp="node1:6030" + +while getopts "hn:r:f:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + firstEp=$(echo $OPTARG) + ;; + r) + dataRootDir=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -n [ dnode number] " + echo " -f [ first ep] " + echo " -r [ data root dir] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "dnodeNumber=${dnodeNumber} dataRootDir=${dataRootDir} firstEp=${firstEp}" + +createTaosCfg() { + cfgFile=$1/cfg/taos.cfg + #dataDir=$1/data + #logDir=$1/log + firstEp=$2 + fqdn=$3 + + echo "debugFlag 131" > ${cfgFile} + echo "firstEp ${firstEp}" >> ${cfgFile} + #echo "dataDir ${dataDir}" >> ${cfgFile} + #echo "logDir ${logDir}" >> ${cfgFile} + echo "fqdn ${fqdn}" >> ${cfgFile} + + echo "supportVnodes 1024" >> ${cfgFile} + echo "asyncLog 0" >> ${cfgFile} + echo "telemetryReporting 0" >> ${cfgFile} +} + +createDnodesDataDir() { + if [ -d ${dataRootDir} ]; then + rm -rf ${dataRootDir}/* + else + echo "${dataRootDir} not exist" + exit 1 + fi + + dnodeNumber=$1 + firstEp=$2 + + serverPort=${startPort} + for ((i=1; i<=${dnodeNumber}; i++)); do + mkdir -p ${dataRootDir}/dnode${i}/cfg + mkdir -p ${dataRootDir}/dnode${i}/log + mkdir -p ${dataRootDir}/dnode${i}/data + mkdir -p ${dataRootDir}/dnode${i}/core + + createTaosCfg ${dataRootDir}/dnode${i} ${firstEp} node${i} + done +} + +######################################################################################## +############################### main process ########################################## + +## create director and taos.cfg for all dnode +createDnodesDataDir ${dnodeNumber} ${firstEp} + +echo "====create end====" +echo " " + + diff --git a/tests/script/sh/dockerbuilder/tdserver/Dockerfile b/tests/script/sh/dockerbuilder/tdserver/Dockerfile new file mode 100644 index 0000000000..7757aa68c7 --- /dev/null +++ b/tests/script/sh/dockerbuilder/tdserver/Dockerfile @@ -0,0 +1,42 @@ +FROM centos:8 AS builder + +ARG PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz +ARG EXTRACTDIR=TDengine-server-3.0.0.0 + +WORKDIR /root + +COPY ${PACKAGE} . +COPY tmux.conf . +COPY addDnodeToCluster.sh . + +RUN tar -zxf ${PACKAGE} +RUN mv ${EXTRACTDIR}/* ./ + +FROM centos:8 + +WORKDIR /root + +RUN yum install -y glibc-langpack-en dmidecode gdb +RUN yum install -y tmux net-tools +RUN yum install -y sysstat +RUN yum install -y vim +RUN echo 'alias ll="ls -l --color=auto"' >> ~/.bashrc + +COPY --from=builder /root/taosd /usr/bin +COPY --from=builder /root/taos /usr/bin +COPY --from=builder /root/create_table /usr/bin +COPY --from=builder /root/tmux.conf /root/.tmux.conf +#COPY --from=builder /root/addDnodeToCluster.sh /root/addDnodeToCluster.sh + +#COPY --from=builder /root/cfg/taos.cfg /etc/taos/ +COPY --from=builder /root/lib/* /usr/lib/ + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +EXPOSE 6030-6042/tcp 6060/tcp 6030-6039/udp + +# VOLUME [ "/var/lib/taos", "/var/log/taos", "/etc/taos" ] + +CMD [ "bash" ] diff --git a/tests/script/sh/dockerbuilder/tmux.conf b/tests/script/sh/dockerbuilder/tmux.conf new file mode 100644 index 0000000000..d1f1801f4a --- /dev/null +++ b/tests/script/sh/dockerbuilder/tmux.conf @@ -0,0 +1,115 @@ +unbind C-b +# remap prefix to Control + z +set -g prefix C-a +# bind 'C-z C-z' to type 'C-z' +bind C-a send-prefix + + +# 0 is too far from ` ;) +set -g base-index 1 + +# Automatically set window title +set-window-option -g automatic-rename on +set-option -g set-titles on + +#set -g default-terminal screen-256color +set -g status-keys vi +set -g history-limit 10000 + +setw -g mode-keys vi +#setw -g mode-mouse on +setw -g monitor-activity on +set -g mouse on + +bind-key v split-window -h -c "#{pane_current_path}" +bind-key s split-window -v -c "#{pane_current_path}" +bind-key c new-window -c "#{pane_current_path}" + +bind-key -n C-S-Left swap-window -t -1 +bind-key -n C-S-Right swap-window -t +1 + +# bind-key J resize-pane -D 5 +# bind-key K resize-pane -U 5 +# bind-key H resize-pane -L 5 +# bind-key L resize-pane -R 5 + +# bind-key M-j resize-pane -D +# bind-key M-k resize-pane -U +# bind-key M-h resize-pane -L +# bind-key M-l resize-pane -R + +# Vim style pane selection +bind h select-pane -L +bind j select-pane -D +bind k select-pane -U +bind l select-pane -R +bind K kill-pane -a + +# Use Alt-vim keys without prefix key to switch panes +bind -n M-h select-pane -L +bind -n M-j select-pane -D +bind -n M-k select-pane -U +bind -n M-l select-pane -R +# bind -n M-h select-pane -L \; resize-pane -Z +# bind -n M-j select-pane -D \; resize-pane -Z +# bind -n M-k select-pane -U \; resize-pane -Z +# bind -n M-l select-pane -R \; resize-pane -Z + +# # Use Alt-arrow keys without prefix key to switch panes +bind -n M-Left select-pane -L +bind -n M-Right select-pane -R +bind -n M-Up select-pane -U +bind -n M-Down select-pane -D + +# Shift arrow to switch windows +bind -n C-Left previous-window +bind -n C-Right next-window + +bind -n M-n previous-window +bind -n M-p nejt-window + +# Map Alt-z to zoom in and out pane +bind -n M-z resize-pane -Z +bind -n M-q resize-pane -Z + +# # Use v to trigger selection +bind-key -T copy-mode-vi v send-keys -X begin-selection +# +# # Use y to yank current selection +bind-key -T copy-mode-vi y send-keys -X copy-selection-and-cancel + +# No delay for escape key press +set -sg escape-time 0 + +# THEME +set -g status-style bg=black,fg=white +# set -g status-bg black +# set -g status-fg white +set -g window-status-current-style bg=green,fg=black,bold +set -g pane-active-border-style fg=green,bg=black +set -g status-interval 60 +set -g status-left-length 30 +set -g status-left '#[fg=green](#S) #(whoami)' +set -g status-right '#[fg=yellow]#(cut -d " " -f 1-3 /proc/loadavg)#[default] #[fg=white]%H:%M#[default]' + +# synchronouse windows +# setw synchronize-panes + +# pane movement +bind-key b command-prompt -p "join pane from:" "join-pane -s ':%%'" +bind-key a command-prompt -p "send pane to:" "join-pane -t ':%%'" +# # for vim +# # Smart pane switching with awareness of Vim splits. +# # See: https://github.com/christoomey/vim-tmux-navigator +# is_vim="ps -o state= -o comm= -t '#{pane_tty}' \ +# | grep -iqE '^[^TXZ ]+ +(\\S+\\/)?g?(view|n?vim?x?)(diff)?$'" +# bind-key -n C-h if-shell "$is_vim" "send-keys C-h" "select-pane -L" +# bind-key -n C-j if-shell "$is_vim" "send-keys C-j" "select-pane -D" +# bind-key -n C-k if-shell "$is_vim" "send-keys C-k" "select-pane -U" +# bind-key -n C-l if-shell "$is_vim" "send-keys C-l" "select-pane -R" +# bind-key -n C-\ if-shell "$is_vim" "send-keys C-\\" "select-pane -l" +# bind-key -T copy-mode-vi C-h select-pane -L +# bind-key -T copy-mode-vi C-j select-pane -D +# bind-key -T copy-mode-vi C-k select-pane -U +# bind-key -T copy-mode-vi C-l select-pane -R +# bind-key -T copy-mode-vi C-\ select-pane -l From cc34ba1afd30d5ffc6151ab536da8fb1ac1d404d Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 24 Jan 2022 15:34:37 +0800 Subject: [PATCH 104/118] [modify] --- tests/script/sh/massiveTable/deployCluster.sh | 20 ++++++++----------- tests/script/sh/massiveTable/setupDnodes.sh | 6 ++++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index beb324321d..503f3ccae2 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -5,10 +5,12 @@ set -e #set -x +fqdn=`hostname` + masterDnode=slave dataRootDir="/data" -firstEp="trd02:7000" -startPort=7000 +firstEp="${fqdn}:6030" +startPort=6030 dnodeNumber=1 updateSrc=no @@ -52,23 +54,17 @@ done curr_dir=$(readlink -f "$(dirname "$0")") echo $curr_dir -${curr_dir}/cleanCluster.sh -r "/data" -#${curr_dir}/cleanCluster.sh -r "/data2" +${curr_dir}/cleanCluster.sh -r ${dataRootDir} if [[ "${updateSrc}" == "yes" ]]; then ${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" fi -${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000 -#${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 +${curr_dir}/setupDnodes.sh -r ${dataRootDir} -n ${dnodeNumber} -f ${firstEp} -p ${startPort} if [[ "${masterDnode}" == "master" ]]; then - # create all dnode into cluster - #taos -s "create dnode trd02 port 8000;" - taos -s "create dnode trd03 port 7000;" - #taos -s "create dnode trd03 port 8000;" - taos -s "create dnode trd04 port 7000;" - #taos -s "create dnode trd04 port 8000;" + taos -s "create dnode trd03 port 6030;" + taos -s "create dnode trd04 port 6030;" fi diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index 37fdbaf784..22c325cd89 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -13,10 +13,12 @@ set -e # -r [ dnode root dir] # set parameters by default value +fqdn=`hostname` + enviMode=new dataRootDir="/data" -firstEp="localhost:7000" -startPort=7000 +firstEp="${fqdn}:6030" +startPort=6030 dnodeNumber=1 From 578d6617f51a9e4fbb99249af728360b6150d604 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 07:45:49 +0000 Subject: [PATCH 105/118] fix coredump --- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 9c89e36903..f619499c8d 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -217,7 +217,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); - free(n); + // free(n); } pFetchRsp->numOfRows = htonl(numOfTables); From 1ee5dbb9caf2ef8172a02de397cae81ca8048843 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 24 Jan 2022 16:14:15 +0800 Subject: [PATCH 106/118] [modify] --- tests/script/sh/dockerbuilder/addDnodeToCluster.sh | 8 +++++--- .../script/sh/dockerbuilder/createDnodeStorage.sh | 14 ++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/script/sh/dockerbuilder/addDnodeToCluster.sh b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh index 394a663b09..1886632350 100755 --- a/tests/script/sh/dockerbuilder/addDnodeToCluster.sh +++ b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh @@ -6,13 +6,17 @@ set -e #set -x dnodeNumber=1 +serverPort=6030 -while getopts "hn:" arg +while getopts "hn:p:" arg do case $arg in n) dnodeNumber=$(echo $OPTARG) ;; + p) + serverPort=$(echo $OPTARG) + ;; h) echo "Usage: `basename $0` -n [ dnode number] " exit 0 @@ -24,8 +28,6 @@ do esac done -serverPort=6030 - for ((i=2; i<=${dnodeNumber}; i++)); do taos -s "create dnode node${i} port ${serverPort};" ||: echo "create dnode node${i} port ${serverPort};" diff --git a/tests/script/sh/dockerbuilder/createDnodeStorage.sh b/tests/script/sh/dockerbuilder/createDnodeStorage.sh index df02c721b4..4e9f8e5121 100755 --- a/tests/script/sh/dockerbuilder/createDnodeStorage.sh +++ b/tests/script/sh/dockerbuilder/createDnodeStorage.sh @@ -40,20 +40,18 @@ echo "dnodeNumber=${dnodeNumber} dataRootDir=${dataRootDir} firstEp=${firstEp}" createTaosCfg() { cfgFile=$1/cfg/taos.cfg - #dataDir=$1/data - #logDir=$1/log firstEp=$2 fqdn=$3 - echo "debugFlag 131" > ${cfgFile} - echo "firstEp ${firstEp}" >> ${cfgFile} + echo "debugFlag 131" > ${cfgFile} + echo "firstEp ${firstEp}" >> ${cfgFile} #echo "dataDir ${dataDir}" >> ${cfgFile} #echo "logDir ${logDir}" >> ${cfgFile} - echo "fqdn ${fqdn}" >> ${cfgFile} + echo "fqdn ${fqdn}" >> ${cfgFile} - echo "supportVnodes 1024" >> ${cfgFile} - echo "asyncLog 0" >> ${cfgFile} - echo "telemetryReporting 0" >> ${cfgFile} + echo "supportVnodes 1024" >> ${cfgFile} + echo "asyncLog 0" >> ${cfgFile} + echo "telemetryReporting 0" >> ${cfgFile} } createDnodesDataDir() { From f15c2cebb4923ec8e887c1c70e1aa75679db0622 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 24 Jan 2022 16:20:36 +0800 Subject: [PATCH 107/118] [add release.sh] --- packaging/release.sh | 79 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100755 packaging/release.sh diff --git a/packaging/release.sh b/packaging/release.sh new file mode 100755 index 0000000000..8ea30223a6 --- /dev/null +++ b/packaging/release.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os + +set -e +#set -x + +# set parameters by default value +version="3.0.0.0" + +curr_dir=$(pwd) + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/..)" + +echo "=======================new version number: ${verNumber}======================================" + +build_time=$(date +"%F %R") + +echo "top_dir: ${top_dir}" + +cd ${top_dir} +git pull || : + +echo "curr_dir: ${curr_dir}" + +# 2. cmake executable file +compile_dir="${top_dir}/debug" +if [ -d ${compile_dir} ]; then + rm -rf ${compile_dir} +fi + +mkdir -p ${compile_dir} + +cd ${compile_dir} + +echo "compile_dir: ${compile_dir}" + +cmake .. +make -j32 + +release_dir="${top_dir}/release" +if [ -d ${release_dir} ]; then + rm -rf ${release_dir} +fi + +mkdir -p ${release_dir} +cd ${release_dir} + +install_dir="${release_dir}/TDengine-server-${version}" +mkdir -p ${install_dir} +mkdir -p ${install_dir}/lib + +bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table" +cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || : + + +cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/ +cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/ +cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/ + +pkg_name=${install_dir}-Linux-x64 + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : + + From bae5e5c41a48c0749d427eea00d27550d62e15f3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 08:24:08 +0000 Subject: [PATCH 108/118] fix write bug --- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index ca2b1069f9..be46f63168 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -359,7 +359,7 @@ void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd) { tsdbResetFSStatus(pfs->nstatus); pfs->nstatus->meta = pfs->cstatus->meta; // if (pfs->cstatus->pmf == NULL) { - pfs->nstatus->meta.version = 0; + pfs->nstatus->meta.version += 1; // } else { // pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1; // } From 022b5f85d8745416152e8ac022f2e7326af73b7d Mon Sep 17 00:00:00 2001 From: Shengliang Date: Mon, 24 Jan 2022 01:14:31 -0800 Subject: [PATCH 109/118] add test cases --- include/util/taoserror.h | 1 + include/util/tdef.h | 1 + source/dnode/mgmt/impl/test/sut/inc/sut.h | 9 + source/dnode/mnode/impl/src/mndFunc.c | 10 +- source/dnode/mnode/impl/test/CMakeLists.txt | 1 + .../dnode/mnode/impl/test/func/CMakeLists.txt | 11 + source/dnode/mnode/impl/test/func/func.cpp | 607 ++++++++++++++++++ source/util/src/terror.c | 1 + 8 files changed, 639 insertions(+), 2 deletions(-) create mode 100644 source/dnode/mnode/impl/test/func/CMakeLists.txt create mode 100644 source/dnode/mnode/impl/test/func/func.cpp diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8c048690c0..69bf085491 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -239,6 +239,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x03C4) #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) +#define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x03C7) // mnode-trans #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4c29d9963d..4d3ef06bd6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -173,6 +173,7 @@ do { \ #define TSDB_FUNC_BUF_SIZE 512 #define TSDB_FUNC_TYPE_SCALAR 1 #define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 9f724faeb9..955f62610a 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -96,6 +96,15 @@ class Testbase { #define CheckBinary(val, len) \ { EXPECT_STREQ(test.GetShowBinary(len), val); } +#define CheckBinaryByte(b, len) \ + { \ + char* bytes = (char*)calloc(1, len); \ + for (int32_t i = 0; i < len - 1; ++i) { \ + bytes[i] = b; \ + } \ + EXPECT_STREQ(test.GetShowBinary(len), bytes); \ + } + #define CheckInt8(val) \ { EXPECT_EQ(test.GetShowInt8(), val); } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 087f82cd37..87de215168 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -152,6 +152,8 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc); + tfree(pFunc->pCode); + tfree(pFunc->pComment); return 0; } @@ -278,7 +280,7 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } - } else if (terrno != TSDB_CODE_MND_FUNC_ALREADY_EXIST) { + } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -307,7 +309,7 @@ static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { return -1; } - if (pCreate->bufSize < 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + if (pCreate->bufSize <= 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -363,6 +365,10 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); + if (pRetrieve->numOfFuncs <= 0 || pRetrieve->numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { + terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; + return -1; + } int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN; int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs; diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 3ca35d58a7..5f6f2f3b4f 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -12,3 +12,4 @@ add_subdirectory(dnode) add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) +add_subdirectory(func) diff --git a/source/dnode/mnode/impl/test/func/CMakeLists.txt b/source/dnode/mnode/impl/test/func/CMakeLists.txt new file mode 100644 index 0000000000..26b0a60968 --- /dev/null +++ b/source/dnode/mnode/impl/test/func/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. FUNC_SRC) +add_executable(mnode_test_func ${FUNC_SRC}) +target_link_libraries( + mnode_test_func + PUBLIC sut +) + +add_test( + NAME mnode_test_func + COMMAND mnode_test_func +) diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp new file mode 100644 index 0000000000..c4e574906f --- /dev/null +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -0,0 +1,607 @@ +/** + * @file func.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module func tests + * @version 1.0 + * @date 2022-01-24 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestFunc : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} +}; + +Testbase MndTestFunc::test; + +TEST_F(MndTestFunc, 01_Show_Func) { + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, PATH_MAX + VARSTR_HEADER_SIZE, "comment"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "aggregate"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE, "outputtype"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_INT, 4, "code_len"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_INT, 4, "bufsize"); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(MndTestFunc, 02_Create_Func) { + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, ""); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN + 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN - 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 0; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 'a'; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 'a'; + pReq->bufSize = htonl(TSDB_FUNC_BUF_SIZE + 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); + } + + for (int32_t i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SCreateFuncReq); + int32_t commentSize = TSDB_FUNC_COMMENT_LEN; + int32_t codeSize = TSDB_FUNC_CODE_LEN; + contLen = (contLen + codeSize + commentSize); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->igExists = 0; + if (i == 2) pReq->igExists = 1; + pReq->funcType = 1; + pReq->scriptType = 2; + pReq->outputType = TSDB_DATA_TYPE_SMALLINT; + pReq->outputLen = htonl(12); + pReq->bufSize = htonl(4); + pReq->signature = htobe64(5); + pReq->commentSize = htonl(commentSize); + pReq->codeSize = htonl(codeSize); + for (int32_t i = 0; i < commentSize - 1; ++i) { + pReq->pCont[i] = 'm'; + } + for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { + pReq->pCont[i] = 'd'; + } + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (i == 0 || i == 2) { + ASSERT_EQ(pRsp->code, 0); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_ALREADY_EXIST); + } + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("f1", TSDB_FUNC_NAME_LEN); + CheckBinaryByte('m', TSDB_FUNC_COMMENT_LEN); + CheckInt32(0); + CheckBinary("SMALLINT", TSDB_TYPE_STR_MAX_LEN); + CheckTimestamp(); + CheckInt32(TSDB_FUNC_CODE_LEN); + CheckInt32(4); +} + +TEST_F(MndTestFunc, 03_Retrieve_Func) { + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); + EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + comments[i] = 'm'; + } + char codes[TSDB_FUNC_CODE_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + codes[i] = 'd'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 0; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + int32_t commentSize = 1024; + int32_t codeSize = 9527; + contLen = (contLen + codeSize + commentSize); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f2"); + pReq->igExists = 1; + pReq->funcType = 2; + pReq->scriptType = 3; + pReq->outputType = TSDB_DATA_TYPE_BINARY; + pReq->outputLen = htonl(24); + pReq->bufSize = htonl(6); + pReq->signature = htobe64(18); + pReq->commentSize = htonl(commentSize); + pReq->codeSize = htonl(codeSize); + for (int32_t i = 0; i < commentSize - 1; ++i) { + pReq->pCont[i] = 'p'; + } + for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { + pReq->pCont[i] = 'q'; + } + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f2"); + EXPECT_EQ(pFuncInfo->funcType, 2); + EXPECT_EQ(pFuncInfo->scriptType, 3); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pFuncInfo->outputLen, 24); + EXPECT_EQ(pFuncInfo->bufSize, 6); + EXPECT_EQ(pFuncInfo->signature, 18); + EXPECT_EQ(pFuncInfo->commentSize, 1024); + EXPECT_EQ(pFuncInfo->codeSize, 9527); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char* comments = (char*)calloc(1, 1024); + for (int32_t i = 0; i < 1024 - 1; ++i) { + comments[i] = 'p'; + } + char* codes = (char*)calloc(1, 9527); + for (int32_t i = 0; i < 9527 - 1; ++i) { + codes[i] = 'q'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + free(comments); + free(codes); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 2; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f2"); + strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + { + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f2"); + EXPECT_EQ(pFuncInfo->funcType, 2); + EXPECT_EQ(pFuncInfo->scriptType, 3); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pFuncInfo->outputLen, 24); + EXPECT_EQ(pFuncInfo->bufSize, 6); + EXPECT_EQ(pFuncInfo->signature, 18); + EXPECT_EQ(pFuncInfo->commentSize, 1024); + EXPECT_EQ(pFuncInfo->codeSize, 9527); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char* comments = (char*)calloc(1, 1024); + for (int32_t i = 0; i < 1024 - 1; ++i) { + comments[i] = 'p'; + } + char* codes = (char*)calloc(1, 9527); + for (int32_t i = 0; i < 9527 - 1; ++i) { + codes[i] = 'q'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + free(comments); + free(codes); + } + + { + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos + sizeof(SFuncInfo) + 1024 + 9527); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + EXPECT_STREQ(pFuncInfo->name, "f1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); + EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + + // char* pComment = pFuncInfo->pCont; + // char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + // char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + // for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + // comments[i] = 'm'; + // } + // char codes[TSDB_FUNC_CODE_LEN] = {0}; + // for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + // codes[i] = 'd'; + // } + // EXPECT_STREQ(pComment, comments); + // EXPECT_STREQ(pCode, codes); + } + } +} + +#if 0 + +TEST_F(MndTestFunc, 04_Drop_Func) { + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, ""); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_FORMAT); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, "u4"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, "u1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 4); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); +} + +TEST_F(MndTestFunc, 05_Create_Drop_Alter_Func) { + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, "u2"); + strcpy(pReq->pass, "p2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 4); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 3); + + CheckBinary("u1", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("u2", TSDB_FUNC_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + + { + int32_t contLen = sizeof(SAlterFuncReq); + + SAlterFuncReq* pReq = (SAlterFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 4); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 3); + + CheckBinary("u1", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("u2", TSDB_FUNC_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->user, "u1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 4); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("u2", TSDB_FUNC_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); + + // restart + test.Restart(); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 4); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("u2", TSDB_FUNC_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("root", TSDB_FUNC_LEN); +} + +#endif \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a67b6515b..56919ff99e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -248,6 +248,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") // mnode-trans TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") From 6ca824b7969d80e0c8afc31ae3607dd1c9bbda16 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Mon, 24 Jan 2022 01:26:10 -0800 Subject: [PATCH 110/118] cache the fqdn --- source/libs/transport/src/rpcMain.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index d870ae98ab..3c58a76a44 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,6 +42,8 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +SHashObj *tsFqdnHash; + #ifndef USE_UV typedef struct { @@ -215,6 +217,8 @@ static void rpcInitImp(void) { tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); + + tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } int32_t rpcInit(void) { @@ -224,6 +228,9 @@ int32_t rpcInit(void) { void rpcCleanup(void) { taosCloseRef(tsRpcRefId); + taosHashClear(tsFqdnHash); + taosHashCleanup(tsFqdnHash); + tsFqdnHash = NULL; tsRpcRefId = -1; } @@ -571,7 +578,17 @@ static void rpcFreeMsg(void *msg) { static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; - uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); + uint32_t peerIp = 0; + uint32_t *pPeerIp = taosHashGet(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1); + if (pPeerIp != NULL) { + peerIp = *pPeerIp; + } else { + peerIp = taosGetIpv4FromFqdn(peerFqdn); + if (peerIp != 0xFFFFFFFF) { + taosHashPut(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1, &peerIp, sizeof(peerIp)); + } + } + if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); terrno = TSDB_CODE_RPC_FQDN_ERROR; From bf26efb5187aae26bd76e8ad2732239bf2407acb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Jan 2022 17:44:44 +0800 Subject: [PATCH 111/118] [td-11818] Fix memory leak in query. --- include/libs/parser/parser.h | 1 + source/client/src/clientImpl.c | 1 + source/libs/parser/inc/dataBlockMgt.h | 2 ++ source/libs/parser/src/astValidate.c | 32 ++++++++++++++----- source/libs/parser/src/dataBlockMgt.c | 23 ++++++++++++-- source/libs/parser/src/insertParser.c | 20 +++++++++++- source/libs/parser/src/parser.c | 11 +++++-- source/libs/parser/src/parserUtil.c | 12 +------- source/libs/parser/src/queryInfoUtil.c | 7 ++++- source/libs/planner/src/logicPlan.c | 15 ++++++--- source/libs/planner/src/physicalPlan.c | 36 ++++++++++++++++++++++ source/libs/planner/src/physicalPlanJson.c | 6 ++-- source/libs/planner/src/planner.c | 20 +----------- source/libs/scheduler/src/scheduler.c | 2 -- 14 files changed, 134 insertions(+), 54 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index ecfa344f85..1d193888c9 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -74,6 +74,7 @@ void columnListCopy(SArray* dst, const SArray* src, uint64_t uid); void columnListDestroy(SArray* pColumnList); void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel); +void dropOneLevelExprInfo(SArray* pExprInfo); typedef struct SSourceParam { SArray *pExprNodeList; //Array diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6014042e11..0115d7f43d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -218,6 +218,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, if (pQueryNode->type == TSDB_SQL_SELECT) { setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); + tfree(pSchema); pRequest->type = TDMT_VND_QUERY; } else { tfree(pSchema); diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index cd84222c65..f53c5ea279 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -171,6 +171,8 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); +void destroyBlockHashmap(SHashObj* pDataBlockHash); + int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 563443d3c3..0a2f2b20f2 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -213,10 +213,11 @@ SQueryStmtInfo *createQueryInfo() { pQueryInfo->slimit.limit = -1; pQueryInfo->slimit.offset = 0; - pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES); pQueryInfo->window = TSWINDOW_INITIALIZER; pQueryInfo->exprList = calloc(10, POINTER_BYTES); + for(int32_t i = 0; i < 10; ++i) { pQueryInfo->exprList[i] = taosArrayInit(4, POINTER_BYTES); } @@ -232,7 +233,8 @@ static void destroyQueryInfoImpl(SQueryStmtInfo* pQueryInfo) { cleanupFieldInfo(&pQueryInfo->fieldsInfo); dropAllExprInfo(pQueryInfo->exprList, 10); - pQueryInfo->exprList = NULL; + + tfree(pQueryInfo->exprList); columnListDestroy(pQueryInfo->colList); pQueryInfo->colList = NULL; @@ -258,10 +260,10 @@ void destroyQueryInfo(SQueryStmtInfo* pQueryInfo) { size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pDownstream); for (int32_t i = 0; i < numOfUpstream; ++i) { - SQueryStmtInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pDownstream, i); - destroyQueryInfoImpl(pUpQueryInfo); - clearAllTableMetaInfo(pUpQueryInfo, false, 0); - tfree(pUpQueryInfo); + SQueryStmtInfo* pDownstream = taosArrayGetP(pQueryInfo->pDownstream, i); + destroyQueryInfoImpl(pDownstream); + clearAllTableMetaInfo(pDownstream, false, 0); + tfree(pDownstream); } destroyQueryInfoImpl(pQueryInfo); @@ -1395,6 +1397,13 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf static void pushDownAggFuncExprInfo(SQueryStmtInfo* pQueryInfo); static void addColumnNodeFromLowerLevel(SQueryStmtInfo* pQueryInfo); +static void freeItemHelper(void* pItem) { + void** p = pItem; + if (*p != NULL) { + tfree(*p); + } +} + int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); @@ -1590,7 +1599,10 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]); extractFunctionDesc(functionList, &pQueryInfo->info); - if ((code = checkForInvalidExpr(pQueryInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { + code = checkForInvalidExpr(pQueryInfo, pMsgBuf); + taosArrayDestroyEx(functionList, freeItemHelper); + + if (code != TSDB_CODE_SUCCESS) { return code; } } @@ -2902,6 +2914,8 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S } pQueryInfo->info.projectionQuery = true; + + taosArrayDestroy(pColumnList); return TSDB_CODE_SUCCESS; } @@ -3983,5 +3997,9 @@ int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtI validateSqlNode(p, pQueryInfo, &buf); } + taosArrayDestroy(data.pTableMeta); + taosArrayDestroy(req.pUdf); + taosArrayDestroy(req.pTableName); + return code; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index bc4eae7ae9..ece3d5f5eb 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -249,7 +249,7 @@ static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlo } } -void destroyDataBlock(STableDataBlocks* pDataBlock) { +static void destroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; } @@ -273,12 +273,29 @@ void destroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { - destroyDataBlock(taosArrayGetP(pDataBlockList, i)); + void* p = taosArrayGetP(pDataBlockList, i); + destroyDataBlock(p); } taosArrayDestroy(pDataBlockList); } +void destroyBlockHashmap(SHashObj* pDataBlockHash) { + if (pDataBlockHash == NULL) { + return; + } + + void** p1 = taosHashIterate(pDataBlockHash, NULL); + while (p1) { + STableDataBlocks* pBlocks = *p1; + destroyDataBlock(pBlocks); + + p1 = taosHashIterate(pDataBlockHash, p1); + } + + taosHashCleanup(pDataBlockHash); +} + // data block is disordered, sort it in ascending order void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; @@ -490,7 +507,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t } // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index a031199992..cb7df824b0 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -525,10 +525,28 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { tdDestroyKVRowBuilder(&pCxt->tagsBuilder); } +static void destroyDataBlock(STableDataBlocks* pDataBlock) { + if (pDataBlock == NULL) { + return; + } + + tfree(pDataBlock->pData); + if (!pDataBlock->cloned) { + // free the refcount for metermeta + if (pDataBlock->pTableMeta != NULL) { + tfree(pDataBlock->pTableMeta); + } + + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + } + tfree(pDataBlock); +} + static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); - taosHashCleanup(pCxt->pTableBlockHashObj); + + destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pTableDataBlocks); destroyBlockArrayList(pCxt->pVgDataBlocks); } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 4271aae451..c98f787f0b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -248,10 +248,15 @@ void qDestroyQuery(SQueryNode* pQueryNode) { if (NULL == pQueryNode) { return; } - if (nodeType(pQueryNode) == TSDB_SQL_INSERT || nodeType(pQueryNode) == TSDB_SQL_CREATE_TABLE) { + + int32_t type = nodeType(pQueryNode); + if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); - } - tfree(pQueryNode); + tfree(pQueryNode); + } else if (type == TSDB_SQL_SELECT) { + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*) pQueryNode; + destroyQueryInfo(pQueryStmtInfo); + } } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index b8545b7486..a87e138ed0 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -732,18 +732,8 @@ void cleanupFieldInfo(SFieldInfo* pFieldInfo) { return; } - if (pFieldInfo->internalField != NULL) { - size_t num = taosArrayGetSize(pFieldInfo->internalField); - for (int32_t i = 0; i < num; ++i) { -// SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i); -// if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) { -// sqlExprDestroy(pfield->pExpr); -// } - } - } - taosArrayDestroy(pFieldInfo->internalField); -// tfree(pFieldInfo->final); + tfree(pFieldInfo->final); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 1aa8836cb2..9a2ca2da98 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -191,10 +191,12 @@ void destroyExprInfo(SExprInfo* pExprInfo) { for(int32_t i = 0; i < pExprInfo->base.numOfParams; ++i) { taosVariantDestroy(&pExprInfo->base.param[i]); } + + tfree(pExprInfo->base.pColumns); tfree(pExprInfo); } -static void dropOneLevelExprInfo(SArray* pExprInfo) { +void dropOneLevelExprInfo(SArray* pExprInfo) { size_t size = taosArrayGetSize(pExprInfo); for (int32_t i = 0; i < size; ++i) { @@ -239,6 +241,9 @@ void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { #endif dst->pExpr = exprdup(src->pExpr); + dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); + memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); + memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); for (int32_t j = 0; j < src->base.numOfParams; ++j) { taosVariantAssign(&dst->base.param[j], &src->base.param[j]); diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index e817b764d5..620dd38360 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -265,7 +265,6 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* } else { // here we can push down the projection to tablescan operator. pNode->numOfExpr = num; - pNode->pExpr = taosArrayInit(num, POINTER_BYTES); taosArrayAddAll(pNode->pExpr, p); } } @@ -357,7 +356,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { SArray* exprList = taosArrayInit(4, POINTER_BYTES); if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; -// dropAllExprInfo(exprList); exit(-1); } @@ -373,7 +371,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { // 4. add the projection query node SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, &info, exprList, tableColumnList); columnListDestroy(tableColumnList); -// dropAllExprInfo(exprList); taosArrayPush(pDownstream, &pNode); } @@ -398,7 +395,8 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { } static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - if (pQueryNode->info.type == QNODE_MODIFY) { + int32_t type = nodeType(pQueryNode); + if (type == QNODE_MODIFY) { SDataPayloadInfo* pInfo = pQueryNode->pExtInfo; size_t size = taosArrayGetSize(pInfo->payload); @@ -410,10 +408,17 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { taosArrayDestroy(pInfo->payload); } + if (type == QNODE_STREAMSCAN || type == QNODE_TABLESCAN) { + SQueryTableInfo* pQueryTableInfo = pQueryNode->pExtInfo; + tfree(pQueryTableInfo->tableName); + } + + printf("----------->Free:%p\n", pQueryNode->pExpr); + taosArrayDestroy(pQueryNode->pExpr); + tfree(pQueryNode->pExtInfo); tfree(pQueryNode->pSchema); tfree(pQueryNode->info.name); -// dropAllExprInfo(pQueryNode->pExpr); if (pQueryNode->pChildren != NULL) { int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 9288c240d0..12c0d0780b 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -155,6 +155,16 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si return node; } +static void cleanupPhyNode(SPhyNode* pPhyNode) { + if (pPhyNode == NULL) { + return; + } + + dropOneLevelExprInfo(pPhyNode->pTargets); + tfree(pPhyNode->targetSchema.pSchema); + tfree(pPhyNode); +} + static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size); @@ -445,3 +455,29 @@ void setExchangSourceNode(uint64_t templateId, SDownstreamSource *pSource, SPhyN void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { setExchangSourceNode(templateId, pSource, subplan->pNode); } + +static void destroyDataSinkNode(SDataSink* pSinkNode) { + if (pSinkNode == NULL) { + return; + } + + if (nodeType(pSinkNode) == DSINK_Dispatch) { + SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode; + tfree(pDdSink->sink.schema.pSchema); + } + + tfree(pSinkNode); +} + +void qDestroySubplan(SSubplan* pSubplan) { + if (pSubplan == NULL) { + return; + } + + taosArrayDestroy(pSubplan->pChildren); + taosArrayDestroy(pSubplan->pParents); + destroyDataSinkNode(pSubplan->pDataSink); + cleanupPhyNode(pSubplan->pNode); + + tfree(pSubplan); +} diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index cf54fdec85..15af5745b8 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -1121,8 +1121,10 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); -// printf("====Physical plan:====\n"); -// printf("%s\n", *str); + cJSON_Delete(json); + + printf("====Physical plan:====\n"); + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 2fc4c8ea3e..d546925c5f 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -18,25 +18,6 @@ static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols); -static void destroyDataSinkNode(SDataSink* pSinkNode) { - if (pSinkNode == NULL) { - return; - } - tfree(pSinkNode); -} - -void qDestroySubplan(SSubplan* pSubplan) { - if (pSubplan == NULL) { - return; - } - - taosArrayDestroy(pSubplan->pChildren); - taosArrayDestroy(pSubplan->pParents); - destroyDataSinkNode(pSubplan->pDataSink); - // todo destroy pNode - tfree(pSubplan); -} - void qDestroyQueryDag(struct SQueryDag* pDag) { if (pDag == NULL) { return; @@ -51,6 +32,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { SSubplan* pSubplan = taosArrayGetP(pa, j); qDestroySubplan(pSubplan); } + taosArrayDestroy(pa); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f90fdd6e11..d17bc20495 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1512,9 +1512,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { info = NULL; _return: - schedulerFreeTaskList(info); - SCH_RET(code); } From 9608965791794fcec816054832f41349ad3ed442 Mon Sep 17 00:00:00 2001 From: Shengliang Date: Mon, 24 Jan 2022 01:55:37 -0800 Subject: [PATCH 112/118] ut for udf manage --- include/common/tmsg.h | 4 +- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/test/func/func.cpp | 164 +++++---------------- 3 files changed, 42 insertions(+), 128 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a1b51596d8..9ee1aeca86 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -539,8 +539,8 @@ typedef struct { } SCreateFuncReq; typedef struct { - char name[TSDB_FUNC_NAME_LEN]; - int8_t igNotExists; + char name[TSDB_FUNC_NAME_LEN]; + int8_t igNotExists; } SDropFuncReq; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 87de215168..f9dde7bc75 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -405,7 +405,7 @@ static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { pFuncInfo->codeSize = htonl(pFunc->codeSize); memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize); memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize); - pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; + pOutput += (sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize); mndReleaseFunc(pMnode, pFunc); } diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index c4e574906f..9d31ec7314 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -363,7 +363,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); - pReq->numOfFuncs = htonl(1); + pReq->numOfFuncs = htonl(numOfFuncs); strcpy(pReq->pFuncNames, "f2"); strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1"); @@ -425,41 +425,54 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); - // char* pComment = pFuncInfo->pCont; - // char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; - // char comments[TSDB_FUNC_COMMENT_LEN] = {0}; - // for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { - // comments[i] = 'm'; - // } - // char codes[TSDB_FUNC_CODE_LEN] = {0}; - // for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { - // codes[i] = 'd'; - // } - // EXPECT_STREQ(pComment, comments); - // EXPECT_STREQ(pCode, codes); + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + comments[i] = 'm'; + } + char codes[TSDB_FUNC_CODE_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + codes[i] = 'd'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); } } -} -#if 0 + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 2; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f3"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + } +} TEST_F(MndTestFunc, 04_Drop_Func) { { int32_t contLen = sizeof(SDropFuncReq); SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, ""); + strcpy(pReq->name, ""); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_FORMAT); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); } { int32_t contLen = sizeof(SDropFuncReq); SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, "u4"); + strcpy(pReq->name, "f3"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -470,100 +483,19 @@ TEST_F(MndTestFunc, 04_Drop_Func) { int32_t contLen = sizeof(SDropFuncReq); SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, "u1"); + strcpy(pReq->name, "f3"); + pReq->igNotExists = 1; SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); - CHECK_META("show functions", 4); - - test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 1); -} - -TEST_F(MndTestFunc, 05_Create_Drop_Alter_Func) { - { - int32_t contLen = sizeof(SCreateFuncReq); - - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p1"); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - { - int32_t contLen = sizeof(SCreateFuncReq); - - SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, "u2"); - strcpy(pReq->pass, "p2"); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); - CHECK_META("show functions", 4); - - test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 3); - - CheckBinary("u1", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("u2", TSDB_FUNC_LEN); - CheckBinary("normal", 10); - CheckBinary("super", 10); - CheckBinary("normal", 10); - CheckTimestamp(); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); - - { - int32_t contLen = sizeof(SAlterFuncReq); - - SAlterFuncReq* pReq = (SAlterFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p2"); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_FUNC, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); - CHECK_META("show functions", 4); - - test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 3); - - CheckBinary("u1", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("u2", TSDB_FUNC_LEN); - CheckBinary("normal", 10); - CheckBinary("super", 10); - CheckBinary("normal", 10); - CheckTimestamp(); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); - { int32_t contLen = sizeof(SDropFuncReq); SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); - strcpy(pReq->user, "u1"); + strcpy(pReq->name, "f1"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -571,37 +503,19 @@ TEST_F(MndTestFunc, 05_Create_Drop_Alter_Func) { } test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); - CHECK_META("show functions", 4); + CHECK_META("show functions", 7); test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 2); - - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("u2", TSDB_FUNC_LEN); - CheckBinary("super", 10); - CheckBinary("normal", 10); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); + EXPECT_EQ(test.GetShowRows(), 1); // restart test.Restart(); test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); - CHECK_META("show functions", 4); + CHECK_META("show functions", 7); test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 2); + EXPECT_EQ(test.GetShowRows(), 1); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("u2", TSDB_FUNC_LEN); - CheckBinary("super", 10); - CheckBinary("normal", 10); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_FUNC_LEN); - CheckBinary("root", TSDB_FUNC_LEN); + CheckBinary("f2", TSDB_FUNC_NAME_LEN); } - -#endif \ No newline at end of file From 07cb902a965326a33abf90ab7f8e4c717ed6f0f4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jan 2022 18:16:57 +0800 Subject: [PATCH 113/118] handle server/client execpet --- source/libs/transport/inc/transComm.h | 6 +- source/libs/transport/src/transCli.c | 85 ++++++++++++++++----------- source/libs/transport/src/transSrv.c | 36 ++++++------ 3 files changed, 73 insertions(+), 54 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index c760acd52e..f0b54a82e7 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -123,9 +123,9 @@ typedef struct { } SRpcReqContext; typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app + SRpcInfo* pTransInst; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app // struct SRpcConn* pConn; // pConn allocated tmsg_t msgType; // message type uint8_t* pCont; // content provided by app diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bfadfe56ef..e5b80d8419 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -30,6 +30,7 @@ typedef struct SCliConn { char spi; char secured; uint64_t expireTime; + int8_t notifyCount; // timers already notify to client } SCliConn; typedef struct SCliMsg { @@ -72,8 +73,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// process data read from server, auth/decompress etc later -static void clientHandleResp(SCliConn* conn); // check whether already read complete packet from server static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read @@ -88,10 +87,15 @@ static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/); -static void clientMsgDestroy(SCliMsg* pMsg); +// process data read from server, auth/decompress etc later +static void clientHandleResp(SCliConn* conn); +// handle except about conn +static void clientHandleExcept(SCliConn* conn); // handle req from app static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void clientMsgDestroy(SCliMsg* pMsg); +static void destroyTransConnCtx(STransConnCtx* ctx); // thread obj static SCliThrdObj* createThrdObj(); static void destroyThrdObj(SCliThrdObj* pThrd); @@ -100,22 +104,38 @@ static void* clientThread(void* arg); static void clientHandleResp(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = pCtx->pRpc; + SRpcInfo* pRpc = pCtx->pTransInst; SRpcMsg rpcMsg; rpcMsg.pCont = conn->readBuf.buf; rpcMsg.contLen = conn->readBuf.len; rpcMsg.ahandle = pCtx->ahandle; (pRpc->cfp)(NULL, &rpcMsg, NULL); + conn->notifyCount += 1; SCliThrdObj* pThrd = conn->hostThrd; addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + + // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } - free(pCtx->ip); - free(pCtx); - // impl + destroyTransConnCtx(pCtx); +} +static void clientHandleExcept(SCliConn* pConn) { + SCliMsg* pMsg = pConn->data; + + STransConnCtx* pCtx = pMsg->ctx; + SRpcInfo* pRpc = pCtx->pTransInst; + + SRpcMsg rpcMsg; + rpcMsg.ahandle = pCtx->ahandle; + // SRpcInfo* pRpc = pMsg->ctx->pRpc; + (pRpc->cfp)(NULL, &rpcMsg, NULL); + + pConn->notifyCount += 1; + destroyTransConnCtx(pCtx); + clientConnDestroy(pConn, true); } static void clientTimeoutCb(uv_timer_t* handle) { @@ -191,6 +211,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + conn->notifyCount = 0; // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -246,19 +267,21 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { - tDebug("alread read complete"); + tDebug("conn read complete"); clientHandleResp(conn); } else { - tDebug("read half packet, continue to read"); + tDebug("conn read half packet, continue to read"); } return; } assert(nread <= 0); if (nread == 0) { + tError("conn closed"); return; } - if (nread != UV_EOF) { - tDebug("read error %s", uv_err_name(nread)); + if (nread < 0) { + tError("conn read error: %s", uv_err_name(nread)); + clientHandleExcept(conn); } // tDebug("Read error %s\n", uv_err_name(nread)); // uv_close((uv_handle_t*)handle, clientDestroy); @@ -282,19 +305,20 @@ static void clientDestroy(uv_handle_t* handle) { static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; + if (status == 0) { - tDebug("data already was written on stream"); + tDebug("conn data already was written out"); } else { tError("failed to write: %s", uv_err_name(status)); - clientConnDestroy(pConn, true); + clientHandleExcept(pConn); return; } SCliThrdObj* pThrd = pConn->hostThrd; - if (pConn->stream == NULL) { - pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); - pConn->stream->data = pConn; - } + // if (pConn->stream == NULL) { + // pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + // uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); + // pConn->stream->data = pConn; + //} uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } @@ -316,23 +340,10 @@ static void clientWrite(SCliConn* pConn) { static void clientConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; - SCliMsg* pMsg = pConn->data; - - STransConnCtx* pCtx = pMsg->ctx; - SRpcInfo* pRpc = pCtx->pRpc; - if (status != 0) { // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); tError("failed to connect server, errmsg: %s", uv_strerror(status)); - // call user fp later - SRpcMsg rpcMsg; - rpcMsg.ahandle = pCtx->ahandle; - // SRpcInfo* pRpc = pMsg->ctx->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, NULL); - - clientConnDestroy(pConn, true); - // uv_close((uv_handle_t*)req->handle, clientDestroy); - return; + clientHandleExcept(pConn); } assert(pConn->stream == req->handle); @@ -462,6 +473,13 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { free(pThrd->loop); free(pThrd); } + +static void destroyTransConnCtx(STransConnCtx* ctx) { + if (ctx != NULL) { + free(ctx->ip); + } + free(ctx); +} // void taosCloseClient(void* arg) { // impl later @@ -472,7 +490,6 @@ void taosCloseClient(void* arg) { free(cli->pThreadObj); free(cli); } - void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]); @@ -487,7 +504,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); - pCtx->pRpc = (SRpcInfo*)shandle; + pCtx->pTransInst = (SRpcInfo*)shandle; pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b519a35f24..3b7b4d6fea 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -16,6 +16,7 @@ #ifdef USE_UV #include "transComm.h" + typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -226,7 +227,7 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { tDebug("%p timeout since no activity", conn); } -static void uvProcessData(SConn* pConn) { +static void uvHandleReq(SConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->connBuf; @@ -283,20 +284,23 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; - tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread); + tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread); if (readComplete(pBuf)) { - tDebug("alread read complete packet"); - uvProcessData(conn); + tDebug("conn %p alread read complete packet", conn); + uvHandleReq(conn); } else { - tDebug("read half packet, continue to read"); + tDebug("conn %p read partial packet, continue to read", conn); } return; } if (nread == 0) { + tDebug("conn %p except read", conn); + // destroyConn(conn, true); return; } if (nread != UV_EOF) { - tDebug("read error %s", uv_err_name(nread)); + tDebug("conn %p read error: %s", conn, uv_err_name(nread)); + destroyConn(conn, true); } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -306,7 +310,8 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b void uvOnTimeoutCb(uv_timer_t* handle) { // opt - tDebug("time out"); + SConn* pConn = handle->data; + tDebug("conn %p time out", pConn); } void uvOnWriteCb(uv_write_t* req, int status) { @@ -317,9 +322,9 @@ void uvOnWriteCb(uv_write_t* req, int status) { memset(buf->buf, 0, buf->cap); buf->left = -1; if (status == 0) { - tDebug("data already was written on stream"); + tDebug("conn %p data already was written on stream", conn); } else { - tDebug("failed to write data, %s", uv_err_name(status)); + tDebug("conn %p failed to write data, %s", conn, uv_err_name(status)); destroyConn(conn, true); } // opt @@ -334,7 +339,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { // impl later; - tDebug("prepare to send back"); + tDebug("conn %p prepare to send resp", conn); SRpcMsg* pMsg = &conn->sendMsg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); @@ -448,7 +453,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); - tDebug("new connection created: %d", fd); + tDebug("conn %p created, fd: %d", pConn, fd); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { tDebug("failed to create new connection"); @@ -517,17 +522,13 @@ static SConn* createConn() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); return pConn; } -static void connCloseCb(uv_handle_t* handle) { - // impl later - // -} + static void destroyConn(SConn* conn, bool clear) { if (conn == NULL) { return; } if (clear) { - uv_handle_t handle = *((uv_handle_t*)conn->pTcp); - uv_close(&handle, NULL); + uv_close((uv_handle_t*)conn->pTcp, NULL); } uv_timer_stop(conn->pTimer); free(conn->pTimer); @@ -646,6 +647,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) { pthread_mutex_lock(&pThrd->connMtx); QUEUE_PUSH(&pThrd->conn, &pConn->queue); pthread_mutex_unlock(&pThrd->connMtx); + tDebug("conn %p start to send resp", pConn); uv_async_send(pConn->pWorkerAsync); } From d410ef1ca2c1aece74c284b2c2fe03a0de307ba1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 24 Jan 2022 18:21:25 +0800 Subject: [PATCH 114/118] fix plan convert error --- include/common/tmsg.h | 4 +- source/client/src/clientImpl.c | 29 +++- source/client/test/clientTests.cpp | 176 +++++++++++---------- source/dnode/mgmt/impl/src/dndTransport.c | 2 + source/dnode/mnode/impl/inc/mndDef.h | 21 ++- source/dnode/mnode/impl/src/mndConsumer.c | 76 ++------- source/dnode/mnode/impl/src/mndSubscribe.c | 82 ++++++---- source/dnode/mnode/impl/src/mndTopic.c | 17 +- source/libs/planner/src/physicalPlanJson.c | 8 +- source/libs/scheduler/src/scheduler.c | 2 +- 10 files changed, 215 insertions(+), 202 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7630c5f5e5..614ce14a6b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1535,7 +1535,7 @@ static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* tlen += taosEncodeFixedU64(buf, pMsg->queryId); tlen += taosEncodeFixedU64(buf, pMsg->taskId); tlen += taosEncodeFixedU32(buf, pMsg->contentLen); - tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); + //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } @@ -1544,7 +1544,7 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { buf = taosDecodeFixedU64(buf, &pMsg->queryId); buf = taosDecodeFixedU64(buf, &pMsg->taskId); buf = taosDecodeFixedU32(buf, &pMsg->contentLen); - buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); + //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); return buf; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f5d7f169de..f10275ff97 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -308,10 +308,10 @@ tmq_conf_t* tmq_conf_new() { } int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id")) { + if (strcmp(key, "group.id") == 0) { strcpy(conf->groupId, value); } - if (strcmp(key, "client.id")) { + if (strcmp(key, "client.id") == 0) { strcpy(conf->clientId, value); } return 0; @@ -359,7 +359,7 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* ptr, char* src) { if (ptr->cnt >= ptr->tot-1) return -1; - ptr->elems[ptr->cnt] = src; + ptr->elems[ptr->cnt] = strdup(src); ptr->cnt++; return 0; } @@ -371,8 +371,23 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { int32_t sz = topic_list->cnt; tmq->clientTopics = taosArrayInit(sz, sizeof(void*)); for (int i = 0; i < sz; i++) { - char* topicName = strdup(topic_list->elems[i]); - taosArrayPush(tmq->clientTopics, &topicName); + char* topicName = topic_list->elems[i]; + + SName name = {0}; + char* dbName = getConnectionDB(tmq->pTscObj); + tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + if (topicFname == NULL) { + + } + tNameExtractFullName(&name, topicFname); + tscDebug("subscribe topic: %s", topicFname); + taosArrayPush(tmq->clientTopics, &topicFname); + /*SMqClientTopic topic = {*/ + /*.*/ + /*};*/ } SCMSubscribeReq req; req.topicNum = taosArrayGetSize(tmq->clientTopics); @@ -396,7 +411,7 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { } pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - pRequest->type = TDMT_MND_CREATE_TOPIC; + pRequest->type = TDMT_MND_SUBSCRIBE; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -529,7 +544,7 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, SCMCreateTopicReq req = { .name = (char*) topicFname, - .igExists = 0, + .igExists = 1, .physicalPlan = (char*) pStr, .sql = (char*) sql, .logicalPlan = "no logic plan", diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index a2a26cf6dc..cbe6b90757 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -297,22 +297,22 @@ TEST(testCase, driverInit_Test) { //} // //TEST(testCase, create_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); -// ASSERT_EQ(taos_errno(pRes), 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); -// ASSERT_NE(taos_errno(pRes), 0); -// -// taos_free_result(pRes); -// taos_close(pConn); + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); + + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + //ASSERT_EQ(taos_errno(pRes), 0); + + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + //ASSERT_NE(taos_errno(pRes), 0); + + //taos_free_result(pRes); + //taos_close(pConn); //} // //TEST(testCase, create_ctable_Test) { @@ -453,37 +453,37 @@ TEST(testCase, driverInit_Test) { // taos_close(pConn); //} -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +//TEST(testCase, show_table_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - } + //TAOS_RES* pRes = taos_query(pConn, "show tables"); + //if (taos_errno(pRes) != 0) { + //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //} - pRes = taos_query(pConn, "show abc1.tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - } + //pRes = taos_query(pConn, "show abc1.tables"); + //if (taos_errno(pRes) != 0) { + //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //} - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); + //TAOS_ROW pRow = NULL; + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); - int32_t count = 0; - char str[512] = {0}; + //int32_t count = 0; + //char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%d: %s\n", ++count, str); - } + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%d: %s\n", ++count, str); + //} - taos_free_result(pRes); - taos_close(pConn); -} + //taos_free_result(pRes); + //taos_close(pConn); +//} //TEST(testCase, drop_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -546,13 +546,23 @@ TEST(testCase, create_topic_Test) { char* sql = "select * from tu"; pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + if (taos_errno(pRes) != 0) { + printf("failed to create topic, reason:%s\n", taos_errstr(pRes)); + ASSERT_TRUE(0); + } + taos_free_result(pRes); taos_close(pConn); } TEST(testCase, tmq_subscribe_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg1"); @@ -593,56 +603,56 @@ TEST(testCase, tmq_commit_TEST) { // taos_close(pConn); //} -TEST(testCase, projection_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); +//TEST(testCase, projection_query_tables) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //taos_free_result(pRes); - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - for(int32_t i = 0; i < 100000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } + //for(int32_t i = 0; i < 100000; ++i) { + //char sql[512] = {0}; + //sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + //TAOS_RES* p = taos_query(pConn, sql); + //if (taos_errno(p) != 0) { + //printf("failed to insert data, reason:%s\n", taos_errstr(p)); + //} - taos_free_result(p); - } + //taos_free_result(p); + //} - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } + //pRes = taos_query(pConn, "select * from tu"); + //if (taos_errno(pRes) != 0) { + //printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); + //TAOS_ROW pRow = NULL; + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} - taos_free_result(pRes); - taos_close(pConn); -} + //taos_free_result(pRes); + //taos_close(pConn); +//} //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 104e702afb..84f1574b24 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -112,6 +112,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg; // Requests handled by VNODE pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 78f371133c..803dd453a1 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -363,8 +363,9 @@ typedef struct SMqConsumerEp { int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; - int32_t execLen; - SSubQueryMsg qExec; + uint32_t qmsgLen; + char* qmsg; + //SSubQueryMsg qExec; } SMqConsumerEp; static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { @@ -373,7 +374,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon tlen += taosEncodeFixedI32(buf, pConsumerEp->status); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); - tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); + //tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); + tlen += taosEncodeFixedU32(buf, pConsumerEp->qmsgLen); + tlen += taosEncodeBinary(buf, pConsumerEp->qmsg, pConsumerEp->qmsgLen); return tlen; } @@ -382,8 +385,9 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu buf = taosDecodeFixedI32(buf, &pConsumerEp->status); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); - pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen; + //buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); + buf = taosDecodeFixedU32(buf, &pConsumerEp->qmsgLen); + buf = taosDecodeBinary(buf, (void**)&pConsumerEp->qmsg, pConsumerEp->qmsgLen); return buf; } @@ -402,11 +406,12 @@ typedef struct SMqSubscribeObj { static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { SMqSubscribeObj* pSub = malloc(sizeof(SMqSubscribeObj)); - pSub->key[0] = 0; - pSub->epoch = 0; if (pSub == NULL) { return NULL; } + pSub->key[0] = 0; + pSub->epoch = 0; + pSub->availConsumer = taosArrayInit(0, sizeof(int64_t)); if (pSub->availConsumer == NULL) { free(pSub); @@ -433,7 +438,7 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { free(pSub); return NULL; } - return NULL; + return pSub; } static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5cdd8e77bd..7591caebc5 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -56,7 +56,9 @@ void mndCleanupConsumer(SMnode *pMnode) {} SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); - SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, tlen); + int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; void* buf = malloc(tlen); @@ -68,34 +70,6 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, buf, tlen, CM_ENCODE_OVER); - -#if 0 - int32_t topicNum = taosArrayGetSize(pConsumer->topics); - SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER); - int32_t len = strlen(pConsumer->cgroup); - SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER); - SDB_SET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CM_ENCODE_OVER); - SDB_SET_INT32(pRaw, dataPos, topicNum, CM_ENCODE_OVER); - for (int i = 0; i < topicNum; i++) { - int32_t len; - SMqConsumerTopic *pConsumerTopic = taosArrayGet(pConsumer->topics, i); - len = strlen(pConsumerTopic->name); - SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER); - SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len, CM_ENCODE_OVER); - int vgSize; - if (pConsumerTopic->vgroups == NULL) { - vgSize = 0; - } else { - vgSize = listNEles(pConsumerTopic->vgroups); - } - SDB_SET_INT32(pRaw, dataPos, vgSize, CM_ENCODE_OVER); - for (int j = 0; j < vgSize; j++) { - // SList* head; - /*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/ - } - } -#endif - SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER); @@ -116,53 +90,35 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CONSUME_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; if (sver != MND_CONSUMER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto CONSUME_DECODE_OVER; + goto CM_DECODE_OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj)); - if (pRow == NULL) goto CONSUME_DECODE_OVER; + if (pRow == NULL) goto CM_DECODE_OVER; SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); - if (pConsumer == NULL) goto CONSUME_DECODE_OVER; + if (pConsumer == NULL) goto CM_DECODE_OVER; int32_t dataPos = 0; int32_t len; - SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); void* buf = malloc(len); - if (buf == NULL) goto CONSUME_DECODE_OVER; - - SDB_GET_BINARY(pRaw, dataPos, buf, len, CONSUME_DECODE_OVER); + if (buf == NULL) goto CM_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); - tDecodeSMqConsumerObj(buf, pConsumer); - - SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CONSUME_DECODE_OVER); + if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) { + goto CM_DECODE_OVER; + } terrno = TSDB_CODE_SUCCESS; -#if 0 - SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER); - for (int i = 0; i < topicNum; i++) { - int32_t topicLen; - SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); - if (pConsumerTopic == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - // TODO - return NULL; - } - /*pConsumerTopic->vgroups = taosArrayInit(topicNum, sizeof(SMqConsumerTopic));*/ - SDB_GET_INT32(pRaw, dataPos, &topicLen, CONSUME_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pConsumerTopic->name, topicLen, CONSUME_DECODE_OVER); - int32_t vgSize; - SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER); - } -#endif - -CONSUME_DECODE_OVER: - if (terrno != 0) { +CM_DECODE_OVER: + if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); return NULL; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 9a573cbe2c..518757aa19 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -96,25 +96,27 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); // build msg - SMqSetCVgReq req = { - .vgId = pCEp->vgId, - .oldConsumerId = -1, - .newConsumerId = consumerId, - }; - strcpy(req.cgroup, cgroup); - strcpy(req.topicName, topic); - strcpy(req.sql, pTopic->sql); - strcpy(req.logicalPlan, pTopic->logicalPlan); - strcpy(req.physicalPlan, pTopic->physicalPlan); - memcpy(&req.msg, &pCEp->qExec, pCEp->execLen); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + + SMqSetCVgReq* pReq = malloc(sizeof(SMqSetCVgReq) + pCEp->qmsgLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + strcpy(pReq->cgroup, cgroup); + strcpy(pReq->topicName, topic); + pReq->sql = strdup(pTopic->sql); + pReq->logicalPlan = strdup(pTopic->logicalPlan); + pReq->physicalPlan = strdup(pTopic->physicalPlan); + pReq->msg.contentLen = pCEp->qmsgLen; + memcpy(pReq->msg.msg, pCEp->qmsg, pCEp->qmsgLen); + int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); void *reqStr = malloc(tlen); if (reqStr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } void *abuf = reqStr; - tEncodeSMqSetCVgReq(abuf, &req); + tEncodeSMqSetCVgReq(&abuf, pReq); // persist msg STransAction action = {0}; @@ -128,6 +130,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SSdbRaw *pRaw = mndSubActionEncode(pSub); mndTransAppendRedolog(pTrans, pRaw); + free(pReq); tfree(topic); tfree(cgroup); } @@ -146,6 +149,14 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas //convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); SArray *pArray; + SArray* inner = taosArrayGet(pDag->pSubplans, 0); + SSubplan *plan = taosArrayGetP(inner, 0); + plan->execNode.inUse = 0; + strcpy(plan->execNode.epAddr[0].fqdn, "localhost"); + plan->execNode.epAddr[0].port = 6030; + plan->execNode.nodeId = 2; + plan->execNode.numOfEps = 1; + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { return -1; } @@ -157,11 +168,17 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; STaskInfo* pTaskInfo = taosArrayGet(pArray, i); tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); + mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]); CEp.vgId = pTaskInfo->addr.nodeId; + CEp.qmsg = malloc(sizeof(pTaskInfo->msg->contentLen)); + if (CEp.qmsg == NULL) { + return -1; + } + memcpy(CEp.qmsg, pTaskInfo->msg->msg, pTaskInfo->msg->contentLen); taosArrayPush(unassignedVg, &CEp); } - qDestroyQueryDag(pDag); + /*qDestroyQueryDag(pDag);*/ return 0; } @@ -178,9 +195,9 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume }; strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.topicName, pTopic->name); - strcpy(req.sql, pTopic->sql); - strcpy(req.logicalPlan, pTopic->logicalPlan); - strcpy(req.physicalPlan, pTopic->physicalPlan); + req.sql = strdup(pTopic->sql); + req.logicalPlan = strdup(pTopic->logicalPlan); + req.physicalPlan = strdup(pTopic->physicalPlan); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *reqStr = malloc(tlen); if (reqStr == NULL) { @@ -208,19 +225,18 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { + terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); - int32_t size = tlen + MND_SUBSCRIBE_RESERVE_SIZE; + int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; void *buf = malloc(tlen); - if (buf == NULL) { - goto SUB_ENCODE_OVER; - } - void *abuf = buf; + if (buf == NULL) goto SUB_ENCODE_OVER; - tEncodeSubscribeObj(&buf, pSub); + void *abuf = buf; + tEncodeSubscribeObj(&abuf, pSub); int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, tlen, SUB_ENCODE_OVER); @@ -228,6 +244,8 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SDB_SET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, SUB_ENCODE_OVER); + terrno = TSDB_CODE_SUCCESS; + SUB_ENCODE_OVER: if (terrno != 0) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); @@ -259,9 +277,9 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; + SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); void *buf = malloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; - SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -269,8 +287,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { goto SUB_DECODE_OVER; } + terrno = TSDB_CODE_SUCCESS; + SUB_DECODE_OVER: - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); // TODO free subscribeobj tfree(pRow); @@ -379,10 +399,10 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name; j++; } else if (j >= oldTopicNum) { - newTopicName = taosArrayGet(newSub, i); + newTopicName = taosArrayGetP(newSub, i); i++; } else { - newTopicName = taosArrayGet(newSub, i); + newTopicName = taosArrayGetP(newSub, i); oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name; int comp = compareLenPrefixedStr(newTopicName, oldTopicName); @@ -466,6 +486,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + char* key = mndMakeSubscribeKey(consumerGroup, newTopicName); + strcpy(pSub->key, key); // set unassigned vg mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); //TODO: disable alter @@ -486,7 +508,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } SSdbRaw *pRaw = mndSubActionEncode(pSub); - /*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/ + sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); #if 0 SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); @@ -519,8 +541,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mndTransAppendRedolog(pTrans, pTopicRaw); #endif - mndReleaseTopic(pMnode, pTopic); - mndReleaseSubscribe(pMnode, pSub); + /*mndReleaseTopic(pMnode, pTopic);*/ + /*mndReleaseSubscribe(pMnode, pSub);*/ } } // part3. persist consumerObj diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index ac66e7d88b..fa043cf7a0 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -60,7 +60,9 @@ void mndCleanupTopic(SMnode *pMnode) {} SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE; + int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; + int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; + int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); if (pRaw == NULL) goto TOPIC_ENCODE_OVER; @@ -74,12 +76,10 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); - int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; - SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->logicalPlan)+1, TOPIC_ENCODE_OVER); + SDB_SET_INT32(pRaw, dataPos, logicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER); - int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; - SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER); + SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); @@ -135,7 +135,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } - SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); pTopic->physicalPlan = calloc(len + 1, sizeof(char)); @@ -144,7 +144,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } - SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); @@ -231,6 +231,7 @@ static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) { } static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { + mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -273,7 +274,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { return 0; } else { terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; - mError("db:%s, failed to create since %s", createTopicReq.name, terrstr()); + mError("topic:%s, failed to create since already exists", createTopicReq.name); return -1; } } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index cf54fdec85..dac6f3f825 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -87,6 +87,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f static const char* jkPnodeType = "Type"; static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { + case OP_StreamScan: case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: @@ -869,6 +870,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* phyNode = (SPhyNode*)obj; switch (phyNode->info.type) { case OP_TableScan: + case OP_StreamScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return tableScanNodeFromJson(json, obj); @@ -1187,14 +1189,14 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) { if(pDag == NULL) { return NULL; } - pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "numOfSubplans")); - pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "queryId")); + pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "Number")); + pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "QueryId")); pDag->pSubplans = taosArrayInit(0, sizeof(SArray)); if (pDag->pSubplans == NULL) { free(pDag); return NULL; } - cJSON* pLevels = cJSON_GetObjectItem(pRoot, "pSubplans"); + cJSON* pLevels = cJSON_GetObjectItem(pRoot, "Subplans"); int level = cJSON_GetArraySize(pLevels); for(int i = 0; i < level; i++) { SArray* plansOneLevel = taosArrayInit(0, sizeof(void*)); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f31a27cb42..6cc08bc3ad 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1476,7 +1476,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { SSubQueryMsg *pMsg = (SSubQueryMsg*) msg; - pMsg->header.vgId = htonl(tInfo.addr.nodeId); + pMsg->header.vgId = tInfo.addr.nodeId; pMsg->sId = schMgmt.sId; pMsg->queryId = plan->id.queryId; From b34a5bdf51dc8acab94042e85ddf4254aa0cb201 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jan 2022 19:19:04 +0800 Subject: [PATCH 115/118] add more trace info and handle conn except --- source/libs/transport/src/transCli.c | 19 +++++++++++-------- source/libs/transport/src/transSrv.c | 7 ++++++- source/libs/transport/test/rclient.c | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e5b80d8419..67afd46b1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -267,20 +267,20 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { - tDebug("conn read complete"); + tDebug("conn %p read complete", conn); clientHandleResp(conn); } else { - tDebug("conn read half packet, continue to read"); + tDebug("conn %p read partial packet, continue to read", conn); } return; } assert(nread <= 0); if (nread == 0) { - tError("conn closed"); + tError("conn %p closed", conn); return; } if (nread < 0) { - tError("conn read error: %s", uv_err_name(nread)); + tError("conn %p read error: %s", conn, uv_err_name(nread)); clientHandleExcept(conn); } // tDebug("Read error %s\n", uv_err_name(nread)); @@ -307,9 +307,9 @@ static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; if (status == 0) { - tDebug("conn data already was written out"); + tDebug("conn %p data already was written out", pConn); } else { - tError("failed to write: %s", uv_err_name(status)); + tError("conn %p failed to write: %s", pConn, uv_err_name(status)); clientHandleExcept(pConn); return; } @@ -334,7 +334,7 @@ static void clientWrite(SCliConn* pConn) { pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen); + tDebug("conn %p data write out, msgType : %d, len: %d", pConn, pHead->msgType, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { @@ -342,9 +342,11 @@ static void clientConnCb(uv_connect_t* req, int status) { SCliConn* pConn = req->data; if (status != 0) { // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); - tError("failed to connect server, errmsg: %s", uv_strerror(status)); + tError("conn %p failed to connect server: %s", pConn, uv_strerror(status)); clientHandleExcept(pConn); + return; } + tDebug("conn %p create", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -360,6 +362,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later + tDebug("conn %p get from conn pool", conn); conn->data = pMsg; conn->writeReq->data = conn; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 3b7b4d6fea..34e621f0f3 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -27,7 +27,6 @@ typedef struct SConn { int ref; int persist; // persist connection or not SConnBuffer connBuf; // read buf, - int count; int inType; void* pTransInst; // rpc init void* ahandle; // @@ -272,6 +271,7 @@ static void uvHandleReq(SConn* pConn) { rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; + pConn->ref++; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -432,6 +432,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(pending == UV_TCP); SConn* pConn = createConn(); + pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); @@ -520,6 +521,7 @@ void* workerThread(void* arg) { static SConn* createConn() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); + ++pConn->ref; return pConn; } @@ -527,6 +529,9 @@ static void destroyConn(SConn* conn, bool clear) { if (conn == NULL) { return; } + if (--conn->ref == 0) { + return; + } if (clear) { uv_close((uv_handle_t*)conn->pTcp, NULL); } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index fd3496cc17..84814f39fc 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -63,7 +63,7 @@ static void *sendRequest(void *param) { if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); - tDebug("recv response"); + tDebug("recv response succefully"); // usleep(100000000); } From fd90848ec31c914bb53d0bc43a09c17f2fe618e6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 24 Jan 2022 21:39:00 +0800 Subject: [PATCH 116/118] add mq msg type --- include/common/tmsg.h | 37 +++++++++++++--------- include/libs/executor/executor.h | 2 +- source/dnode/mgmt/impl/src/dndTransport.c | 5 ++- source/dnode/mnode/impl/src/mndSubscribe.c | 33 +++++++++++-------- source/dnode/vnode/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 13 ++++---- source/dnode/vnode/src/vnd/vnodeWrite.c | 3 +- source/libs/executor/src/executor.c | 6 ++-- 8 files changed, 60 insertions(+), 41 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 614ce14a6b..a80a77f0c4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1518,15 +1518,17 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { } typedef struct SMqSetCVgReq { - int32_t vgId; - int64_t oldConsumerId; - int64_t newConsumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CONSUMER_GROUP_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - SSubQueryMsg msg; + int32_t vgId; + int64_t oldConsumerId; + int64_t newConsumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + uint32_t qmsgLen; + void* qmsg; + //SSubQueryMsg msg; } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { @@ -1558,7 +1560,9 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); + tlen += taosEncodeFixedU32(buf, pReq->qmsgLen); + tlen += taosEncodeBinary(buf, pReq->qmsg, pReq->qmsgLen); + //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } @@ -1571,15 +1575,18 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = tDecodeSSubQueryMsg(buf, &pReq->msg); + buf = taosDecodeFixedU32(buf, &pReq->qmsgLen); + buf = taosDecodeBinary(buf, &pReq->qmsg, pReq->qmsgLen); + //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } typedef struct SMqSetCVgRsp { - int32_t vgId; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_CONSUMER_GROUP_LEN]; + SMsgHead header; + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; typedef struct SMqColData { diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 457245e9a3..020f9f7ccd 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -32,7 +32,7 @@ struct SSubplan; * @param streamReadHandle * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* streamReadHandle); +qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 84f1574b24..a2ca86438b 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -113,7 +113,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg; + /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; // Requests handled by VNODE pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; @@ -143,6 +144,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 518757aa19..78e9a7c17c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -55,7 +55,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); return sdbSetTable(pMnode->pSdb, table); } @@ -107,8 +107,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { pReq->sql = strdup(pTopic->sql); pReq->logicalPlan = strdup(pTopic->logicalPlan); pReq->physicalPlan = strdup(pTopic->physicalPlan); - pReq->msg.contentLen = pCEp->qmsgLen; - memcpy(pReq->msg.msg, pCEp->qmsg, pCEp->qmsgLen); + pReq->qmsgLen = pCEp->qmsgLen; + memcpy(pReq->qmsg, pCEp->qmsg, pCEp->qmsgLen); int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); void *reqStr = malloc(tlen); if (reqStr == NULL) { @@ -168,9 +168,10 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; STaskInfo* pTaskInfo = taosArrayGet(pArray, i); tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); - mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]); + /*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/ CEp.vgId = pTaskInfo->addr.nodeId; - CEp.qmsg = malloc(sizeof(pTaskInfo->msg->contentLen)); + CEp.qmsgLen = pTaskInfo->msg->contentLen; + CEp.qmsg = malloc(CEp.qmsgLen); if (CEp.qmsg == NULL) { return -1; } @@ -195,27 +196,33 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume }; strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.topicName, pTopic->name); - req.sql = strdup(pTopic->sql); - req.logicalPlan = strdup(pTopic->logicalPlan); - req.physicalPlan = strdup(pTopic->physicalPlan); + req.sql = pTopic->sql; + req.logicalPlan = pTopic->logicalPlan; + req.physicalPlan = pTopic->physicalPlan; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *reqStr = malloc(tlen); - if (reqStr == NULL) { + void *buf = malloc(sizeof(SMsgHead) + tlen); + if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - void *abuf = reqStr; + + SMsgHead* pMsgHead = (SMsgHead*)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); + pMsgHead->vgId = htonl(vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncodeSMqSetCVgReq(&abuf, &req); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = reqStr; + action.pCont = buf; action.contLen = tlen; action.msgType = TDMT_VND_MQ_SET_CONN; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(reqStr); + free(buf); return -1; } } diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 0f318dea0b..9e33c04022 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -319,7 +319,7 @@ int tqSendLaunchQuery(STqMsgItem*, int64_t offset); #endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); -int32_t tqProcessSetConnReq(STQ* pTq, char* msg); +int32_t tqProcessSetConnReq(STQ* pTq, char* msg, SRpcMsg** ppRsp); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 49bbb77797..1034a05443 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -779,7 +779,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { return 0; } -int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { +int32_t tqProcessSetConnReq(STQ* pTq, char* msg, SRpcMsg** ppRsp) { SMqSetCVgReq req; tDecodeSMqSetCVgReq(msg, &req); STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); @@ -795,9 +795,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { return -1; } strcpy(pTopic->topicName, req.topicName); - strcpy(pTopic->sql, req.sql); - strcpy(pTopic->logicalPlan, req.logicalPlan); - strcpy(pTopic->physicalPlan, req.physicalPlan); + pTopic->sql = strdup(req.sql); + pTopic->logicalPlan = strdup(req.logicalPlan); + pTopic->physicalPlan = strdup(req.physicalPlan); pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; @@ -807,9 +807,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&req.msg, pReadHandle); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&req.qmsg, pReadHandle); } taosArrayPush(pConsumer->topics, pTopic); + terrno = TSDB_CODE_SUCCESS; return 0; } @@ -822,7 +823,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->pMsg = NULL; pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; - return NULL; + return pReadHandle; } void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ccddfd56d8..211bb9cc4b 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -112,7 +112,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_SET_CONN: { - if (tqProcessSetConnReq(pVnode->pTq, ptr) < 0) { + if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead)), NULL) < 0) { + // TODO: handle error } } break; default: diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2f1f40813c..b7be85dc34 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -60,8 +60,8 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { return code; } -qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle) { - if (pMsg == NULL || streamReadHandle == NULL) { +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { + if (msg == NULL || streamReadHandle == NULL) { return NULL; } @@ -74,7 +74,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle #endif struct SSubplan* plan = NULL; - int32_t code = qStringToSubplan(pMsg->msg, &plan); + int32_t code = qStringToSubplan(msg, &plan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; From f3aa81399c5d3bde8fabe3088a51404ec9bd2d90 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 24 Jan 2022 21:46:37 +0800 Subject: [PATCH 117/118] merge from 3.0 --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 471b9b68cb..179e46527f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -380,7 +380,7 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { char* topicName = topic_list->elems[i]; SName name = {0}; - char* dbName = getConnectionDB(tmq->pTscObj); + char* dbName = getDbOfConnection(tmq->pTscObj); tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); tNameFromString(&name, topicName, T_NAME_TABLE); From 2fd97f4c7dad710997565bca3a86ea999d8f83f5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jan 2022 23:42:41 +0800 Subject: [PATCH 118/118] handle close except --- source/libs/transport/inc/transComm.h | 3 ++- source/libs/transport/src/transCli.c | 23 ++++++++++++++++++++--- source/libs/transport/src/transComm.c | 7 +++++++ source/libs/transport/src/transSrv.c | 4 ++++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index f0b54a82e7..c7a23a2482 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -182,7 +182,7 @@ typedef struct { #define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) -#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead) + sizeof(STransDigestMsg)) +#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead)) #define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) #define transContFromHead(msg) (msg + sizeof(STransMsgHead)) #define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead)) @@ -201,6 +201,7 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); void transConnCtxDestroy(STransConnCtx* ctx); +void transFreeMsg(void* msg); typedef struct SConnBuffer { char* buf; int len; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 67afd46b1c..4d384814ec 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -107,8 +107,17 @@ static void clientHandleResp(SCliConn* conn) { SRpcInfo* pRpc = pCtx->pTransInst; SRpcMsg rpcMsg; - rpcMsg.pCont = conn->readBuf.buf; - rpcMsg.contLen = conn->readBuf.len; + STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); + pHead->code = htonl(pHead->code); + pHead->msgLen = htonl(pHead->msgLen); + + rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); + rpcMsg.pCont = transContFromHead(pHead); + rpcMsg.code = pHead->code; + rpcMsg.msgType = pHead->msgType; + + // rpcMsg.pCont = conn->readBuf.buf; + // rpcMsg.contLen = conn->readBuf.len; rpcMsg.ahandle = pCtx->ahandle; (pRpc->cfp)(NULL, &rpcMsg, NULL); conn->notifyCount += 1; @@ -128,8 +137,12 @@ static void clientHandleExcept(SCliConn* pConn) { STransConnCtx* pCtx = pMsg->ctx; SRpcInfo* pRpc = pCtx->pTransInst; - SRpcMsg rpcMsg; + transFreeMsg((pMsg->msg.pCont)); + pMsg->msg.pCont = NULL; + + SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; + rpcMsg.code = -1; // SRpcInfo* pRpc = pMsg->ctx->pRpc; (pRpc->cfp)(NULL, &rpcMsg, NULL); @@ -306,6 +319,10 @@ static void clientDestroy(uv_handle_t* handle) { static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; + SCliMsg* pMsg = pConn->data; + transFreeMsg((pMsg->msg.pCont)); + pMsg->msg.pCont = NULL; + if (status == 0) { tDebug("conn %p data already was written out", pConn); } else { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 617abeea39..5bece11bec 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -191,4 +191,11 @@ void transConnCtxDestroy(STransConnCtx* ctx) { free(ctx->ip); free(ctx); } + +void transFreeMsg(void* msg) { + if (msg == NULL) { + return; + } + free((char*)msg - sizeof(STransMsgHead)); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 34e621f0f3..c70b1a5b28 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -321,6 +321,10 @@ void uvOnWriteCb(uv_write_t* req, int status) { buf->len = 0; memset(buf->buf, 0, buf->cap); buf->left = -1; + + SRpcMsg* pMsg = &conn->sendMsg; + transFreeMsg(pMsg->pCont); + if (status == 0) { tDebug("conn %p data already was written on stream", conn); } else {