From 49be2ab41d2c2d5f194783f56f394e8ac5f74c1e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 11:43:58 +0000 Subject: [PATCH 01/63] more tkv --- source/libs/tkv/src/tDiskMgr.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index d759171f85..c8ce3c6c2a 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -25,12 +25,35 @@ struct SDiskMgr { #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - // TODO + SDiskMgr *pDiskMgr; + + pDiskMgr = malloc(sizeof(*pDiskMgr)); + if (pDiskMgr == NULL) { + return -1; + } + + pDiskMgr->fname = strdup(fname); + if (pDiskMgr->fname == NULL) { + free(pDiskMgr); + return -1; + } + pDiskMgr->pgsize = pgsize; + pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); + if (pDiskMgr->fd < 0) { + free(pDiskMgr->fname); + free(pDiskMgr); + return -1; + } + + *ppDiskMgr = pDiskMgr; + return 0; } int tdmClose(SDiskMgr *pDiskMgr) { - // TODO + close(pDiskMgr->fd); + free(pDiskMgr->fname); + free(pDiskMgr); return 0; } From 358b5641aa4a20f164102bb736627765f5309839 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 11:55:41 +0000 Subject: [PATCH 02/63] more tkv --- include/libs/tkv/tkv.h | 64 ---------------------------------- source/libs/tkv/inc/tDiskMgr.h | 11 +++--- source/libs/tkv/inc/tkv.h | 32 +++++++++++++++++ source/libs/tkv/src/tDiskMgr.c | 10 +++--- 4 files changed, 44 insertions(+), 73 deletions(-) delete mode 100644 include/libs/tkv/tkv.h create mode 100644 source/libs/tkv/inc/tkv.h diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h deleted file mode 100644 index 98194f090c..0000000000 --- a/include/libs/tkv/tkv.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_H_ -#define _TD_TKV_H_ - -#if 0 -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Types exported -typedef struct STkvDb STkvDb; -typedef struct STkvOpts STkvOpts; -typedef struct STkvCache STkvCache; -typedef struct STkvReadOpts STkvReadOpts; -typedef struct STkvWriteOpts STkvWriteOpts; - -// DB operations -STkvDb *tkvOpen(const STkvOpts *options, const char *path); -void tkvClose(STkvDb *db); -void tkvPut(STkvDb *db, const STkvWriteOpts *, const char *key, size_t keylen, const char *val, size_t vallen); -char * tkvGet(STkvDb *db, const STkvReadOpts *, const char *key, size_t keylen, size_t *vallen); -void tkvCommit(STkvDb *db); - -// DB options -STkvOpts *tkvOptsCreate(); -void tkvOptsDestroy(STkvOpts *); -void tkvOptionsSetCache(STkvOpts *, STkvCache *); -void tkvOptsSetCreateIfMissing(STkvOpts *, unsigned char); - -// DB cache -typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType; -STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type); -void tkvCacheDestroy(STkvCache *); - -// STkvReadOpts -STkvReadOpts *tkvReadOptsCreate(); -void tkvReadOptsDestroy(STkvReadOpts *); - -// STkvWriteOpts -STkvWriteOpts *tkvWriteOptsCreate(); -void tkvWriteOptsDestroy(STkvWriteOpts *); - -#ifdef __cplusplus -} -#endif - -#endif -#endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/inc/tDiskMgr.h index 03622284f4..65ab4d7a3d 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/inc/tDiskMgr.h @@ -26,11 +26,12 @@ extern "C" { typedef struct SDiskMgr SDiskMgr; -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(SDiskMgr *pDiskMgr); -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int32_t tdmAllocPage(SDiskMgr *pDiskMgr); +int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); +int tdmClose(SDiskMgr *pDiskMgr); +int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); +int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); +int tdmFlush(SDiskMgr *pDiskMgr); +pgid_t tdmAllocPage(SDiskMgr *pDiskMgr); #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tkv/inc/tkv.h new file mode 100644 index 0000000000..3a08906033 --- /dev/null +++ b/source/libs/tkv/inc/tkv.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_H_ +#define _TD_TKV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// SKey +typedef struct SKey { + void *bdata; +} SKey, SValue; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index c8ce3c6c2a..5774555ae0 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -16,10 +16,10 @@ #include "tDiskMgr.h" struct SDiskMgr { - const char *fname; - uint16_t pgsize; - FileFd fd; - int32_t npgid; + char * fname; + uint16_t pgsize; + FileFd fd; + pgid_t npgid; }; #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) @@ -69,4 +69,6 @@ int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { return 0; } +int tdmFlush(SDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } + int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file From dca02a8566bb931178894502bbe35a0168abe102 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 12:40:15 +0000 Subject: [PATCH 03/63] more tkv --- source/libs/tkv/CMakeLists.txt | 1 + source/libs/tkv/inc/tkvBufPool.h | 39 ++++++++++++++ source/libs/tkv/inc/tkvDef.h | 3 ++ .../libs/tkv/inc/{tDiskMgr.h => tkvDiskMgr.h} | 14 ++--- source/libs/tkv/inc/{tPage.h => tkvPage.h} | 20 ++++--- source/libs/tkv/src/tDiskMgr.c | 18 +++---- source/libs/tkv/src/tkvBufPool.c | 54 +++++++++++++++++++ 7 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 source/libs/tkv/inc/tkvBufPool.h rename source/libs/tkv/inc/{tDiskMgr.h => tkvDiskMgr.h} (67%) rename source/libs/tkv/inc/{tPage.h => tkvPage.h} (71%) create mode 100644 source/libs/tkv/src/tkvBufPool.c diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index 0620e12f55..ec3259d1f2 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -8,4 +8,5 @@ target_include_directories( target_link_libraries( tkv PUBLIC os + PUBLIC util ) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvBufPool.h b/source/libs/tkv/inc/tkvBufPool.h new file mode 100644 index 0000000000..ec8d177a9a --- /dev/null +++ b/source/libs/tkv/inc/tkvBufPool.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_BUF_POOL_H_ +#define _TD_TKV_BUF_POOL_H_ + +#include "tkvPage.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STkvBufPool STkvBufPool; + +int tbpOpen(STkvBufPool **ppTkvBufPool); +int tbpClose(STkvBufPool *pTkvBufPool); +STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +int tbpDelPage(STkvBufPool *pTkvBufPool); +STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STkvBufPool *pTkvBufPool); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/inc/tkvDef.h index 6f1072abd5..a80c395364 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/inc/tkvDef.h @@ -26,6 +26,9 @@ extern "C" { typedef int32_t pgid_t; #define TKV_IVLD_PGID ((pgid_t)-1) +// framd_id_t +typedef int32_t frame_id_t; + #ifdef __cplusplus } #endif diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/inc/tkvDiskMgr.h similarity index 67% rename from source/libs/tkv/inc/tDiskMgr.h rename to source/libs/tkv/inc/tkvDiskMgr.h index 65ab4d7a3d..2ebe98ace2 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/inc/tkvDiskMgr.h @@ -24,14 +24,14 @@ extern "C" { #include "tkvDef.h" -typedef struct SDiskMgr SDiskMgr; +typedef struct STkvDiskMgr STkvDiskMgr; -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(SDiskMgr *pDiskMgr); -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(SDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(SDiskMgr *pDiskMgr); +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); +int tdmClose(STkvDiskMgr *pDiskMgr); +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); +int tdmFlush(STkvDiskMgr *pDiskMgr); +pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tPage.h b/source/libs/tkv/inc/tkvPage.h similarity index 71% rename from source/libs/tkv/inc/tPage.h rename to source/libs/tkv/inc/tkvPage.h index 0546f6184f..d596d215cd 100644 --- a/source/libs/tkv/inc/tPage.h +++ b/source/libs/tkv/inc/tkvPage.h @@ -17,22 +17,30 @@ #define _TD_TKV_PAGE_H_ #include "os.h" +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif +typedef struct STkvPage { + pgid_t pgid; + int32_t pinCount; + bool idDirty; + char* pData; +} STkvPage; + typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} SPgHdr; +} STkvPgHdr; -typedef struct { - SPgHdr chdr; - uint16_t used; // number of used slots - uint16_t loffset; // the offset of the starting location of the last slot used -} SSlottedPgHdr; +// typedef struct { +// SPgHdr chdr; +// uint16_t used; // number of used slots +// uint16_t loffset; // the offset of the starting location of the last slot used +// } SSlottedPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index 5774555ae0..fa8f6062d8 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "tDiskMgr.h" +#include "tkvDiskMgr.h" -struct SDiskMgr { +struct STkvDiskMgr { char * fname; uint16_t pgsize; FileFd fd; @@ -24,8 +24,8 @@ struct SDiskMgr { #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - SDiskMgr *pDiskMgr; +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { + STkvDiskMgr *pDiskMgr; pDiskMgr = malloc(sizeof(*pDiskMgr)); if (pDiskMgr == NULL) { @@ -50,25 +50,25 @@ int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { return 0; } -int tdmClose(SDiskMgr *pDiskMgr) { +int tdmClose(STkvDiskMgr *pDiskMgr) { close(pDiskMgr->fd); free(pDiskMgr->fname); free(pDiskMgr); return 0; } -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmFlush(SDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } +int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } -int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file +int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tkv/src/tkvBufPool.c new file mode 100644 index 0000000000..86bfa0ba3e --- /dev/null +++ b/source/libs/tkv/src/tkvBufPool.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "thash.h" +#include "tlist.h" + +#include "tkvBufPool.h" +#include "tkvDiskMgr.h" +#include "tkvPage.h" + +struct SFrameIdWrapper { + TD_SLIST_NODE(SFrameIdWrapper); + frame_id_t id; +}; + +struct STkvBufPool { + STkvPage* pages; + STkvDiskMgr* pDiskMgr; + SHashObj* pgTb; // page_id_t --> frame_id_t + TD_SLIST(SFrameIdWrapper) freeList; + pthread_mutex_t mutex; +}; + +typedef struct STkvLRUReplacer { +} STkvLRUReplacer; + +typedef struct STkvLFUReplacer { +} STkvLFUReplacer; + +typedef struct STkvCLKReplacer { +} STkvCLKReplacer; + +typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; + +typedef struct STkvReplacer { + tkv_replacer_t type; + union { + STkvLRUReplacer lruRep; + STkvLFUReplacer lfuRep; + STkvCLKReplacer clkRep; + }; +} STkvReplacer; \ No newline at end of file From 3513390fb8828d3d9cfa443303891b0699bd1b58 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 12:51:40 +0000 Subject: [PATCH 04/63] just have a try --- include/util/tarray.h | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/include/util/tarray.h b/include/util/tarray.h index f7c72add01..6d6120a49b 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -23,10 +23,23 @@ extern "C" { #include "os.h" #include "talgo.h" +#if 0 +#define TARRAY(TYPE) \ + struct { \ + int32_t tarray_size_; \ + int32_t tarray_neles_; \ + struct TYPE* td_array_data_; \ + } + +#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ +#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ +#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) +#endif + #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) -#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) -#define TARRAY_GET_START(array) ((array)->pData) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; @@ -57,7 +70,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); * @param nEles * @return */ -void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles); /** * @@ -65,7 +78,7 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); * @param pData position array list * @param numOfElems the number of removed position */ -void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); +void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems); /** * @@ -73,7 +86,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle * @param comparFn * @param fp */ -void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)); +void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); /** * add all element from the source array list into the destination @@ -242,19 +255,18 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t */ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); - /** * sort the pointer data in the array * @param pArray - * @param compar - * @param param + * @param compar + * @param param * @return */ -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param); +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_ARRAY_H*/ +#endif /*_TD_UTIL_ARRAY_H*/ From 3d1c8dcb7d1cc872c9e54a63b0f058719e21263f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 11 Jan 2022 06:55:44 +0000 Subject: [PATCH 05/63] more --- source/libs/tkv/inc/tkvDef.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/inc/tkvDef.h index a80c395364..cd418019be 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/inc/tkvDef.h @@ -29,6 +29,12 @@ typedef int32_t pgid_t; // framd_id_t typedef int32_t frame_id_t; +// pgsize_t +typedef int32_t pgsize_t; +#define TKV_MIN_PGSIZE 512 +#define TKV_MAX_PGSIZE 16384 +#define TKV_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) + #ifdef __cplusplus } #endif From e456c6391aac090f143fabe0eeaf5cd6e304024c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 11 Jan 2022 07:12:38 +0000 Subject: [PATCH 06/63] refact --- source/libs/tkv/CMakeLists.txt | 9 +++++++-- source/libs/tkv/inc/tkv.h | 13 ++++++++++--- source/libs/tkv/{ => src}/inc/tkvBufPool.h | 0 source/libs/tkv/{ => src}/inc/tkvDef.h | 0 source/libs/tkv/{ => src}/inc/tkvDiskMgr.h | 0 source/libs/tkv/{ => src}/inc/tkvPage.h | 0 6 files changed, 17 insertions(+), 5 deletions(-) rename source/libs/tkv/{ => src}/inc/tkvBufPool.h (100%) rename source/libs/tkv/{ => src}/inc/tkvDef.h (100%) rename source/libs/tkv/{ => src}/inc/tkvDiskMgr.h (100%) rename source/libs/tkv/{ => src}/inc/tkvPage.h (100%) diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index ec3259d1f2..fec3f37cd5 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -1,9 +1,14 @@ aux_source_directory(src TKV_SRC) add_library(tkv STATIC ${TKV_SRC}) +# target_include_directories( +# tkv +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +# ) target_include_directories( tkv - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc" ) target_link_libraries( tkv diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tkv/inc/tkv.h index 3a08906033..00534d2827 100644 --- a/source/libs/tkv/inc/tkv.h +++ b/source/libs/tkv/inc/tkv.h @@ -16,14 +16,21 @@ #ifndef _TD_TKV_H_ #define _TD_TKV_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif +// Forward declaration +typedef struct TDB TDB; +typedef struct TDB_ENV TDB_ENV; + // SKey -typedef struct SKey { - void *bdata; -} SKey, SValue; +typedef struct { + void * bdata; + uint32_t size; +} TDB_KEY, TDB_VALUE; #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tkvBufPool.h b/source/libs/tkv/src/inc/tkvBufPool.h similarity index 100% rename from source/libs/tkv/inc/tkvBufPool.h rename to source/libs/tkv/src/inc/tkvBufPool.h diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/src/inc/tkvDef.h similarity index 100% rename from source/libs/tkv/inc/tkvDef.h rename to source/libs/tkv/src/inc/tkvDef.h diff --git a/source/libs/tkv/inc/tkvDiskMgr.h b/source/libs/tkv/src/inc/tkvDiskMgr.h similarity index 100% rename from source/libs/tkv/inc/tkvDiskMgr.h rename to source/libs/tkv/src/inc/tkvDiskMgr.h diff --git a/source/libs/tkv/inc/tkvPage.h b/source/libs/tkv/src/inc/tkvPage.h similarity index 100% rename from source/libs/tkv/inc/tkvPage.h rename to source/libs/tkv/src/inc/tkvPage.h From 05372238d0c78b3e03850b5601872273fb0df679 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 11 Jan 2022 19:55:00 +0800 Subject: [PATCH 07/63] refactor walRepairMeta --- source/libs/wal/src/walMeta.c | 71 +++++++++++++++-------------------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index cac80c0a5f..dea178eb54 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -36,7 +36,7 @@ void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { char* limit; if (nlen == 0 || hlen < nlen) { - return false; + return NULL; } limit = haystack + hlen - nlen + 1; @@ -54,10 +54,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { ASSERT(pWal->fileInfoSet != NULL); int sz = taosArrayGetSize(pWal->fileInfoSet); ASSERT(sz > 0); +#if 0 for (int i = 0; i < sz; i++) { SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); - } +#endif + SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz-1); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); @@ -143,8 +145,6 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo fileInfo; memset(&fileInfo, -1, sizeof(SWalFileInfo)); sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); - //get lastVer - //get size taosArrayPush(pLogInfoArray, &fileInfo); } } @@ -158,54 +158,43 @@ int walCheckAndRepairMeta(SWal* pWal) { oldSz = taosArrayGetSize(pWal->fileInfoSet); } int newSz = taosArrayGetSize(pLogInfoArray); - // case 1. meta file not exist / cannot be parsed - if (oldSz < newSz) { + + if (oldSz > newSz) { + taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz); + } else if (oldSz < newSz) { for (int i = oldSz; i < newSz; i++) { SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); taosArrayPush(pWal->fileInfoSet, pFileInfo); } - - pWal->writeCur = newSz - 1; - pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; - pWal->vers.lastVer = walScanLogGetLastVer(pWal); - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; - ASSERT(pWal->vers.lastVer != -1); - - int code = walSaveMeta(pWal); - if (code < 0) { - taosArrayDestroy(pLogInfoArray); - return -1; - } } - - // case 2. versions in meta not match log - // or some log not included in meta - // (e.g. program killed) - // - // case 3. other corrupt cases - // -#if 0 - int sz = taosArrayGetSize(pLogInfoArray); - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, i); - if (i == 0 && pFileInfo->firstVer != walGetFirstVer(pWal)) { - //repair - } + taosArrayDestroy(pLogInfoArray); - if (i > 0) { - SWalFileInfo* pLastFileInfo = taosArrayGet(pLogInfoArray, i-1); - if (pLastFileInfo->lastVer != pFileInfo->firstVer) { + pWal->writeCur = newSz - 1; + if (newSz > 0) { + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); + char fnameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + struct stat statbuf; + stat(fnameStr, &statbuf); + + if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) { + pLastFileInfo->fileSize = statbuf.st_size; + pWal->vers.lastVer = walScanLogGetLastVer(pWal); + ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; + ASSERT(pWal->vers.lastVer != -1); + + int code = walSaveMeta(pWal); + if (code < 0) { + taosArrayDestroy(pLogInfoArray); + return -1; } } } -#endif - - // get last version of this file - // - // rebuild meta - taosArrayDestroy(pLogInfoArray); + //TODO: set fileSize and lastVer if necessary + return 0; } From b94500077956a634a8928d89e30d81f0b0856df7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 11 Jan 2022 23:06:36 +0800 Subject: [PATCH 08/63] add libuv --- cmake/libuv_CMakeLists.txt.in | 8 +- source/libs/transport/CMakeLists.txt | 17 +- source/libs/transport/inc/rpcCache.h | 2 + source/libs/transport/inc/rpcHead.h | 60 +- source/libs/transport/inc/rpcTcp.h | 13 +- source/libs/transport/inc/transportInt.h | 7 +- source/libs/transport/src/rpcCache.c | 53 +- source/libs/transport/src/rpcMain.c | 883 +++++++++++++---------- source/libs/transport/src/rpcTcp.c | 140 ++-- source/libs/transport/src/rpcUdp.c | 81 ++- source/libs/transport/src/transport.c | 2 +- 11 files changed, 720 insertions(+), 546 deletions(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index ed406e089e..6c7ab79ed2 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -4,9 +4,9 @@ ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git GIT_TAG v1.42.0 SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/libuv" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + ) diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 98a380dc8f..c4eeef5df2 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -12,4 +12,19 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common -) \ No newline at end of file +) +if (${BUILD_WITH_UV}) + target_include_directories( + transport + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + ) + +#LINK_DIRECTORIES("${CMAKE_SOURCE_DIR}/debug/contrib/libuv") + target_link_libraries( + transport + PUBLIC uv_a + ) + add_definitions(-DUSE_UV) +endif(${BUILD_WITH_UV}) + + diff --git a/source/libs/transport/inc/rpcCache.h b/source/libs/transport/inc/rpcCache.h index 3a996aab7c..5148f5e23c 100644 --- a/source/libs/transport/inc/rpcCache.h +++ b/source/libs/transport/inc/rpcCache.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_RPC_CACHE_H #define TDENGINE_RPC_CACHE_H +#include + #ifdef __cplusplus extern "C" { #endif diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 6e98bbd563..7317d84af1 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -16,52 +16,57 @@ #ifndef TDENGINE_RPCHEAD_H #define TDENGINE_RPCHEAD_H +#include #ifdef __cplusplus extern "C" { #endif -#define RPC_CONN_TCP 2 +#ifdef USE_UV + +#else + +#define RPC_CONN_TCP 2 extern int tsRpcOverhead; typedef struct { - void *msg; + void* msg; int msgLen; - uint32_t ip; + uint32_t ip; uint16_t port; int connType; - void *shandle; - void *thandle; - void *chandle; + void* shandle; + void* thandle; + void* chandle; } SRecvInfo; #pragma pack(push, 1) typedef struct { - char version:4; // RPC version - char comp:4; // compression algorithm, 0:no compression 1:lz4 - char resflag:2; // reserved bits - char spi:3; // security parameter index - char encrypt:3; // encrypt algorithm, 0: no encryption - uint16_t tranId; // transcation ID - uint32_t linkUid; // for unique connection ID assigned by client - uint64_t ahandle; // ahandle assigned by client - uint32_t sourceId; // source ID, an index for connection list - uint32_t destId; // destination ID, an index for connection list - uint32_t destIp; // destination IP address, for NAT scenario - char user[TSDB_UNI_LEN]; // user ID - uint16_t port; // for UDP only, port may be changed - char empty[1]; // reserved - uint16_t msgType; // message type - int32_t msgLen; // message length including the header iteslf + char version : 4; // RPC version + char comp : 4; // compression algorithm, 0:no compression 1:lz4 + char resflag : 2; // reserved bits + char spi : 3; // security parameter index + char encrypt : 3; // encrypt algorithm, 0: no encryption + uint16_t tranId; // transcation ID + uint32_t linkUid; // for unique connection ID assigned by client + uint64_t ahandle; // ahandle assigned by client + uint32_t sourceId; // source ID, an index for connection list + uint32_t destId; // destination ID, an index for connection list + uint32_t destIp; // destination IP address, for NAT scenario + char user[TSDB_UNI_LEN]; // user ID + uint16_t port; // for UDP only, port may be changed + char empty[1]; // reserved + uint16_t msgType; // message type + int32_t msgLen; // message length including the header iteslf uint32_t msgVer; - int32_t code; // code in response message - uint8_t content[0]; // message body starts from here + int32_t code; // code in response message + uint8_t content[0]; // message body starts from here } SRpcHead; typedef struct { - int32_t reserved; - int32_t contLen; + int32_t reserved; + int32_t contLen; } SRpcComp; typedef struct { @@ -70,11 +75,10 @@ typedef struct { } SRpcDigest; #pragma pack(pop) - +#endif #ifdef __cplusplus } #endif #endif // TDENGINE_RPCHEAD_H - diff --git a/source/libs/transport/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h index 6ef8fc2d92..5e5c43a1db 100644 --- a/source/libs/transport/inc/rpcTcp.h +++ b/source/libs/transport/inc/rpcTcp.h @@ -15,23 +15,28 @@ #ifndef _rpc_tcp_header_ #define _rpc_tcp_header_ +#include #ifdef __cplusplus extern "C" { #endif +#ifdef USE_UV +#else void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosStopTcpServer(void *param); -void taosCleanUpTcpServer(void *param); +void taosStopTcpServer(void *param); +void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); -void taosStopTcpClient(void *chandle); -void taosCleanUpTcpClient(void *chandle); +void taosStopTcpClient(void *chandle); +void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); void taosCloseTcpConnection(void *chandle); int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); +#endif + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 24a2e0ef89..9809f7ee1a 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -20,8 +20,13 @@ extern "C" { #endif +#ifdef USE_UV + +#else + +#endif #ifdef __cplusplus } #endif -#endif /*_TD_TRANSPORT_INT_H_*/ \ No newline at end of file +#endif /*_TD_TRANSPORT_INT_H_*/ diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 7de7aa341b..40767d2ba5 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -13,37 +13,40 @@ * along with this program. If not, see . */ +#include "rpcCache.h" #include "os.h" +#include "rpcLog.h" #include "taosdef.h" #include "tglobal.h" #include "tmempool.h" #include "ttimer.h" #include "tutil.h" -#include "rpcLog.h" -#include "rpcCache.h" +#ifdef USE_UV + +#else typedef struct SConnHash { char fqdn[TSDB_FQDN_LEN]; uint16_t port; char connType; struct SConnHash *prev; struct SConnHash *next; - void *data; + void * data; uint64_t time; } SConnHash; typedef struct { - SConnHash **connHashList; + SConnHash ** connHashList; mpool_h connHashMemPool; int maxSessions; int total; int * count; int64_t keepTimer; pthread_mutex_t mutex; - void (*cleanFp)(void *); - void *tmrCtrl; - void *pTimer; - int64_t *lockedBy; + void (*cleanFp)(void *); + void * tmrCtrl; + void * pTimer; + int64_t *lockedBy; } SConnCache; static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType); @@ -122,7 +125,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in uint64_t time = taosGetTimestampMs(); pCache = (SConnCache *)handle; - assert(pCache); + assert(pCache); assert(data); hash = rpcHashConn(pCache, fqdn, port, connType); @@ -134,7 +137,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pNode->prev = NULL; pNode->time = time; - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode->next = pCache->connHashList[hash]; if (pCache->connHashList[hash] != NULL) (pCache->connHashList[hash])->prev = pNode; @@ -143,10 +146,11 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pCache->count[hash]++; rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); pCache->total++; - // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, + // pCache->count[hash]); return; } @@ -158,12 +162,12 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy void * pData = NULL; pCache = (SConnCache *)handle; - assert(pCache); + assert(pCache); uint64_t time = taosGetTimestampMs(); hash = rpcHashConn(pCache, fqdn, port, connType); - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode = pCache->connHashList[hash]; while (pNode) { @@ -197,12 +201,14 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy pCache->count[hash]--; } - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); if (pData) { - //tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, + // pCache->count[hash]); } else { - //tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]); + // tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, + // pCache->count[hash]); } return pData; @@ -221,10 +227,10 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { uint64_t time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode = pCache->connHashList[hash]; rpcRemoveExpiredNodes(pCache, pNode, hash, time); - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); } // tTrace("timer, total connections in cache:%d", pCache->total); @@ -233,7 +239,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { } static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { - if (pNode == NULL || (time < pCache->keepTimer + pNode->time) ) return; + if (pNode == NULL || (time < pCache->keepTimer + pNode->time)) return; SConnHash *pPrev = pNode->prev, *pNext; @@ -242,7 +248,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash pNext = pNode->next; pCache->total--; pCache->count[hash]--; - //tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode, + // tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, + // pNode->connType, hash, pNode, // pCache->count[hash]); taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); pNode = pNext; @@ -257,7 +264,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) { SConnCache *pCache = (SConnCache *)handle; int hash = 0; - char *temp = fqdn; + char * temp = fqdn; while (*temp) { hash += *temp; @@ -288,4 +295,4 @@ static void rpcUnlockCache(int64_t *lockedBy) { assert(false); } } - +#endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index d58ea63c4f..9bba0dca0a 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,117 +13,265 @@ * along with this program. If not, see . */ +#include +#include "lz4.h" #include "os.h" +#include "rpcCache.h" +#include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" #include "tidpool.h" #include "tmd5.h" #include "tmempool.h" +#include "tmsg.h" +#include "tref.h" +#include "trpc.h" #include "ttimer.h" #include "tutil.h" -#include "lz4.h" -#include "tref.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tmsg.h" -#include "trpc.h" -#include "thash.h" -#include "rpcLog.h" -#include "rpcUdp.h" -#include "rpcCache.h" -#include "rpcTcp.h" -#include "rpcHead.h" -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead *) ((char*)cont - sizeof(SRpcHead))) +#ifdef USE_UV + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) + +int32_t rpcInit() { return -1; } +void rpcCleanup() { return; }; +void* rpcOpen(const SRpcInit* pRpc) { return NULL; } +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t* pipe; + uv_loop_t* loop; + uv_async_t* workerAsync; // + int fd; +} SThreadObj; + +typedef struct SServerObj { + uv_tcp_t server; + uv_loop_t* loop; + int workerIdx; + int numOfThread; + SThreadObj** pThreadObj; + uv_pipe_t** pipe; +} SServerObj; + +typedef struct SConnCtx { + uv_tcp_t* pClient; + uv_timer_t* pTimer; + uv_async_t* pWorkerAsync; + int ref; +} SConnCtx; + +void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(suggested_size); + buf->len = suggested_size; +} + +void onTimeout(uv_timer_t* handle) { + // opt + tDebug("time out"); +} +void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + tDebug("data already was read on a stream"); +} + +void onWrite(uv_write_t* req, int status) { + // opt + if (req) tDebug("data already was written on stream"); +} + +static void workerAsyncCB(uv_async_t* handle) { + // opt + SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); +} +void onAccept(uv_stream_t* stream, int status) { + if (status == -1) { + return; + } + SServerObj* pObj = container_of(stream, SServerObj, server); + tDebug("new conntion accepted by main server, dispatch to one worker thread"); + + uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, cli); + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + + uv_buf_t buf = uv_buf_init("a", 1); + // despatch to worker thread + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); + } else { + uv_close((uv_handle_t*)cli, NULL); + } +} +void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + if (nread < 0) { + if (nread != UV_EOF) { + tError("read error %s", uv_err_name(nread)); + } + // TODO(log other failure reason) + uv_close((uv_handle_t*)q, NULL); + return; + } + SThreadObj* pObj = (SThreadObj*)container_of(q, struct SThreadObj, pipe); + + uv_pipe_t* pipe = (uv_pipe_t*)q; + if (!uv_pipe_pending_count(pipe)) { + tError("No pending count"); + return; + } + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SConnCtx* pConn = malloc(sizeof(SConnCtx)); + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pObj->loop, pConn->pTimer); + + pConn->pClient = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pWorkerAsync = pObj->workerAsync; // thread safty + uv_tcp_init(pObj->loop, pConn->pClient); + + if (uv_accept(q, (uv_stream_t*)(pConn->pClient)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t*)pConn->pClient, &fd); + tDebug("new connection created: %d", fd); + uv_timer_start(pConn->pTimer, onTimeout, 10, 0); + uv_read_start((uv_stream_t*)(pConn->pClient), allocBuffer, onRead); + } else { + uv_timer_stop(pConn->pTimer); + free(pConn->pTimer); + uv_close((uv_handle_t*)pConn->pClient, NULL); + free(pConn->pClient); + free(pConn); + } +} + +void* workerThread(void* arg) { + SThreadObj* pObj = (SThreadObj*)arg; + int fd = pObj->fd; + pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pObj->loop); + + uv_pipe_init(pObj->loop, pObj->pipe, 1); + uv_pipe_open(pObj->pipe, fd); + + pObj->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); + uv_read_start((uv_stream_t*)pObj->pipe, allocBuffer, onConnection); +} +#else + +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) #define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; uint16_t localPort; int8_t connType; - int index; // for UDP server only, round robin for multiple threads + int index; // for UDP server only, round robin for multiple threads char label[TSDB_LABEL_LEN]; - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); + void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); - int32_t refCount; - void *parent; - void *idPool; // handle to ID pool - void *tmrCtrl; // handle to timer - SHashObj *hash; // handle returned by hash utility - void *tcphandle;// returned handle from TCP initialization - void *udphandle;// returned handle from UDP initialization - void *pCache; // connection cache + int32_t refCount; + void * parent; + void * idPool; // handle to ID pool + void * tmrCtrl; // handle to timer + SHashObj * hash; // handle returned by hash utility + void * tcphandle; // returned handle from TCP initialization + void * udphandle; // returned handle from UDP initialization + void * pCache; // connection cache pthread_mutex_t mutex; struct SRpcConn *connList; // connection list } SRpcInfo; typedef struct { - SRpcInfo *pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void *ahandle; // handle provided by app - struct SRpcConn *pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t *pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg *pRsp; // for synchronous API - tsem_t *pSem; // for synchronous API - SEpSet *pSet; // for synchronous API - char msg[0]; // RpcHead starts from here + SRpcInfo * pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void * ahandle; // handle provided by app + struct SRpcConn *pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t * pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg * pRsp; // for synchronous API + tsem_t * pSem; // for synchronous API + SEpSet * pSet; // for synchronous API + char msg[0]; // RpcHead starts from here } SRpcReqContext; typedef struct SRpcConn { - char info[48];// debug info: label + pConn + ahandle - int sid; // session ID - uint32_t ownId; // own link ID - uint32_t peerId; // peer link ID - char user[TSDB_UNI_LEN]; // user ID for the link - char spi; // security parameter index - char encrypt; // encryption, 0:1 - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - char secured; // if set to 1, no authentication - uint16_t localPort; // for UDP only - uint32_t linkUid; // connection unique ID assigned by client - uint32_t peerIp; // peer IP - uint16_t peerPort; // peer port - char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string - uint16_t tranId; // outgoing transcation ID, for build message - uint16_t outTranId; // outgoing transcation ID - uint16_t inTranId; // transcation ID for incoming msg - tmsg_t outType; // message type for outgoing request - tmsg_t inType; // message type for incoming request - void *chandle; // handle passed by TCP/UDP connection layer - void *ahandle; // handle provided by upper app layter - int retry; // number of retry for sending request - int tretry; // total retry - void *pTimer; // retry timer to monitor the response - void *pIdleTimer; // idle timer - char *pRspMsg; // response message including header - int rspMsgLen; // response messag length - char *pReqMsg; // request message including header - int reqMsgLen; // request message length - SRpcInfo *pRpc; // the associated SRpcInfo - int8_t connType; // connection type - int64_t lockedBy; // lock for connection - SRpcReqContext *pContext; // request context + char info[48]; // debug info: label + pConn + ahandle + int sid; // session ID + uint32_t ownId; // own link ID + uint32_t peerId; // peer link ID + char user[TSDB_UNI_LEN]; // user ID for the link + char spi; // security parameter index + char encrypt; // encryption, 0:1 + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + char secured; // if set to 1, no authentication + uint16_t localPort; // for UDP only + uint32_t linkUid; // connection unique ID assigned by client + uint32_t peerIp; // peer IP + uint16_t peerPort; // peer port + char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string + uint16_t tranId; // outgoing transcation ID, for build message + uint16_t outTranId; // outgoing transcation ID + uint16_t inTranId; // transcation ID for incoming msg + tmsg_t outType; // message type for outgoing request + tmsg_t inType; // message type for incoming request + void * chandle; // handle passed by TCP/UDP connection layer + void * ahandle; // handle provided by upper app layter + int retry; // number of retry for sending request + int tretry; // total retry + void * pTimer; // retry timer to monitor the response + void * pIdleTimer; // idle timer + char * pRspMsg; // response message including header + int rspMsgLen; // response messag length + char * pReqMsg; // request message including header + int reqMsgLen; // request message length + SRpcInfo * pRpc; // the associated SRpcInfo + int8_t connType; // connection type + int64_t lockedBy; // lock for connection + SRpcReqContext *pContext; // request context } SRpcConn; static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; @@ -137,41 +285,29 @@ int tsRpcOverhead; static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; +// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 +#define RPC_CONN_UDPS 0 +#define RPC_CONN_UDPC 1 +#define RPC_CONN_TCPS 2 +#define RPC_CONN_TCPC 3 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { - taosInitUdpConnection, - taosInitUdpConnection, - taosInitTcpServer, - taosInitTcpClient -}; + taosInitUdpConnection, taosInitUdpConnection, taosInitTcpServer, taosInitTcpClient}; -void (*taosCleanUpConn[])(void *thandle) = { - taosCleanUpUdpConnection, - taosCleanUpUdpConnection, - taosCleanUpTcpServer, - taosCleanUpTcpClient -}; +void (*taosCleanUpConn[])(void *thandle) = {taosCleanUpUdpConnection, taosCleanUpUdpConnection, taosCleanUpTcpServer, + taosCleanUpTcpClient}; void (*taosStopConn[])(void *thandle) = { - taosStopUdpConnection, - taosStopUdpConnection, + taosStopUdpConnection, + taosStopUdpConnection, taosStopTcpServer, taosStopTcpClient, }; int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { - taosSendUdpData, - taosSendUdpData, - taosSendTcpData, - taosSendTcpData -}; + taosSendUdpData, taosSendUdpData, taosSendTcpData, taosSendTcpData}; void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = { taosOpenUdpConnection, @@ -180,12 +316,7 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port taosOpenTcpClientConnection, }; -void (*taosCloseConn[])(void *chandle) = { - NULL, - NULL, - taosCloseTcpConnection, - taosCloseTcpConnection -}; +void (*taosCloseConn[])(void *chandle) = {NULL, NULL, taosCloseTcpConnection, taosCloseTcpConnection}; static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType); static void rpcCloseConn(void *thandle); @@ -194,11 +325,11 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc); static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv); static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv); -static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); -static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); -static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); -static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); -static void rpcSendReqHead(SRpcConn *pConn); +static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); +static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); +static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); +static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); +static void rpcSendReqHead(SRpcConn *pConn); static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv); static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext); @@ -207,15 +338,15 @@ static void rpcProcessRetryTimer(void *, void *); static void rpcProcessIdleTimer(void *param, void *tmrId); static void rpcProcessProgressTimer(void *param, void *tmrId); -static void rpcFreeMsg(void *msg); -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +static void rpcFreeMsg(void *msg); +static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen); static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead); -static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); -static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); -static void rpcLockConn(SRpcConn *pConn); -static void rpcUnlockConn(SRpcConn *pConn); -static void rpcAddRef(SRpcInfo *pRpc); -static void rpcDecRef(SRpcInfo *pRpc); +static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); +static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); +static void rpcLockConn(SRpcConn *pConn); +static void rpcUnlockConn(SRpcConn *pConn); +static void rpcAddRef(SRpcInfo *pRpc); +static void rpcDecRef(SRpcInfo *pRpc); static void rpcFree(void *p) { tTrace("free mem: %p", p); @@ -240,26 +371,26 @@ void rpcCleanup(void) { taosCloseRef(tsRpcRefId); tsRpcRefId = -1; } - + void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; - //pthread_once(&tsRpcInit, rpcInit); + // pthread_once(&tsRpcInit, rpcInit); pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; - if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + if (pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); pRpc->connType = pInit->connType; if (pRpc->connType == TAOS_CONN_CLIENT) { pRpc->numOfThreads = pInit->numOfThreads; } else { - pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } pRpc->idleTime = pInit->idleTime; pRpc->localPort = pInit->localPort; pRpc->afp = pInit->afp; - pRpc->sessions = pInit->sessions+1; + pRpc->sessions = pInit->sessions + 1; if (pInit->user) tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret)); if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey)); @@ -279,14 +410,14 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } - pRpc->idPool = taosInitIdPool(pRpc->sessions-1); + pRpc->idPool = taosInitIdPool(pRpc->sessions - 1); if (pRpc->idPool == NULL) { tError("%s failed to init ID pool", pRpc->label); rpcClose(pRpc); return NULL; } - pRpc->tmrCtrl = taosTmrInit(pRpc->sessions*2 + 1, 50, 10000, pRpc->label); + pRpc->tmrCtrl = taosTmrInit(pRpc->sessions * 2 + 1, 50, 10000, pRpc->label); if (pRpc->tmrCtrl == NULL) { tError("%s failed to init timers", pRpc->label); rpcClose(pRpc); @@ -301,8 +432,8 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } } else { - pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); - if ( pRpc->pCache == NULL ) { + pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); + if (pRpc->pCache == NULL) { tError("%s failed to init connection cache", pRpc->label); rpcClose(pRpc); return NULL; @@ -311,10 +442,10 @@ void *rpcOpen(const SRpcInit *pInit) { pthread_mutex_init(&pRpc->mutex, NULL); - pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, - pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); - pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, - pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); + pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, + rpcProcessMsgFromPeer, pRpc); + pRpc->udphandle = + (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) { tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort); @@ -334,7 +465,7 @@ void rpcClose(void *param) { (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); (*taosStopConn[pRpc->connType])(pRpc->udphandle); - // close all connections + // close all connections for (int i = 0; i < pRpc->sessions; ++i) { if (pRpc->connList && pRpc->connList[i].user[0]) { rpcCloseConn((void *)(pRpc->connList + i)); @@ -375,8 +506,8 @@ void *rpcReallocCont(void *ptr, int contLen) { if (ptr == NULL) return rpcMallocCont(contLen); char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); - if (contLen == 0 ) { - free(start); + if (contLen == 0) { + free(start); return NULL; } @@ -385,17 +516,17 @@ void *rpcReallocCont(void *ptr, int contLen) { if (start == NULL) { tError("failed to realloc cont, size:%d", size); return NULL; - } + } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { - SRpcInfo *pRpc = (SRpcInfo *)shandle; + SRpcInfo * pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); - pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); + pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); pContext->ahandle = pMsg->ahandle; pContext->pRpc = (SRpcInfo *)shandle; pContext->epSet = *pEpSet; @@ -404,16 +535,15 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t pContext->msgType = pMsg->msgType; pContext->oldInUse = pEpSet->inUse; - pContext->connType = RPC_CONN_UDPC; - if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp ) pContext->connType = RPC_CONN_TCPC; + pContext->connType = RPC_CONN_UDPC; + if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp) pContext->connType = RPC_CONN_TCPC; - // connection type is application specific. + // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection tmsg_t type = pMsg->msgType; - if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE - || type == TDMT_VND_FETCH || type == TDMT_MND_VGROUP_LIST - || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META - || type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) + if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE || type == TDMT_VND_FETCH || + type == TDMT_MND_VGROUP_LIST || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META || + type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -425,26 +555,26 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t void rpcSendResponse(const SRpcMsg *pRsp) { if (pRsp->handle == NULL) return; - int msgLen = 0; - SRpcConn *pConn = (SRpcConn *)pRsp->handle; - SRpcMsg rpcMsg = *pRsp; - SRpcMsg *pMsg = &rpcMsg; - SRpcInfo *pRpc = pConn->pRpc; + int msgLen = 0; + SRpcConn *pConn = (SRpcConn *)pRsp->handle; + SRpcMsg rpcMsg = *pRsp; + SRpcMsg * pMsg = &rpcMsg; + SRpcInfo *pRpc = pConn->pRpc; - if ( pMsg->pCont == NULL ) { + if (pMsg->pCont == NULL) { pMsg->pCont = rpcMallocCont(0); pMsg->contLen = 0; } - SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); - char *msg = (char *)pHead; + SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); + char * msg = (char *)pHead; pMsg->contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); msgLen = rpcMsgLenFromCont(pMsg->contLen); rpcLockConn(pConn); - if ( pConn->inType == 0 || pConn->user[0] == 0 ) { + if (pConn->inType == 0 || pConn->user[0] == 0) { tError("%s, connection is already released, rsp wont be sent", pConn->info); rpcUnlockConn(pConn); rpcFreeCont(pMsg->pCont); @@ -454,7 +584,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set msg header pHead->version = 1; - pHead->msgType = pConn->inType+1; + pHead->msgType = pConn->inType + 1; pHead->spi = pConn->spi; pHead->encrypt = pConn->encrypt; pHead->tranId = pConn->inTranId; @@ -463,13 +593,13 @@ void rpcSendResponse(const SRpcMsg *pRsp) { pHead->linkUid = pConn->linkUid; pHead->port = htons(pConn->localPort); pHead->code = htonl(pMsg->code); - pHead->ahandle = (uint64_t) pConn->ahandle; - + pHead->ahandle = (uint64_t)pConn->ahandle; + // set pConn parameters pConn->inType = 0; // response message is released until new response is sent - rpcFreeMsg(pConn->pRspMsg); + rpcFreeMsg(pConn->pRspMsg); pConn->pRspMsg = msg; pConn->rspMsgLen = msgLen; if (pMsg->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) pConn->inTranId--; @@ -482,23 +612,22 @@ void rpcSendResponse(const SRpcMsg *pRsp) { rpcSendMsgToPeer(pConn, msg, msgLen); // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured - if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) - pConn->secured = 1; // connection shall be secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; rpcUnlockConn(pConn); - rpcDecRef(pRpc); // decrease the referene count + rpcDecRef(pRpc); // decrease the referene count return; } void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - + rpcMsg.contLen = sizeof(SEpSet); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); if (rpcMsg.pCont == NULL) return; @@ -514,24 +643,24 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { } int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { - SRpcConn *pConn = (SRpcConn *)thandle; + SRpcConn *pConn = (SRpcConn *)thandle; if (pConn->user[0] == 0) return -1; pInfo->clientIp = pConn->peerIp; pInfo->clientPort = pConn->peerPort; // pInfo->serverIp = pConn->destIp; - + tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); return 0; } void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; - pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); + pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); memset(pRsp, 0, sizeof(SRpcMsg)); - - tsem_t sem; + + tsem_t sem; tsem_init(&sem, 0, 0); pContext->pSem = &sem; pContext->pRsp = pRsp; @@ -548,13 +677,13 @@ void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { // this API is used by server app to keep an APP context in case connection is broken int rpcReportProgress(void *handle, char *pCont, int contLen) { SRpcConn *pConn = (SRpcConn *)handle; - int code = 0; + int code = 0; rpcLockConn(pConn); if (pConn->user[0]) { // pReqMsg and reqMsgLen is re-used to store the context from app server - pConn->pReqMsg = pCont; + pConn->pReqMsg = pCont; pConn->reqMsgLen = contLen; } else { tDebug("%s, rpc connection is already released", pConn->info); @@ -567,7 +696,6 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { } void rpcCancelRequest(int64_t rid) { - SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid); if (pContext == NULL) return; @@ -577,7 +705,7 @@ void rpcCancelRequest(int64_t rid) { } static void rpcFreeMsg(void *msg) { - if ( msg ) { + if (msg) { char *temp = (char *)msg - sizeof(SRpcReqContext); free(temp); tTrace("free mem: %p", temp); @@ -589,14 +717,14 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { - tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_FQDN_ERROR; + tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } - pConn = rpcAllocateClientConn(pRpc); + pConn = rpcAllocateClientConn(pRpc); - if (pConn) { + if (pConn) { tstrncpy(pConn->peerFqdn, peerFqdn, sizeof(pConn->peerFqdn)); pConn->peerIp = peerIp; pConn->peerPort = peerPort; @@ -604,7 +732,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, pConn->connType = connType; if (taosOpenConn[connType]) { - void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle; + void *shandle = (connType & RPC_CONN_TCP) ? pRpc->tcphandle : pRpc->udphandle; pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort); if (pConn->chandle == NULL) { tError("failed to connect to:%s:%d", taosIpStr(pConn->peerIp), pConn->peerPort); @@ -629,13 +757,14 @@ static void rpcReleaseConn(SRpcConn *pConn) { taosTmrStopA(&pConn->pTimer); taosTmrStopA(&pConn->pIdleTimer); - if ( pRpc->connType == TAOS_CONN_SERVER) { - char hashstr[40] = {0}; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); + if (pRpc->connType == TAOS_CONN_SERVER) { + char hashstr[40] = {0}; + size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, + pConn->connType); taosHashRemove(pRpc->hash, hashstr, size); - rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg + rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; - + // if server has ever reported progress, free content if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); // do not use rpcFreeMsg } else { @@ -643,17 +772,17 @@ static void rpcReleaseConn(SRpcConn *pConn) { if (pConn->outType && pConn->pReqMsg) { SRpcReqContext *pContext = pConn->pContext; if (pContext) { - if (pContext->pRsp) { - // for synchronous API, post semaphore to unblock app + if (pContext->pRsp) { + // for synchronous API, post semaphore to unblock app pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; pContext->pRsp->pCont = NULL; pContext->pRsp->contLen = 0; tsem_post(pContext->pSem); } - pContext->pConn = NULL; + pContext->pConn = NULL; taosRemoveRef(tsRpcRefId, pContext->rid); } else { - assert(0); + assert(0); } } } @@ -682,8 +811,7 @@ static void rpcCloseConn(void *thandle) { rpcLockConn(pConn); - if (pConn->user[0]) - rpcReleaseConn(pConn); + if (pConn->user[0]) rpcReleaseConn(pConn); rpcUnlockConn(pConn); } @@ -717,8 +845,9 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); - + size_t size = + snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + // check if it is already allocated SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; @@ -767,22 +896,23 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, hashstr); + tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, + hashstr); } return pConn; } static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { - SRpcConn *pConn = NULL; + SRpcConn *pConn = NULL; SRpcHead *pHead = (SRpcHead *)pRecv->msg; if (sid) { pConn = pRpc->connList + sid; if (pConn->user[0] == 0) pConn = NULL; - } + } - if (pConn == NULL) { + if (pConn == NULL) { if (pRpc->connType == TAOS_CONN_SERVER) { pConn = rpcAllocateServerConn(pRpc, pRecv); } else { @@ -805,12 +935,13 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcConn *pConn; SRpcInfo *pRpc = pContext->pRpc; - SEpSet *pEpSet = &pContext->epSet; + SEpSet * pEpSet = &pContext->epSet; - pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); - if ( pConn == NULL || pConn->user[0] == 0) { + pConn = + rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + if (pConn == NULL || pConn->user[0] == 0) { pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); - } + } if (pConn) { pConn->tretry = 0; @@ -825,55 +956,52 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { } static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { - - if (pConn->peerId == 0) { - pConn->peerId = pHead->sourceId; - } else { - if (pConn->peerId != pHead->sourceId) { - tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, - pConn->peerId, pHead->sourceId); - return TSDB_CODE_RPC_INVALID_VALUE; - } + if (pConn->peerId == 0) { + pConn->peerId = pHead->sourceId; + } else { + if (pConn->peerId != pHead->sourceId) { + tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, pConn->peerId, pHead->sourceId); + return TSDB_CODE_RPC_INVALID_VALUE; } + } - if (pConn->inTranId == pHead->tranId) { - if (pConn->inType == pHead->msgType) { - if (pHead->code == 0) { - tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); - rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); - } else { - // do nothing, it is heart beat from client - } - } else if (pConn->inType == 0) { - tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); - rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response + if (pConn->inTranId == pHead->tranId) { + if (pConn->inType == pHead->msgType) { + if (pHead->code == 0) { + tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); + rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); } else { - tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); + // do nothing, it is heart beat from client } - - // do not reply any message - return TSDB_CODE_RPC_ALREADY_PROCESSED; + } else if (pConn->inType == 0) { + tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); + rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response + } else { + tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); } - if (pConn->inType != 0) { - tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, - pConn->inTranId, pHead->tranId); - return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; - } + // do not reply any message + return TSDB_CODE_RPC_ALREADY_PROCESSED; + } - if (rpcContLenFromMsg(pHead->msgLen) <= 0) { - tDebug("%s, message body is empty, ignore", pConn->info); - return TSDB_CODE_RPC_APP_ERROR; - } + if (pConn->inType != 0) { + tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, pConn->inTranId, pHead->tranId); + return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; + } - pConn->inTranId = pHead->tranId; - pConn->inType = pHead->msgType; + if (rpcContLenFromMsg(pHead->msgLen) <= 0) { + tDebug("%s, message body is empty, ignore", pConn->info); + return TSDB_CODE_RPC_APP_ERROR; + } - // start the progress timer to monitor the response from server app - if (pConn->connType != RPC_CONN_TCPS) - pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); - - return 0; + pConn->inTranId = pHead->tranId; + pConn->inType = pHead->msgType; + + // start the progress timer to monitor the response from server app + if (pConn->connType != RPC_CONN_TCPS) + pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); + + return 0; } static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { @@ -898,7 +1026,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_AUTH_REQUIRED && pRpc->spi) { tDebug("%s, authentication shall be restarted", pConn->info); pConn->secured = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); if (pConn->connType != RPC_CONN_TCPC) pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); return TSDB_CODE_RPC_ALREADY_PROCESSED; @@ -908,7 +1036,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { tDebug("%s, mismatched linkUid, link shall be restarted", pConn->info); pConn->secured = 0; ((SRpcHead *)pConn->pReqMsg)->destId = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); if (pConn->connType != RPC_CONN_TCPC) pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); return TSDB_CODE_RPC_ALREADY_PROCESSED; @@ -934,25 +1062,25 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { pConn->reqMsgLen = 0; SRpcReqContext *pContext = pConn->pContext; - if (pHead->code == TSDB_CODE_RPC_REDIRECT) { + if (pHead->code == TSDB_CODE_RPC_REDIRECT) { if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SEpSet)) { // if EpSet is not included in the msg, treat it as NOT_READY - pHead->code = TSDB_CODE_RPC_NOT_READY; + pHead->code = TSDB_CODE_RPC_NOT_READY; } else { pContext->redirect++; if (pContext->redirect > TSDB_MAX_REPLICA) { - pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; tWarn("%s, too many redirects, quit", pConn->info); } } - } + } return TSDB_CODE_SUCCESS; } static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqContext **ppContext) { - int32_t sid; - SRpcConn *pConn = NULL; + int32_t sid; + SRpcConn *pConn = NULL; SRpcHead *pHead = (SRpcHead *)pRecv->msg; @@ -961,25 +1089,29 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont if (TMSG_INDEX(pHead->msgType) >= TDMT_MAX || TMSG_INDEX(pHead->msgType) <= 0) { tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType); - terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; return NULL; + terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; + return NULL; } if (sid < 0 || sid >= pRpc->sessions) { - tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, - pRpc->sessions, TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; + tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, pRpc->sessions, + TMSG_INFO(pHead->msgType)); + terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; + return NULL; } if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { - tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, + TMSG_INFO(pHead->msgType)); + terrno = TSDB_CODE_RPC_INVALID_VERSION; + return NULL; } pConn = rpcGetConnObj(pRpc, sid, pRecv); if (pConn == NULL) { - tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); + tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); return NULL; - } + } rpcLockConn(pConn); @@ -990,9 +1122,9 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont sid = pConn->sid; if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle; - pConn->peerIp = pRecv->ip; + pConn->peerIp = pRecv->ip; pConn->peerPort = pRecv->port; - if (pHead->port) pConn->peerPort = htons(pHead->port); + if (pHead->port) pConn->peerPort = htons(pHead->port); terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen); @@ -1004,16 +1136,16 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont // decrypt here } - if ( rpcIsReq(pHead->msgType) ) { + if (rpcIsReq(pHead->msgType)) { pConn->connType = pRecv->connType; terrno = rpcProcessReqHead(pConn, pHead); // stop idle timer - taosTmrStopA(&pConn->pIdleTimer); + taosTmrStopA(&pConn->pIdleTimer); - // client shall send the request within tsRpcTime again for UDP, double it + // client shall send the request within tsRpcTime again for UDP, double it if (pConn->connType != RPC_CONN_TCPS) - pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer*2, pConn, pRpc->tmrCtrl); + pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer * 2, pConn, pRpc->tmrCtrl); } else { terrno = rpcProcessRspHead(pConn, pHead); *ppContext = pConn->pContext; @@ -1026,9 +1158,9 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont } static void doRpcReportBrokenLinkToServer(void *param, void *id) { - SRpcMsg *pRpcMsg = (SRpcMsg *)(param); - SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); - SRpcInfo *pRpc = pConn->pRpc; + SRpcMsg * pRpcMsg = (SRpcMsg *)(param); + SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); + SRpcInfo *pRpc = pConn->pRpc; (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); free(pRpcMsg); } @@ -1041,12 +1173,12 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { tDebug("%s, notify the server app, connection is gone", pConn->info); SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); - rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server - rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length + rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server + rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length rpcMsg->ahandle = pConn->ahandle; rpcMsg->handle = pConn; rpcMsg->msgType = pConn->inType; - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; if (pRpc->cfp) { @@ -1070,22 +1202,22 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); } - - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); rpcUnlockConn(pConn); } static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; - SRpcConn *pConn = (SRpcConn *)pRecv->thandle; + SRpcHead *pHead = (SRpcHead *)pRecv->msg; + SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; + SRpcConn *pConn = (SRpcConn *)pRecv->thandle; tDump(pRecv->msg, pRecv->msgLen); // underlying UDP layer does not know it is server or client - pRecv->connType = pRecv->connType | pRpc->connType; + pRecv->connType = pRecv->connType | pRpc->connType; if (pRecv->msg == NULL) { rpcProcessBrokenLink(pConn); @@ -1100,62 +1232,62 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { taosIpPort2String(pRecv->ip, pRecv->port, ipstr); if (TMSG_INDEX(pHead->msgType) >= 1 && TMSG_INDEX(pHead->msgType) < TDMT_MAX) { - tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, - pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); + tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, + (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, pHead->sourceId, + pHead->destId, pHead->tranId, pHead->code); } else { - tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, - pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); + tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, + (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, + pHead->tranId, pHead->code); } int32_t code = terrno; if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) { - if (code != 0) { // parsing error + if (code != 0) { // parsing error if (rpcIsReq(pHead->msgType)) { rpcSendErrorMsgToPeer(pRecv, code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { rpcCloseConn(pConn); } if (TMSG_INDEX(pHead->msgType) + 1 > 1 && TMSG_INDEX(pHead->msgType) + 1 < TDMT_MAX) { - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType+1), code); + tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, + TMSG_INFO(pHead->msgType + 1), code); } else { - tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), code); + tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, + TMSG_INFO(pHead->msgType), code); } - } - } else { // msg is passed to app only parsing is ok + } + } else { // msg is passed to app only parsing is ok rpcProcessIncomingMsg(pConn, pHead, pContext); } } - if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed + if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed return pConn; } static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { - SRpcInfo *pRpc = pContext->pRpc; + SRpcInfo *pRpc = pContext->pRpc; pContext->pConn = NULL; - if (pContext->pRsp) { + if (pContext->pRsp) { // for synchronous API memcpy(pContext->pSet, &pContext->epSet, sizeof(SEpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); tsem_post(pContext->pSem); } else { - // for asynchronous API + // for asynchronous API SEpSet *pEpSet = NULL; - if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) - pEpSet = &pContext->epSet; + if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet); } // free the request message - taosRemoveRef(tsRpcRefId, pContext->rid); + taosRemoveRef(tsRpcRefId, pContext->rid); } static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { - SRpcInfo *pRpc = pConn->pRpc; SRpcMsg rpcMsg; @@ -1180,14 +1312,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType); + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], + pConn->connType); } else { rpcCloseConn(pConn); } if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - SEpSet *pEpSet = (SEpSet*)pHead->content; + SEpSet *pEpSet = (SEpSet *)pHead->content; if (pEpSet->numOfEps > 0) { memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, @@ -1200,7 +1333,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte } rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); - } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || pHead->code == TSDB_CODE_DND_OFFLINE) { + } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || + pHead->code == TSDB_CODE_DND_OFFLINE) { pContext->code = pHead->code; rpcProcessConnError(pContext, NULL); rpcFreeCont(rpcMsg.pCont); @@ -1211,14 +1345,14 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte } static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; + char msg[RPC_MSG_OVERHEAD]; + SRpcHead *pHead; // set msg header memset(msg, 0, sizeof(SRpcHead)); pHead = (SRpcHead *)msg; pHead->version = 1; - pHead->msgType = pConn->inType+1; + pHead->msgType = pConn->inType + 1; pHead->spi = pConn->spi; pHead->encrypt = 0; pHead->tranId = pConn->inTranId; @@ -1230,12 +1364,12 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { pHead->code = htonl(code); rpcSendMsgToPeer(pConn, msg, sizeof(SRpcHead)); - pConn->secured = 1; // connection shall be secured + pConn->secured = 1; // connection shall be secured } static void rpcSendReqHead(SRpcConn *pConn) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; + char msg[RPC_MSG_OVERHEAD]; + SRpcHead *pHead; // set msg header memset(msg, 0, sizeof(SRpcHead)); @@ -1257,10 +1391,10 @@ static void rpcSendReqHead(SRpcConn *pConn) { } static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { - SRpcHead *pRecvHead, *pReplyHead; - char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t) ]; - uint32_t timeStamp; - int msgLen; + SRpcHead *pRecvHead, *pReplyHead; + char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t)]; + uint32_t timeStamp; + int msgLen; pRecvHead = (SRpcHead *)pRecv->msg; pReplyHead = (SRpcHead *)msg; @@ -1290,14 +1424,14 @@ static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { pReplyHead->msgLen = (int32_t)htonl((uint32_t)msgLen); (*taosSendData[pRecv->connType])(pRecv->ip, pRecv->port, msg, msgLen, pRecv->chandle); - return; + return; } static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { - SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); - char *msg = (char *)pHead; - int msgLen = rpcMsgLenFromCont(pContext->contLen); - tmsg_t msgType = pContext->msgType; + SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); + char * msg = (char *)pHead; + int msgLen = rpcMsgLenFromCont(pContext->contLen); + tmsg_t msgType = pContext->msgType; pContext->numOfTry++; SRpcConn *pConn = rpcSetupConnToServer(pContext); @@ -1311,13 +1445,13 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { pConn->ahandle = pContext->ahandle; rpcLockConn(pConn); - // set the message header + // set the message header pHead->version = 1; pHead->msgVer = htonl(tsVersion >> 8); pHead->msgType = msgType; pHead->encrypt = 0; pConn->tranId++; - if ( pConn->tranId == 0 ) pConn->tranId++; + if (pConn->tranId == 0) pConn->tranId++; pHead->tranId = pConn->tranId; pHead->sourceId = pConn->ownId; pHead->destId = pConn->peerId; @@ -1341,45 +1475,43 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { } static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { - int writtenLen = 0; - SRpcHead *pHead = (SRpcHead *)msg; + int writtenLen = 0; + SRpcHead *pHead = (SRpcHead *)msg; msgLen = rpcAddAuthPart(pConn, msg, msgLen); - if ( rpcIsReq(pHead->msgType)) { - tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pConn->info, TMSG_INFO(pHead->msgType), pConn->peerFqdn, pConn->peerPort, - msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (rpcIsReq(pHead->msgType)) { + tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { - if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured - tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pConn->info, TMSG_INFO(pHead->msgType), pConn->peerIp, pConn->peerPort, - htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured + tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } - //tTrace("connection type is: %d", pConn->connType); + // tTrace("connection type is: %d", pConn->connType); writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); if (writtenLen != msgLen) { tError("%s, failed to send, msgLen:%d written:%d, reason:%s", pConn->info, msgLen, writtenLen, strerror(errno)); } - + tDump(msg, msgLen); } static void rpcProcessConnError(void *param, void *id) { SRpcReqContext *pContext = (SRpcReqContext *)param; - SRpcInfo *pRpc = pContext->pRpc; + SRpcInfo * pRpc = pContext->pRpc; SRpcMsg rpcMsg; - + if (pRpc == NULL) { return; } - + tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TDMT_VND_FETCH) { - rpcMsg.msgType = pContext->msgType+1; + rpcMsg.msgType = pContext->msgType + 1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; @@ -1387,7 +1519,7 @@ static void rpcProcessConnError(void *param, void *id) { rpcNotifyClient(pContext, &rpcMsg); } else { - // move to next IP + // move to next IP pContext->epSet.inUse++; pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps; rpcSendReqToServer(pRpc, pContext); @@ -1407,11 +1539,12 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { if (pConn->retry < 4) { tDebug("%s, re-send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); } else { // close the connection - tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); + tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, + pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pContext->pConn = NULL; @@ -1434,7 +1567,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) { if (pConn->user[0]) { tDebug("%s, close the connection since no activity", pConn->info); - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); } else { tDebug("%s, idle timer:%p not processed", pConn->info, tmrId); @@ -1460,34 +1593,34 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) { rpcUnlockConn(pConn); } -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { - SRpcHead *pHead = rpcHeadFromCont(pCont); - int32_t finalLen = 0; - int overhead = sizeof(SRpcComp); - +static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { + SRpcHead *pHead = rpcHeadFromCont(pCont); + int32_t finalLen = 0; + int overhead = sizeof(SRpcComp); + if (!NEEDTO_COMPRESSS_MSG(contLen)) { return contLen; } - - char *buf = malloc (contLen + overhead + 8); // 8 extra bytes + + char *buf = malloc(contLen + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); return contLen; } - + int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); - + /* * only the compressed size is less than the value of contLen - overhead, the compression is applied * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message */ if (compLen > 0 && compLen < contLen - overhead) { SRpcComp *pComp = (SRpcComp *)pCont; - pComp->reserved = 0; - pComp->contLen = htonl(contLen); + pComp->reserved = 0; + pComp->contLen = htonl(contLen); memcpy(pCont + overhead, buf, compLen); - + pHead->comp = 1; tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); finalLen = compLen + overhead; @@ -1500,29 +1633,29 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { } static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { - int overhead = sizeof(SRpcComp); - SRpcHead *pNewHead = NULL; - uint8_t *pCont = pHead->content; - SRpcComp *pComp = (SRpcComp *)pHead->content; + int overhead = sizeof(SRpcComp); + SRpcHead *pNewHead = NULL; + uint8_t * pCont = pHead->content; + SRpcComp *pComp = (SRpcComp *)pHead->content; if (pHead->comp) { // decompress the content assert(pComp->reserved == 0); int contLen = htonl(pComp->contLen); - + // prepare the temporary buffer to decompress message char *temp = (char *)malloc(contLen + RPC_MSG_OVERHEAD); - pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext - + pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext + if (pNewHead) { int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; - int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); + int origLen = LZ4_decompress_safe((char *)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); assert(origLen == contLen); - + memcpy(pNewHead, pHead, sizeof(SRpcHead)); pNewHead->msgLen = rpcMsgLenFromCont(origLen); - rpcFreeMsg(pHead); // free the compressed message buffer - pHead = pNewHead; + rpcFreeMsg(pHead); // free the compressed message buffer + pHead = pNewHead; tTrace("decomp malloc mem:%p", temp); } else { tError("failed to allocate memory to decompress msg, contLen:%d", contLen); @@ -1534,7 +1667,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { T_MD5_CTX context; - int ret = -1; + int ret = -1; tMD5Init(&context); tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); @@ -1582,25 +1715,25 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { SRpcHead *pHead = (SRpcHead *)msg; int code = 0; - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)){ - // secured link, or no authentication + if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { + // secured link, or no authentication pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, secured link, no auth is required", pConn->info); return 0; } - if ( !rpcIsReq(pHead->msgType) ) { + if (!rpcIsReq(pHead->msgType)) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || - code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || + code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); return 0; } } - + code = 0; if (pHead->spi == pConn->spi) { // authentication @@ -1613,12 +1746,12 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); code = TSDB_CODE_RPC_INVALID_TIME_STAMP; } else { - if (rpcAuthenticateMsg(pHead, msgLen-TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + if (rpcAuthenticateMsg(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { tDebug("%s, authentication failed, msg discarded", pConn->info); code = TSDB_CODE_RPC_AUTH_FAILURE; } else { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if ( !rpcIsReq(pHead->msgType) ) pConn->secured = 1; // link is secured for client + if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client // tTrace("%s, message is authenticated", pConn->info); } } @@ -1647,13 +1780,9 @@ static void rpcUnlockConn(SRpcConn *pConn) { } } -static void rpcAddRef(SRpcInfo *pRpc) -{ - atomic_add_fetch_32(&pRpc->refCount, 1); -} +static void rpcAddRef(SRpcInfo *pRpc) { atomic_add_fetch_32(&pRpc->refCount, 1); } -static void rpcDecRef(SRpcInfo *pRpc) -{ +static void rpcDecRef(SRpcInfo *pRpc) { if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { rpcCloseConnCache(pRpc->pCache); taosHashCleanup(pRpc->hash); @@ -1668,4 +1797,4 @@ static void rpcDecRef(SRpcInfo *pRpc) atomic_sub_fetch_32(&tsRpcNum, 1); } } - +#endif diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index d0710c883f..a3e1f2434b 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -13,24 +13,28 @@ * along with this program. If not, see . */ +#include "rpcTcp.h" +#include #include "os.h" -#include "tutil.h" +#include "rpcHead.h" +#include "rpcLog.h" #include "taosdef.h" #include "taoserror.h" -#include "rpcLog.h" -#include "rpcHead.h" -#include "rpcTcp.h" +#include "tutil.h" +#ifdef USE_UV + +#else typedef struct SFdObj { - void *signature; - SOCKET fd; // TCP socket FD - void *thandle; // handle from upper layer, like TAOS + void * signature; + SOCKET fd; // TCP socket FD + void * thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; - int16_t closedByApp; // 1: already closed by App + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; - struct SFdObj *prev; - struct SFdObj *next; + struct SFdObj * prev; + struct SFdObj * next; } SFdObj; typedef struct SThreadObj { @@ -43,35 +47,35 @@ typedef struct SThreadObj { int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; - void *shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pPacket); + void * shandle; // handle passed by upper layer during server initialization + void *(*processData)(SRecvInfo *pPacket); } SThreadObj; typedef struct { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; SThreadObj **pThreadObj; } SClientObj; typedef struct { - SOCKET fd; - uint32_t ip; - uint16_t port; - int8_t stop; - int8_t reserve; - char label[TSDB_LABEL_LEN]; - int numOfThreads; - void * shandle; + SOCKET fd; + uint32_t ip; + uint16_t port; + int8_t stop; + int8_t reserve; + char label[TSDB_LABEL_LEN]; + int numOfThreads; + void * shandle; SThreadObj **pThreadObj; - pthread_t thread; + pthread_t thread; } SServerObj; -static void *taosProcessTcpData(void *param); +static void * taosProcessTcpData(void *param); static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); -static void *taosAcceptTcpConnection(void *arg); +static void * taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { SServerObj *pServerObj; @@ -99,7 +103,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return NULL; } - int code = 0; + int code = 0; pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -110,7 +114,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j=0; jpThreadObj[j]); + for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]); free(pServerObj->pThreadObj); free(pServerObj); return NULL; @@ -172,8 +176,10 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return (void *)pServerObj; } -static void taosStopTcpThread(SThreadObj* pThreadObj) { - if (pThreadObj == NULL) { return;} +static void taosStopTcpThread(SThreadObj *pThreadObj) { + if (pThreadObj == NULL) { + return; + } // save thread into local variable and signal thread to stop pthread_t thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { @@ -194,7 +200,7 @@ void taosStopTcpServer(void *handle) { pServerObj->stop = 1; if (pServerObj->fd >= 0) { - taosShutDownSocketRD(pServerObj->fd); + taosShutDownSocketRD(pServerObj->fd); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -227,8 +233,8 @@ static void *taosAcceptTcpConnection(void *arg) { SOCKET connFd = -1; struct sockaddr_in caddr; int threadId = 0; - SThreadObj *pThreadObj; - SServerObj *pServerObj; + SThreadObj * pThreadObj; + SServerObj * pServerObj; pServerObj = (SServerObj *)arg; tDebug("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); @@ -253,8 +259,8 @@ static void *taosAcceptTcpConnection(void *arg) { } taosKeepTcpAlive(connFd); - struct timeval to={5, 0}; - int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + struct timeval to = {5, 0}; + int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { taosCloseSocket(connFd); tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), @@ -262,7 +268,6 @@ static void *taosAcceptTcpConnection(void *arg) { continue; } - // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; @@ -271,7 +276,7 @@ static void *taosAcceptTcpConnection(void *arg) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), @@ -297,14 +302,14 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); + pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); tfree(pClientObj); terrno = TAOS_SYSTEM_ERROR(errno); } - int code = 0; + int code = 0; pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -314,15 +319,15 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j=0; jpThreadObj[j]); + for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); free(pClientObj); pthread_attr_destroy(&thattr); return NULL; } pClientObj->pThreadObj[i] = pThreadObj; taosResetPthread(&pThreadObj->thread); - pThreadObj->ip = ip; - pThreadObj->stop = false; + pThreadObj->ip = ip; + pThreadObj->stop = false; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); pThreadObj->shandle = shandle; pThreadObj->processData = fp; @@ -364,14 +369,14 @@ void taosStopTcpClient(void *chandle) { if (pClientObj == NULL) return; - tDebug ("%s TCP client is stopped", pClientObj->label); + tDebug("%s TCP client is stopped", pClientObj->label); } void taosCleanUpTcpClient(void *chandle) { SClientObj *pClientObj = chandle; if (pClientObj == NULL) return; for (int i = 0; i < pClientObj->numOfThreads; ++i) { - SThreadObj *pThreadObj= pClientObj->pThreadObj[i]; + SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; taosStopTcpThread(pThreadObj); } @@ -381,9 +386,9 @@ void taosCleanUpTcpClient(void *chandle) { } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SClientObj * pClientObj = shandle; - int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; - atomic_store_32(&pClientObj->index, index + 1); + SClientObj *pClientObj = shandle; + int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; + atomic_store_32(&pClientObj->index, index + 1); SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); @@ -394,10 +399,9 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin #endif struct sockaddr_in sin; - uint16_t localPort = 0; - unsigned int addrlen = sizeof(sin); - if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + uint16_t localPort = 0; + unsigned int addrlen = sizeof(sin); + if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { localPort = (uint16_t)ntohs(sin.sin_port); } @@ -407,8 +411,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", - pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); + tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, + ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); taosCloseSocket(fd); @@ -441,7 +445,6 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand } static void taosReportBrokenLink(SFdObj *pFdObj) { - SThreadObj *pThreadObj = pFdObj->pThreadObj; // notify the upper layer, so it will clean the associated context @@ -464,9 +467,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { } static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { - SRpcHead rpcHead; - int32_t msgLen, leftLen, retLen, headLen; - char *buffer, *msg; + SRpcHead rpcHead; + int32_t msgLen, leftLen, retLen, headLen; + char * buffer, *msg; SThreadObj *pThreadObj = pFdObj->pThreadObj; @@ -483,7 +486,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer); + tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, + buffer); } msg = buffer + tsRpcOverhead; @@ -491,8 +495,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", - pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } @@ -519,8 +522,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { #define maxEvents 10 static void *taosProcessTcpData(void *param) { - SThreadObj *pThreadObj = param; - SFdObj *pFdObj; + SThreadObj * pThreadObj = param; + SFdObj * pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; @@ -569,7 +572,7 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) { + if (pThreadObj->pollFd >= 0) { EpollClose(pThreadObj->pollFd); pThreadObj->pollFd = -1; } @@ -620,7 +623,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { } static void taosFreeFdObj(SFdObj *pFdObj) { - if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; @@ -638,8 +640,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", - pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, + pThreadObj->threadId); if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; @@ -653,8 +655,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", - pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, + pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); } + +#endif diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 5bc31c189a..79956cc98d 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -13,50 +13,53 @@ * along with this program. If not, see . */ +#include "rpcUdp.h" #include "os.h" -#include "ttimer.h" -#include "tutil.h" +#include "rpcHead.h" +#include "rpcLog.h" #include "taosdef.h" #include "taoserror.h" -#include "rpcLog.h" -#include "rpcUdp.h" -#include "rpcHead.h" +#include "ttimer.h" +#include "tutil.h" +#ifdef USE_UV +// no support upd currently +#else #define RPC_MAX_UDP_CONNS 256 #define RPC_MAX_UDP_PKTS 1000 #define RPC_UDP_BUF_TIME 5 // mseconds #define RPC_MAX_UDP_SIZE 65480 typedef struct { - int index; - SOCKET fd; - uint16_t port; // peer port - uint16_t localPort; // local port - char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; - void *hash; - void *shandle; // handle passed by upper layer during server initialization - void *pSet; - void *(*processData)(SRecvInfo *pRecv); - char *buffer; // buffer to receive data + int index; + SOCKET fd; + uint16_t port; // peer port + uint16_t localPort; // local port + char label[TSDB_LABEL_LEN]; // copy from udpConnSet; + pthread_t thread; + void * hash; + void * shandle; // handle passed by upper layer during server initialization + void * pSet; + void *(*processData)(SRecvInfo *pRecv); + char *buffer; // buffer to receive data } SUdpConn; typedef struct { - int index; - int server; - uint32_t ip; // local IP - uint16_t port; // local Port - void *shandle; // handle passed by upper layer during server initialization - int threads; - char label[TSDB_LABEL_LEN]; - void *(*fp)(SRecvInfo *pPacket); - SUdpConn udpConn[]; + int index; + int server; + uint32_t ip; // local IP + uint16_t port; // local Port + void * shandle; // handle passed by upper layer during server initialization + int threads; + char label[TSDB_LABEL_LEN]; + void *(*fp)(SRecvInfo *pPacket); + SUdpConn udpConn[]; } SUdpConnSet; static void *taosRecvUdpData(void *param); void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn *pConn; + SUdpConn * pConn; SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); @@ -79,7 +82,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - int i; + int i; uint16_t ownPort; for (i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -97,9 +100,9 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads } struct sockaddr_in sin; - unsigned int addrlen = sizeof(sin); - if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + unsigned int addrlen = sizeof(sin); + if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && + addrlen == sizeof(sin)) { pConn->localPort = (uint16_t)ntohs(sin.sin_port); } @@ -118,7 +121,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pthread_attr_destroy(&thAttr); - if (i != threads) { + if (i != threads) { terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpUdpConnection(pSet); return NULL; @@ -130,14 +133,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads void taosStopUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn *pConn; + SUdpConn * pConn; if (pSet == NULL) return; for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); - if (pConn->fd >=0) taosCloseSocket(pConn->fd); + if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR); + if (pConn->fd >= 0) taosCloseSocket(pConn->fd); pConn->fd = -1; } @@ -155,13 +158,13 @@ void taosStopUdpConnection(void *handle) { void taosCleanUpUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn *pConn; + SUdpConn * pConn; if (pSet == NULL) return; for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >=0) taosCloseSocket(pConn->fd); + if (pConn->fd >= 0) taosCloseSocket(pConn->fd); } tDebug("%s UDP is cleaned up", pSet->label); @@ -182,7 +185,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t } static void *taosRecvUdpData(void *param) { - SUdpConn *pConn = param; + SUdpConn * pConn = param; struct sockaddr_in sourceAdd; ssize_t dataLen; unsigned int addLen; @@ -218,7 +221,7 @@ static void *taosRecvUdpData(void *param) { } int32_t size = dataLen + tsRpcOverhead; - char *tmsg = malloc(size); + char * tmsg = malloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; @@ -257,4 +260,4 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c return ret; } - +#endif diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index 6dea4a4e57..f2f48bbc8a 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -11,4 +11,4 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ From 4aa8be83339472d639ec24543c4671dd9f1dfabf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jan 2022 23:16:45 +0800 Subject: [PATCH 09/63] [td-11818] Support the select *. --- include/libs/planner/planner.h | 4 +- source/client/src/clientImpl.c | 26 ++++++++++++- source/client/src/clientMsgHandler.c | 3 -- source/libs/executor/src/executorimpl.c | 39 +++++++++++++------ source/libs/planner/src/physicalPlan.c | 19 +++++---- source/libs/planner/src/physicalPlanJson.c | 29 ++++++++++---- source/libs/scheduler/src/scheduler.c | 10 +---- source/libs/scheduler/test/schedulerTests.cpp | 6 +-- 8 files changed, 92 insertions(+), 44 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index da70f21498..d98c1ca595 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -91,8 +91,10 @@ typedef struct SPhyNode { typedef struct SScanPhyNode { SPhyNode node; - uint64_t uid; // unique id of the table + uint64_t uid; // unique id of the table int8_t tableType; + int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC + int32_t count; // repeat count } SScanPhyNode; typedef SScanPhyNode SSystemTableScanPhyNode; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 727b668f3a..d476670c78 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,8 +25,9 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +static void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp); -static bool stringLengthCheck(const char* str, size_t maxsize) { + static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { return false; } @@ -212,6 +213,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) SSubplan* pPlan = taosArrayGetP(pa, 0); SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); setResSchemaInfo(pResInfo, pDataBlockSchema); + pRequest->type = TDMT_VND_QUERY; } @@ -547,7 +549,8 @@ void* doFetchRow(SRequestObj* pRequest) { if (pRequest->type == TDMT_VND_QUERY) { pRequest->type = TDMT_VND_FETCH; scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); - + setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + pResultInfo->current = 0; if (pResultInfo->numOfRows <= pResultInfo->current) { return NULL; @@ -611,12 +614,23 @@ _return: return pResultInfo->row; } +static void doPrepareResPtr(SReqResultInfo* pResInfo) { + if (pResInfo->row == NULL) { + pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + } +} + void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { return; } + // todo check for the failure of malloc + doPrepareResPtr(pResultInfo); + int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { pResultInfo->length[i] = pResultInfo->fields[i].bytes; @@ -642,3 +656,11 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } +void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp) { + assert(pRsp != NULL && pResultInfo != NULL); + pResultInfo->pRspMsg = pRsp; + pResultInfo->pData = (void*) pRsp->data; + pResultInfo->numOfRows = htonl(pRsp->numOfRows); + + setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); +} \ No newline at end of file diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 85f3fb06a7..4f1a96ad29 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -154,9 +154,6 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pResInfo->fields = pFields; pResInfo->numOfCols = pMetaMsg->numOfColumns; - pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); pRequest->body.showInfo.execId = pShow->showId; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9aeb979806..0dfcc9b1d8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -18,7 +18,6 @@ #include "ttime.h" #include "exception.h" -#include "../../../../contrib/cJson/cJSON.h" #include "executorimpl.h" #include "function.h" #include "tcompare.h" @@ -7188,31 +7187,47 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*) pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); pTaskInfo->pRoot = pOperatorInfo; + } else { + assert(0); } } } int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { - STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + STsdbQueryCond cond = {.loadExternalRows = false}; cond.twindow.skey = INT64_MIN; cond.twindow.ekey = INT64_MAX; - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - // todo set the correct table column info - cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; - cond.colList[0].bytes = sizeof(uint64_t); - cond.colList[0].colId = 1; + uint64_t uid = 0; + SPhyNode* pPhyNode = pPlan->pNode; + if (pPhyNode->info.type == OP_TableScan) { - cond.colList[1].type = TSDB_DATA_TYPE_INT; - cond.colList[1].bytes = sizeof(int32_t); - cond.colList[1].colId = 2; + SScanPhyNode* pScanNode = (SScanPhyNode*) pPhyNode; + uid = pScanNode->uid; + cond.order = pScanNode->order; + cond.numOfCols = taosArrayGetSize(pScanNode->node.pTargets); + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + + for(int32_t i = 0; i < cond.numOfCols; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pScanNode->node.pTargets, i); + assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE); + + SSchema* pSchema = pExprInfo->pExpr->pSchema; + cond.colList[i].type = pSchema->type; + cond.colList[i].bytes = pSchema->bytes; + cond.colList[i].colId = pSchema->colId; + } + } else { + assert(0); + } STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; taosArrayPush(pa, &info); taosArrayPush(group.pGroupList, &pa); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 01fb3c2513..0604ba3920 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -156,9 +156,14 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si } static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { - SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size); - node->uid = pTable->pMeta->pTableMeta->uid; - node->tableType = pTable->pMeta->pTableMeta->tableType; + SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size); + + STableMeta *pTableMeta = pTable->pMeta->pTableMeta; + node->uid = pTableMeta->uid; + node->count = 1; + node->order = TSDB_ORDER_ASC; + node->tableType = pTableMeta->tableType; + return (SPhyNode*)node; } @@ -176,10 +181,10 @@ static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { return MAIN_SCAN; } -static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { - STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode)); - node->scanFlag = getScanFlag(pPlanNode, pTable); - node->window = pTable->window; +static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pQueryTableInfo, int32_t op) { + STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pQueryTableInfo, op, sizeof(STableScanPhyNode)); + node->scanFlag = getScanFlag(pPlanNode, pQueryTableInfo); + node->window = pQueryTableInfo->window; // todo tag cond return (SPhyNode*)node; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 895788c550..64490932fa 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -136,7 +136,7 @@ static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size) static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) { for (int32_t i = 0; i < size; ++i) { - if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) { + if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize * i)) { return false; } } @@ -165,9 +165,7 @@ static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, vo static char* getString(const cJSON* json, const char* name) { char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); - char* res = calloc(1, strlen(p) + 1); - strcpy(res, p); - return res; + return strdup(p); } static void copyString(const cJSON* json, const char* name, char* dst) { @@ -285,7 +283,7 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) { static bool columnInfoFromJson(const cJSON* json, void* obj) { SColumnInfo* col = (SColumnInfo*)obj; col->colId = getNumber(json, jkColumnInfoColId); - col->type = getNumber(json, jkColumnInfoType); + col->type = getNumber(json, jkColumnInfoType); col->bytes = getNumber(json, jkColumnInfoBytes); int32_t size = 0; bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); @@ -530,13 +528,25 @@ static bool timeWindowFromJson(const cJSON* json, void* obj) { static const char* jkScanNodeTableId = "TableId"; static const char* jkScanNodeTableType = "TableType"; +static const char* jkScanNodeTableOrder = "Order"; +static const char* jkScanNodeTableCount = "Count"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* scan = (const SScanPhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); + } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); + } + return res; } @@ -544,6 +554,8 @@ static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* scan = (SScanPhyNode*)obj; scan->uid = getNumber(json, jkScanNodeTableId); scan->tableType = getNumber(json, jkScanNodeTableType); + scan->count = getNumber(json, jkScanNodeTableCount); + scan->order = getNumber(json, jkScanNodeTableOrder); return true; } @@ -748,9 +760,11 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { } static bool phyNodeFromJson(const cJSON* json, void* obj) { - SPhyNode* node = (SPhyNode*)obj; + SPhyNode* node = (SPhyNode*) obj; + node->info.name = getString(json, jkPnodeName); node->info.type = opNameToOpType(node->info.name); + bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); if (res) { res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); @@ -896,7 +910,8 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false); + size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); + res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); } if (res) { res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ed12408844..525d6ebe87 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -798,7 +798,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SQueryTableRsp *rsp = (SQueryTableRsp *)msg; if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); @@ -1364,16 +1364,10 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = NULL; - - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false)); - - *pJob = job; - + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, false)); return TSDB_CODE_SUCCESS; } - int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index b438521234..ccf22173cb 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -160,7 +160,7 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { } void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - + } @@ -218,8 +218,6 @@ void *schtSendRsp(void *param) { } struct SSchJob *pInsertJob = NULL; - - } TEST(queryTest, normalCase) { @@ -336,7 +334,7 @@ TEST(insertTest, normalCase) { uint64_t numOfRows = 0; schtInitLogFile(); - + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; From dde3dfd7bd9fc58075f58ecce04ab6311e31c502 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 11 Jan 2022 23:53:18 +0800 Subject: [PATCH 10/63] add libuv --- source/libs/transport/src/rpcMain.c | 158 +++++++++++++++++++--------- 1 file changed, 109 insertions(+), 49 deletions(-) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 9bba0dca0a..802c5ae4ea 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -33,28 +33,40 @@ #include "ttimer.h" #include "tutil.h" +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + int index; // for UDP server only, round robin for multiple threads + char label[TSDB_LABEL_LEN]; + + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization + void* udphandle; // returned handle from UDP initialization + void* pCache; // connection cache + pthread_mutex_t mutex; + struct SRpcConn* connList; // connection list +} SRpcInfo; + #ifdef USE_UV #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -int32_t rpcInit() { return -1; } -void rpcCleanup() { return; }; -void* rpcOpen(const SRpcInit* pRpc) { return NULL; } -void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } -void rpcFreeCont(void* cont) { return; } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } - -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } - -void rpcSendResponse(const SRpcMsg* pMsg) {} - -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } -int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } -void rpcCancelRequest(int64_t rid) { return; } - typedef struct SThreadObj { pthread_t thread; uv_pipe_t* pipe; @@ -79,6 +91,84 @@ typedef struct SConnCtx { int ref; } SConnCtx; +static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void onTimeout(uv_timer_t* handle); +static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void onWrite(uv_write_t* req, int status); +static void onAccept(uv_stream_t* stream, int status); +void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void workerAsyncCB(uv_async_t* handle); +static void* workerThread(void* arg); + +int32_t rpcInit() { return -1; } +void rpcCleanup() { return; }; +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + return NULL; + } + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + + SServerObj* srv = calloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + srv->numOfThread = pRpc->numOfThreads; + srv->workerIdx = 0; + srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); + srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); + uv_loop_init(srv->loop); + + for (int i = 0; i < srv->numOfThread; i++) { + srv->pThreadObj[i] = (SThreadObj*)calloc(1, sizeof(SThreadObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + return NULL; + } + uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); + uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write + + srv->pThreadObj[i]->fd = fds[0]; + srv->pThreadObj[i]->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(srv->pThreadObj[i]->thread), NULL, workerThread, (void*)(srv->pThreadObj[i])); + if (err == 0) { + tError("sucess to create worker thread %d", i); + // printf("thread %d create\n", i); + } else { + tError("failed to create worker thread %d", i); + return NULL; + } + } + uv_tcp_init(srv->loop, &srv->server); + struct sockaddr_in bind_addr; + uv_ip4_addr("0.0.0.0", pInit->localPort, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); + + return pRpc; +} +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->base = malloc(suggested_size); buf->len = suggested_size; @@ -98,7 +188,7 @@ void onWrite(uv_write_t* req, int status) { if (req) tDebug("data already was written on stream"); } -static void workerAsyncCB(uv_async_t* handle) { +void workerAsyncCB(uv_async_t* handle) { // opt SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); } @@ -187,36 +277,6 @@ void* workerThread(void* arg) { #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) -typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; - uint16_t localPort; - int8_t connType; - int index; // for UDP server only, round robin for multiple threads - char label[TSDB_LABEL_LEN]; - - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); - - int32_t refCount; - void * parent; - void * idPool; // handle to ID pool - void * tmrCtrl; // handle to timer - SHashObj * hash; // handle returned by hash utility - void * tcphandle; // returned handle from TCP initialization - void * udphandle; // returned handle from UDP initialization - void * pCache; // connection cache - pthread_mutex_t mutex; - struct SRpcConn *connList; // connection list -} SRpcInfo; - typedef struct { SRpcInfo * pRpc; // associated SRpcInfo SEpSet epSet; // ip list provided by app From 876942b4f2a44de024a0afb5585467d3edb39f3d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 12 Jan 2022 09:22:48 +0800 Subject: [PATCH 11/63] feature/qnode --- source/libs/scheduler/inc/schedulerInt.h | 28 ++++++++-- source/libs/scheduler/src/scheduler.c | 55 ++++++++++--------- source/libs/scheduler/test/schedulerTests.cpp | 21 +++++++ 3 files changed, 75 insertions(+), 29 deletions(-) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index c83eba4232..661beee5d5 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -36,11 +36,31 @@ enum { SCH_WRITE, }; +typedef struct SSchApiStat { + +} SSchApiStat; + +typedef struct SSchRuntimeStat { + +} SSchRuntimeStat; + +typedef struct SSchJobStat { + +} SSchJobStat; + +typedef struct SSchedulerStat { + SSchApiStat api; + SSchRuntimeStat runtime; + SSchJobStat job; +} SSchedulerStat; + + typedef struct SSchedulerMgmt { - uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId - SSchedulerCfg cfg; - SHashObj *jobs; // key: queryId, value: SQueryJob* + uint64_t taskId; // sequential taksId + uint64_t sId; // schedulerId + SSchedulerCfg cfg; + SHashObj *jobs; // key: queryId, value: SQueryJob* + SSchedulerStat stat; } SSchedulerMgmt; typedef struct SSchCallbackParam { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ed12408844..aeec1ff5a0 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1462,35 +1462,38 @@ void scheduleFreeJob(void *job) { } SSchJob *pJob = job; + uint64_t queryId = pJob->queryId; - if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { - SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); - return; - } - - schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); - - SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); - - while (true) { - int32_t ref = atomic_load_32(&pJob->ref); - if (0 == ref) { - break; - } else if (ref > 0) { - usleep(1); - } else { - assert(0); + if (SCH_GET_JOB_STATUS(pJob) > 0) { + if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { + SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); + return; } + + schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); + + SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); + + while (true) { + int32_t ref = atomic_load_32(&pJob->ref); + if (0 == ref) { + break; + } else if (ref > 0) { + usleep(1); + } else { + assert(0); + } + } + + SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); + + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { + schCancelJob(pJob); + } + + schDropJobAllTasks(pJob); } - SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); - - if (pJob->status == JOB_TASK_STATUS_EXECUTING) { - schCancelJob(pJob); - } - - schDropJobAllTasks(pJob); - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans int32_t numOfLevels = taosArrayGetSize(pJob->levels); @@ -1515,6 +1518,8 @@ void scheduleFreeJob(void *job) { tfree(pJob->res); tfree(pJob); + + qDebug("QID:%"PRIx64" job freed", queryId); } void schedulerDestroy(void) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index b438521234..114b3c02b5 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -79,6 +79,7 @@ void schtBuildQueryDag(SQueryDag *dag) { scanPlan->level = 1; scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; mergePlan->id.templateId = 0x4444444444; @@ -89,6 +90,7 @@ void schtBuildQueryDag(SQueryDag *dag) { mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); mergePlan->pParents = NULL; mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + mergePlan->msgType = TDMT_VND_QUERY; SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); @@ -163,6 +165,11 @@ void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { } +void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { + +} + + void schtSetPlanToString() { static Stub stub; @@ -190,6 +197,20 @@ void schtSetExecNode() { } } +void schtSetRpcSendRequest() { + static Stub stub; + stub.set(rpcSendRequest, schtRpcSendRequest); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRequest$", result); + for (const auto& f : result) { + stub.set(f.second, schtRpcSendRequest); + } + } +} + + void *schtSendRsp(void *param) { SSchJob *job = NULL; int32_t code = 0; From a5df21beab302198b655ecd6951ac0bbf5505ead Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jan 2022 01:26:00 +0000 Subject: [PATCH 12/63] more --- source/libs/tkv/src/inc/tkvDB.h | 31 +++++++++++++++++++++++++++++++ source/libs/tkv/src/inc/tkvEnv.h | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 source/libs/tkv/src/inc/tkvDB.h create mode 100644 source/libs/tkv/src/inc/tkvEnv.h diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h new file mode 100644 index 0000000000..1a45702540 --- /dev/null +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_DB_H_ +#define _TD_TKV_DB_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB { + // TODO +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvEnv.h b/source/libs/tkv/src/inc/tkvEnv.h new file mode 100644 index 0000000000..eba442e5a5 --- /dev/null +++ b/source/libs/tkv/src/inc/tkvEnv.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_ENV_H_ +#define _TD_TKV_ENV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB_ENV { + char *homeDir; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_ENV_H_*/ \ No newline at end of file From 6d10ef5e69678b0d7e3ad350e6991df95925b1f3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 12 Jan 2022 09:44:23 +0800 Subject: [PATCH 13/63] add libuv --- source/libs/transport/src/rpcMain.c | 2 ++ source/libs/transport/src/rpcTcp.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 802c5ae4ea..3095ddb9d2 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#ifdef USE_UV #include +#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index a3e1f2434b..9fa51a6fdc 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -14,7 +14,9 @@ */ #include "rpcTcp.h" +#ifdef USE_UV #include +#endif #include "os.h" #include "rpcHead.h" #include "rpcLog.h" From 8b5e6b689555f586b43b21b5d4169f0476fe79f9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jan 2022 01:55:37 +0000 Subject: [PATCH 14/63] fix memory leakage --- source/dnode/vnode/impl/src/vnodeWrite.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index ddcb93863a..185487757f 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -83,8 +83,18 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } + if (pCreateTbReq->type == TD_SUPER_TABLE) { + free(pCreateTbReq->stbCfg.pSchema); + free(pCreateTbReq->stbCfg.pTagSchema); + } else if (pCreateTbReq->type == TD_CHILD_TABLE) { + free(pCreateTbReq->ctbCfg.pTag); + } else { + free(pCreateTbReq->ntbCfg.pSchema); + } } - + taosArrayDestroy(vCreateTbBatchReq.pArray); + break; + case TDMT_VND_DROP_STB: case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { From 90e25132e1e006797861b4b42ed1ef55335106ca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:16:59 +0800 Subject: [PATCH 15/63] [td-11818] support select * from child_table. --- source/client/inc/clientInt.h | 3 +-- source/client/src/clientImpl.c | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7292375e7f..ebb8502d6c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -93,13 +93,12 @@ typedef struct SReqResultInfo { const char *pData; TAOS_FIELD *fields; uint32_t numOfCols; - int32_t *length; TAOS_ROW row; char **pCol; - uint32_t numOfRows; uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d476670c78..40354998ee 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,7 +25,7 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -static void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp); +static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -547,12 +547,15 @@ void* doFetchRow(SRequestObj* pRequest) { if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { - pRequest->type = TDMT_VND_FETCH; + // All data has returned to App already, no need to try again + if (pResultInfo->completed) { + return NULL; + } + scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); - pResultInfo->current = 0; - if (pResultInfo->numOfRows <= pResultInfo->current) { + if (pResultInfo->numOfRows == 0) { return NULL; } @@ -656,11 +659,14 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } -void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp) { - assert(pRsp != NULL && pResultInfo != NULL); - pResultInfo->pRspMsg = pRsp; +void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { + assert(pResultInfo != NULL && pRsp != NULL); + + pResultInfo->pRspMsg = (const char*) pRsp; pResultInfo->pData = (void*) pRsp->data; pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->current = 0; + pResultInfo->completed = (pRsp->completed == 1); setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); } \ No newline at end of file From 027109727089329cd327dfeb8e4e63a88e6b93dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:26:00 +0800 Subject: [PATCH 16/63] [td-11818] disable print query plan. --- source/client/src/clientImpl.c | 2 +- source/libs/parser/src/queryInfoUtil.c | 3 +++ source/libs/planner/src/physicalPlanJson.c | 4 ++-- source/libs/planner/src/planner.c | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 40354998ee..6b2d89c9be 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -230,7 +230,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlo SSchema* pSchema = &pDataBlockSchema->pSchema[i]; pResInfo->fields[i].bytes = pSchema->bytes; pResInfo->fields[i].type = pSchema->type; - tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name)); } } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index d7aa758576..1aa8836cb2 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -160,11 +160,14 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } +#if 0 if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); } else { printf("add operator: %s, level:%d, total:%ld\n", pExprInfo->base.resSchema.name, level, taosArrayGetSize(pExprList)); } +#endif + } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 64490932fa..7e36e0e124 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -940,8 +940,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); - - printf("%s\n", *str); +// printf("====Physical plan:====\n") +// printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 3047ef4f5a..21f57d95d5 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From 35b0ee1a10a6930c09e7b81abd287d94d15868ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:53:53 +0800 Subject: [PATCH 17/63] [td-11818]remove SParseBasicCtx --- include/libs/parser/parsenodes.h | 9 -------- include/libs/parser/parser.h | 8 ++++++- source/client/src/clientImpl.c | 13 +++++++----- source/libs/parser/inc/astToMsg.h | 8 +++---- source/libs/parser/inc/parserInt.h | 8 +++---- source/libs/parser/inc/parserUtil.h | 2 +- source/libs/parser/src/astToMsg.c | 8 +++---- source/libs/parser/src/astValidate.c | 4 ++-- source/libs/parser/src/dCDAstProcess.c | 10 ++++----- source/libs/parser/src/insertParser.c | 8 +++---- source/libs/parser/src/parser.c | 14 ++++++------ source/libs/parser/src/parserUtil.c | 2 +- source/libs/parser/test/parserTests.cpp | 26 +++++++++++------------ source/libs/parser/test/plannerTest.cpp | 4 ++-- source/libs/parser/test/tokenizerTest.cpp | 2 +- 15 files changed, 63 insertions(+), 63 deletions(-) diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index ac8a10067d..374ae02e4a 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -43,15 +43,6 @@ typedef struct SField { int32_t bytes; } SField; -typedef struct SParseBasicCtx { - uint64_t requestId; - int32_t acctId; - const char *db; - void *pTransporter; - SEpSet mgmtEpSet; - struct SCatalog *pCatalog; -} SParseBasicCtx; - typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result SField *final; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index edf9cf461f..0cdbfdb07f 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -23,11 +23,17 @@ extern "C" { #include "parsenodes.h" typedef struct SParseContext { - SParseBasicCtx ctx; + uint64_t requestId; + int32_t acctId; + const char *db; + void *pTransporter; + SEpSet mgmtEpSet; const char *pSql; // sql string size_t sqlLen; // length of the sql string char *pMsg; // extended error message if exists to help identifying the problem in sql statement. int32_t msgLen; // max length of the msg + + struct SCatalog *pCatalog; } SParseContext; /** diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6b2d89c9be..a614d2c3c1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -151,23 +151,26 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { - .ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj), .pTransporter = pTscObj->pTransporter}, + .requestId = pRequest->requestId, + .acctId = pTscObj->acctId, + .db = getConnectionDB(pTscObj), + .pTransporter = pTscObj->pTransporter, .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE }; - cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.ctx.pCatalog); + cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); if (code != TSDB_CODE_SUCCESS) { - tfree(cxt.ctx.db); + tfree(cxt.db); return code; } code = qParseQuerySql(&cxt, pQuery); - tfree(cxt.ctx.db); + tfree(cxt.db); return code; } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 83683c2bfc..8acbc6bc11 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -8,10 +8,10 @@ SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, char* msgBuf, int32_t msgLen); +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 10ec335fc8..abb19b3ab6 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -51,7 +51,7 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t * @param msgBufLen * @return */ -int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen); +int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen); /** * validate the ddl ast, and convert the ast to the corresponding message format @@ -60,7 +60,7 @@ int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmt * @param type * @return */ -SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen); /** * @@ -70,7 +70,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c * @param msgBufLen * @return */ -SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); +SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. @@ -98,7 +98,7 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf); * @param msgBufLen * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen); +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen); /** * Destroy the meta data request structure. diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 764b363394..d660d36d3f 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -81,7 +81,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param); typedef int32_t (*_row_append_fn_t)(const void *value, int32_t len, void *param); int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf); -int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 5f45ce824e..5b841594e0 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -85,7 +85,7 @@ SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, cha return pMsg; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, int32_t msgLen) { +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, char* msgBuf, int32_t msgLen) { SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); pShowMsg->type = pShowInfo->showType; @@ -210,7 +210,7 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) { SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { tfree(pCreateMsg); @@ -230,7 +230,7 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt return pCreateMsg; } -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SSchema* pSchema; int32_t numOfTags = 0; @@ -315,7 +315,7 @@ SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len return pCreateStbMsg; } -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index faa8c526a0..5d56a88c14 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3628,7 +3628,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) return TSDB_CODE_SUCCESS; } -int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVgList) { +int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); if (code != TSDB_CODE_SUCCESS) { @@ -3655,7 +3655,7 @@ int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVg return TSDB_CODE_SUCCESS; } -int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) { +int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) { assert(pCtx != NULL && pInfo != NULL); int32_t code = 0; diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6676e1ebf6..4d4d68e962 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { return false; } -static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, +static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** output, int32_t* outputLen, SEpSet* pEpSet, void** pExtension, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid name"; const char* msg2 = "wildcard string should be less than %d characters"; @@ -396,7 +396,7 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) { +static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) { const char* msg1 = "invalid table name"; const char* msg2 = "tags number not matched"; const char* msg3 = "tag value too long"; @@ -634,7 +634,7 @@ static int32_t doBuildSingleTableBatchReq(SName* pTableName, SArray* pColumns, S return TSDB_CODE_SUCCESS; } -int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { +int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { SArray* pBufArray = NULL; int32_t code = 0; @@ -703,7 +703,7 @@ SArray* doSerializeVgroupCreateTableInfo(SHashObj* pVgroupHashmap) { return pBufArray; } -SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); @@ -961,7 +961,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c return NULL; } -SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { +SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) { SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; assert(pCreateTable->type == TSDB_SQL_CREATE_TABLE); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 04c287baf1..8b1506bad1 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -84,11 +84,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); if (NULL != p) { // db.table - int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->ctx.acctId); + int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->acctId); strncpy(fullDbName + n, pStname->z, p - pStname->z); strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); } else { - snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db); + snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->acctId, pCxt->pComCxt->db); strncpy(tableName, pStname->z, pStname->n); } @@ -97,11 +97,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { SName name = {0}; - createSName(&name, pTname, &pCxt->pComCxt->ctx, &pCxt->msg); + createSName(&name, pTname, pCxt->pComCxt, &pCxt->msg); char tableName[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&name, tableName); - SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx; + SParseContext* pBasicCtx = pCxt->pComCxt; CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); SVgroupInfo vg; CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5f17fcfaee..fea215238c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -45,14 +45,14 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { if (!isDqlSqlStatement(&info)) { if (info.type == TSDB_SQL_CREATE_TABLE) { - SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pModifStmtInfo == NULL) { return terrno; } *pQuery = (SQueryNode*)pModifStmtInfo; } else { - SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pDcl == NULL) { return terrno; } @@ -67,7 +67,7 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { return terrno; } - int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); + int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } else { @@ -92,7 +92,7 @@ int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) { return 0; } -static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); +static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf); static int32_t tnameComparFn(const void* p1, const void* p2) { SName* pn1 = (SName*)p1; @@ -116,7 +116,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) { } } -static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { +static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf) { int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); for (int32_t j = 0; j < numOfSub; ++j) { @@ -139,7 +139,7 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis return TSDB_CODE_SUCCESS; } -int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pParseCtx, SMsgBuf* pMsgBuf) { +int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid table name"; int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); @@ -173,7 +173,7 @@ static void freePtrElem(void* p) { tfree(*(char**)p); } -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen) { +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen) { int32_t code = TSDB_CODE_SUCCESS; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 22545255a3..7bb0f8ee83 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1943,7 +1943,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) { return TSDB_CODE_SUCCESS; } -int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 8758fdbc71..4847b50082 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -77,7 +77,7 @@ void sqlCheck(const char* sql, bool valid) { buf.len = 128; buf.buf = msg; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); @@ -122,7 +122,7 @@ TEST(testCase, validateAST_test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -184,7 +184,7 @@ TEST(testCase, function_Test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -234,7 +234,7 @@ TEST(testCase, function_Test2) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -284,7 +284,7 @@ TEST(testCase, function_Test3) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -333,7 +333,7 @@ TEST(testCase, function_Test4) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -385,7 +385,7 @@ TEST(testCase, function_Test5) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -474,7 +474,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -556,7 +556,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -622,7 +622,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -705,7 +705,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -756,7 +756,7 @@ TEST(testCase, show_user_Test) { SSqlInfo info1 = doGenerateAST(sql1); ASSERT_EQ(info1.valid, true); - SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; + SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); ASSERT_NE(output, nullptr); @@ -776,7 +776,7 @@ TEST(testCase, create_user_Test) { ASSERT_EQ(info1.valid, true); ASSERT_EQ(isDclSqlStatement(&info1), true); - SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"}; + SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc"}; SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); ASSERT_NE(output, nullptr); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 8d9fbadfad..a278e0053c 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -81,7 +81,7 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -122,7 +122,7 @@ TEST(testCase, planner_test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); ASSERT_EQ(ret, 0); diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index ee01a50148..7ed052e8cc 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -711,7 +711,7 @@ TEST(testCase, extractMeta_test) { char msg[128] = {0}; SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); From b55da7758e98d18044bb4eaa7d50ac1471bffb9a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:55:01 +0800 Subject: [PATCH 18/63] [td-11818]remove SParseBasicCtx --- source/libs/parser/test/insertParserTest.cpp | 4 ++-- source/libs/planner/test/phyPlanTests.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp index 9b007fb36d..86e8b1d7aa 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/insertParserTest.cpp @@ -43,8 +43,8 @@ protected: void bind(const char* sql) { reset(); - cxt_.ctx.acctId = atoi(acctId_.c_str()); - cxt_.ctx.db = (char*) db_.c_str(); + cxt_.acctId = atoi(acctId_.c_str()); + cxt_.db = (char*) db_.c_str(); strcpy(sqlBuf_, sql); cxt_.sqlLen = strlen(sql); sqlBuf_[cxt_.sqlLen] = '\0'; diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 2733a73a3f..29f6e48dc7 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -135,9 +135,9 @@ private: _sql = sql; memset(_msg, 0, _msgMaxLen); - pCxt->ctx.acctId = 1; - pCxt->ctx.db = _db.c_str(); - pCxt->ctx.requestId = 1; + pCxt->acctId = 1; + pCxt->db = _db.c_str(); + pCxt->requestId = 1; pCxt->pSql = _sql.c_str(); pCxt->sqlLen = _sql.length(); pCxt->pMsg = _msg; From 6c9ad20bafeae5b6e3b0f08155c37d95b6a3240e Mon Sep 17 00:00:00 2001 From: lihui Date: Wed, 12 Jan 2022 11:01:47 +0800 Subject: [PATCH 19/63] [add show tables] --- tests/test/c/create_table.c | 233 +++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 82 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index a5a55bbc91..aae4dc7074 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -30,6 +30,7 @@ int32_t createTable = 1; int32_t insertData = 0; int32_t batchNum = 100; int32_t numOfVgroups = 2; +int32_t showTablesFlag = 0; typedef struct { int64_t tableBeginIndex; @@ -45,88 +46,9 @@ typedef struct { pthread_t thread; } SThreadInfo; -void parseArgument(int32_t argc, char *argv[]); -void *threadFunc(void *param); -void createDbAndStb(); - -int32_t main(int32_t argc, char *argv[]) { - parseArgument(argc, argv); - createDbAndStb(); - - pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - - //int64_t numOfTablesPerThread = numOfTables / numOfThreads; - //numOfTables = numOfTablesPerThread * numOfThreads; - - - if (numOfThreads < 1) { - numOfThreads = 1; - } - - int64_t a = numOfTables / numOfThreads; - if (a < 1) { - numOfThreads = numOfTables; - a = 1; - } - - int64_t b = 0; - b = numOfTables % numOfThreads; - - int64_t tableFrom = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = tableFrom; - pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pInfo[i].tableEndIndex + 1; - pInfo[i].threadIndex = i; - pInfo[i].minDelay = INT64_MAX; - strcpy(pInfo[i].dbName, dbName); - strcpy(pInfo[i].stbName, stbName); - pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); - } - - taosMsleep(300); - for (int32_t i = 0; i < numOfThreads; i++) { - pthread_join(pInfo[i].thread, NULL); - } - - int64_t maxDelay = 0; - int64_t minDelay = INT64_MAX; - - float createTableSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - createTableSpeed += pInfo[i].createTableSpeed; - - if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; - if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; - } - - float insertDataSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - insertDataSpeed += pInfo[i].insertDataSpeed; - } - - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", - GREEN, - numOfTables, - createTableSpeed, - numOfThreads, - maxDelay, - minDelay, - NC); - - if (insertData) { - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, - numOfThreads, NC); - } - - pthread_attr_destroy(&thattr); - free(pInfo); -} +//void parseArgument(int32_t argc, char *argv[]); +//void *threadFunc(void *param); +//void createDbAndStb(); void createDbAndStb() { pPrint("start to create db and stable"); @@ -188,6 +110,62 @@ void printInsertProgress(SThreadInfo *pInfo, int64_t t) { totalTables, seconds, speed); } +static int64_t getResult(TAOS_RES *tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + int64_t numOfRows = 0; + do { + numOfRows++; + row = taos_fetch_row(tres); + } while (row != NULL); + + return numOfRows; +} + + +void showTables() { + pPrint("start to show tables"); + char qstr[32]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + sprintf(qstr, "use %s", dbName); + TAOS_RES *pRes = taos_query(con, qstr); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(1); + } + taos_free_result(pRes); + + sprintf(qstr, "show tables"); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + + int64_t totalTableNum = getResult(pRes); + taos_free_result(pRes); + + pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); + + taos_close(con); +} + + void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; char *qstr = malloc(2000 * 1000); @@ -298,6 +276,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); printf("%s%s\n", indent, "-b"); printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); exit(EXIT_SUCCESS); } @@ -325,6 +305,8 @@ void parseArgument(int32_t argc, char *argv[]) { insertData = atoi(argv[++i]); } else if (strcmp(argv[i], "-b") == 0) { batchNum = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + showTablesFlag = atoi(argv[++i]); } else { } } @@ -338,6 +320,93 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s createTable:%d %s", GREEN, createTable, NC); pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s batchNum:%d %s", GREEN, batchNum, NC); + pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s start create table performace test %s", GREEN, NC); } + +int32_t main(int32_t argc, char *argv[]) { + parseArgument(argc, argv); + + if (showTablesFlag) { + showTables(); + return 0; + } + + createDbAndStb(); + + pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + //int64_t numOfTablesPerThread = numOfTables / numOfThreads; + //numOfTables = numOfTablesPerThread * numOfThreads; + + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pInfo[i].tableEndIndex + 1; + pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stbName, stbName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int32_t i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + + float createTableSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; + } + + float insertDataSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + insertDataSpeed += pInfo[i].insertDataSpeed; + } + + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", + GREEN, + numOfTables, + createTableSpeed, + numOfThreads, + maxDelay, + minDelay, + NC); + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + numOfThreads, NC); + } + + pthread_attr_destroy(&thattr); + free(pInfo); +} + From 57281f6e8f544997fe73dc5475ec66f7211f1fd9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 11:26:23 +0800 Subject: [PATCH 20/63] [td-11818]refactor. --- source/client/inc/clientInt.h | 1 - source/client/src/clientImpl.c | 5 +++-- source/client/src/clientMain.c | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ebb8502d6c..39f84ffd86 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -129,7 +129,6 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; - uint64_t affectedRows; // todo remove it SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a614d2c3c1..ba08e67127 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -250,8 +250,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { } } - pRequest->affectedRows = res.numOfRows; - return res.code; + pRequest->body.resInfo.numOfRows = res.numOfRows; + pRequest->code = res.code; + return pRequest->code; } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1238976b97..ba2a21d7ea 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -265,7 +265,13 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } int taos_affected_rows(TAOS_RES *res) { - return ((SRequestObj*)res)->affectedRows; + if (res == NULL) { + return 0; + } + + SRequestObj* pRequest = (SRequestObj*) res; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + return pResInfo->numOfRows; } int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } From eadb5c27356837036e0508772327bac7bbf8d761 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 12:06:19 +0800 Subject: [PATCH 21/63] [td-11818] fix memory leak. --- source/client/test/clientTests.cpp | 120 ++++++++++++------------- source/libs/planner/src/logicPlan.c | 6 +- source/libs/planner/src/physicalPlan.c | 4 +- source/libs/scheduler/src/scheduler.c | 4 + 4 files changed, 69 insertions(+), 65 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d507c565df..d1093bb1a6 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -279,7 +279,7 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// + //TEST(testCase, create_table_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -292,26 +292,26 @@ TEST(testCase, connect_Test) { // // taos_close(pConn); //} -// -//TEST(testCase, create_ctable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, show_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -533,6 +533,7 @@ TEST(testCase, connect_Test) { // tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); // taos_close(pConn); //} + //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_EQ(pConn, nullptr); @@ -550,49 +551,48 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -//#endif -TEST(testCase, projection_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +//TEST(testCase, projection_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +//// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +//// taos_free_result(pRes); +//// return; +//// } +// +//// taos_free_result(pRes); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// +//// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); +// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +//// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +//// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); -// return; +// ASSERT_TRUE(false); // } - -// taos_free_result(pRes); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - -// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); - taos_free_result(pRes); // -// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); -// taos_free_result(pRes); +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } // -// pRes = taos_query(pConn, "insert into tu values(now, 1)"); // taos_free_result(pRes); - - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +// taos_close(pConn); +//} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 9a9b40473b..93f72bba95 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -38,10 +38,10 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { } static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { - SVnodeModifOpStmtInfo* pInsert = (SVnodeModifOpStmtInfo*)pNode; + SVnodeModifOpStmtInfo* pModifStmtInfo = (SVnodeModifOpStmtInfo*)pNode; *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); - SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); + SArray* blocks = taosArrayInit(taosArrayGetSize(pModifStmtInfo->pDataBlocks), POINTER_BYTES); SDataPayloadInfo* pPayload = calloc(1, sizeof(SDataPayloadInfo)); if (NULL == *pQueryPlan || NULL == blocks || NULL == pPayload) { @@ -49,7 +49,7 @@ static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode* } (*pQueryPlan)->info.type = QNODE_MODIFY; - taosArrayAddAll(blocks, pInsert->pDataBlocks); + taosArrayAddAll(blocks, pModifStmtInfo->pDataBlocks); if (pNode->type == TSDB_SQL_INSERT) { pPayload->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 0604ba3920..71e69c67d2 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -369,9 +369,9 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, - .pDag = validPointer(calloc(1, sizeof(SQueryDag))), + .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, - .nextId = {.queryId = requestId}, + .nextId = {.queryId = requestId}, }; *pDag = context.pDag; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 525d6ebe87..09e5291c85 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -50,6 +50,10 @@ void schFreeTask(SSchTask* pTask) { if (pTask->parents) { taosArrayDestroy(pTask->parents); } + + if (pTask->execAddrs) { + taosArrayDestroy(pTask->execAddrs); + } } From 97c66b5ee234cf6369b3ba00ccd3cd2b0527cdf2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 15:51:34 +0800 Subject: [PATCH 22/63] [td-11818] refactor. --- include/libs/parser/parser.h | 2 +- include/util/taoserror.h | 1 + source/client/src/clientImpl.c | 12 +- source/client/test/clientTests.cpp | 986 ++++++++++++----------- source/dnode/vnode/impl/src/vnodeQuery.c | 1 + source/dnode/vnode/tsdb/src/tsdbRead.c | 13 +- source/libs/parser/inc/astGenerator.h | 2 - source/libs/parser/src/parser.c | 6 +- source/util/src/terror.c | 3 +- 9 files changed, 518 insertions(+), 508 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 0cdbfdb07f..418c43fab9 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -44,7 +44,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery); +int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQueryNode); /** * Return true if it is a ddl/dcl sql statement diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 80241405a6..752007a61f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -305,6 +305,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied") #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing") #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) //"Invalid tsdb state") +#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) // "Table not exists") // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID") diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ba08e67127..16961799b9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -152,13 +152,13 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { SParseContext cxt = { .requestId = pRequest->requestId, - .acctId = pTscObj->acctId, - .db = getConnectionDB(pTscObj), + .acctId = pTscObj->acctId, + .db = getConnectionDB(pTscObj), + .pSql = pRequest->sqlstr, + .sqlLen = pRequest->sqlLen, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pTransporter = pTscObj->pTransporter, - .pSql = pRequest->sqlstr, - .sqlLen = pRequest->sqlLen, - .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE }; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d1093bb1a6..f6bda9abcf 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -56,242 +56,256 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -//TEST(testCase, create_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show users"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show databases"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, create_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create database abc1 vgroups 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_close(pConn); -//} -// -//TEST(testCase, create_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); -// if (taos_errno(pRes) != 0) { -// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// taos_close(pConn); -//} -// -//TEST(testCase, drop_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, use_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_close(pConn); -//} -// -//// TEST(testCase, drop_db_test) { -//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -//// assert(pConn != NULL); -//// -//// showDB(pConn); -//// -//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// showDB(pConn); -//// -//// pRes = taos_query(pConn, "create database abc1"); -//// if (taos_errno(pRes) != 0) { -//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// taos_close(pConn); -////} -// -//TEST(testCase, create_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -//TEST(testCase, create_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); -// taos_free_result(pRes); -// -// taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop database, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("failed to create database abc1, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); + if (taos_errno(pRes) != 0) { + printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); + if (taos_errno(pRes) != 0) { + printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + taos_close(pConn); +} + +TEST(testCase, drop_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + pRes = taos_query(pConn, "drop dnode 4"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, use_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + + TEST(testCase, drop_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + showDB(pConn); + + TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + showDB(pConn); + + pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("create to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); + taos_free_result(pRes); + + taos_close(pConn); +} TEST(testCase, create_ctable_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -312,193 +326,193 @@ TEST(testCase, create_ctable_Test) { taos_close(pConn); } -//TEST(testCase, show_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show stables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_vgroup_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show vgroups"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_multiple_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// taos_close(pConn); -// return; -// } -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// -// for (int32_t i = 0; i < 20; ++i) { -// char sql[512] = {0}; -// snprintf(sql, tListLen(sql), -// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, -// (i + 1) * 30, (i + 2) * 40); -// TAOS_RES* pres = taos_query(pConn, sql); -// if (taos_errno(pres) != 0) { -// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); -// } -// taos_free_result(pres); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, show_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show tables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in using db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop stable st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, generated_request_id_test) { -// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); -// -// for (int32_t i = 0; i < 50000; ++i) { -// uint64_t v = generateRequestId(); -// void* result = taosHashGet(phash, &v, sizeof(v)); -// if (result != nullptr) { -// printf("0x%lx, index:%d\n", v, i); -// } -// assert(result == nullptr); -// taosHashPut(phash, &v, sizeof(v), NULL, 0); -// } -// -// taosHashCleanup(phash); -//} +TEST(testCase, show_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show stables"); + if (taos_errno(pRes) != 0) { + printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_vgroup_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show vgroups"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + taos_close(pConn); + return; + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + + for (int32_t i = 0; i < 20; ++i) { + char sql[512] = {0}; + snprintf(sql, tListLen(sql), + "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, + (i + 1) * 30, (i + 2) * 40); + TAOS_RES* pres = taos_query(pConn, sql); + if (taos_errno(pres) != 0) { + printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); + } + taos_free_result(pres); + } + + taos_close(pConn); +} + +TEST(testCase, show_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "show tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("error in creating db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in using db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop stable st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, generated_request_id_test) { + SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + for (int32_t i = 0; i < 50000; ++i) { + uint64_t v = generateRequestId(); + void* result = taosHashGet(phash, &v, sizeof(v)); + if (result != nullptr) { + printf("0x%lx, index:%d\n", v, i); + } + assert(result == nullptr); + taosHashPut(phash, &v, sizeof(v), NULL, 0); + } + + taosHashCleanup(phash); +} // TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -534,65 +548,71 @@ TEST(testCase, create_ctable_Test) { // taos_close(pConn); //} -//TEST(testCase, insert_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_EQ(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, insert_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); -//TEST(testCase, projection_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -//// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -//// taos_free_result(pRes); -//// return; -//// } -// -//// taos_free_result(pRes); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// -//// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); -// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); -//// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "insert into tu values(now, 1)"); -//// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "select * from tu"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, projection_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 10000; ++i) { + char sql[512] = {0}; + sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } + + pRes = taos_query(pConn, "select * from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 909b233efb..2a9b4ed714 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -68,6 +68,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); if (pTbCfg == NULL) { + code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; } diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c index 906046ed9a..9b29d36c80 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/tsdb/src/tsdbRead.c @@ -713,13 +713,6 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe pCheckInfo->initBuf = true; int32_t order = pHandle->order; - // no data in buffer, abort -// if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) { -// return false; -// } -// -// assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); -// STbData** pMem = NULL; STbData** pIMem = NULL; @@ -787,8 +780,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, - pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } return true; @@ -2554,9 +2546,6 @@ static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { pTsdbReadHandle->activeIndex += 1; } - // no data in memtable or imemtable, decrease the memory reference. - // TODO !! -// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); return false; } diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 7f357a2bbd..c601c4e3e3 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -170,8 +170,6 @@ typedef struct SCreateDbInfo { int8_t update; int8_t cachelast; SArray *keep; -// int8_t dbType; -// int16_t partitions; } SCreateDbInfo; typedef struct SCreateFuncInfo { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index fea215238c..58e368aa0d 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -80,11 +80,11 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { return TSDB_CODE_SUCCESS; } -int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { +int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQueryNode) { if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) { - return parseInsertSql(pCxt, (SVnodeModifOpStmtInfo**)pQuery); + return parseInsertSql(pCxt, (SVnodeModifOpStmtInfo**)pQueryNode); } else { - return parseQuerySql(pCxt, pQuery); + return parseQuerySql(pCxt, pQueryNode); } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3ea564722b..29998a7d25 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -299,12 +299,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_CFG_FILE, "Invalid config file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TERM_FILE, "Invalid term file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, "Database memory is full") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, "Database is dropping") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_UPDATING, "Database is updating") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_UPDATING, "Database is updating") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_CLOSING, "Database is closing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") From c78f3e357f3f80acb3025e692eb6a70ad8455402 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jan 2022 09:26:42 +0000 Subject: [PATCH 23/63] refact vnode code --- source/dnode/vnode/CMakeLists.txt | 37 +- source/dnode/vnode/impl/test/CMakeLists.txt | 12 - .../dnode/vnode/impl/test/vBenchmarkTest.cpp | 2 - .../dnode/vnode/impl/test/vnodeApiTests.cpp | 285 ------- .../meta => source/dnode/vnode/inc}/meta.h | 0 .../vnode/tq => source/dnode/vnode/inc}/tq.h | 0 .../tsdb => source/dnode/vnode/inc}/tsdb.h | 0 .../vnode => source/dnode/vnode/inc}/vnode.h | 0 source/dnode/vnode/meta/src/metaSQLiteImpl.c | 212 ----- source/dnode/vnode/meta/test/CMakeLists.txt | 24 - source/dnode/vnode/meta/test/metaAPITest.cpp | 105 --- .../dnode/vnode/{meta => src}/inc/metaCache.h | 0 .../dnode/vnode/{meta => src}/inc/metaCfg.h | 0 source/dnode/vnode/{meta => src}/inc/metaDB.h | 0 .../dnode/vnode/{meta => src}/inc/metaDef.h | 0 .../dnode/vnode/{meta => src}/inc/metaIdx.h | 0 .../dnode/vnode/{meta => src}/inc/metaQuery.h | 0 .../dnode/vnode/{meta => src}/inc/metaTbCfg.h | 0 .../dnode/vnode/{meta => src}/inc/metaTbTag.h | 0 .../dnode/vnode/{meta => src}/inc/metaTbUid.h | 0 source/dnode/vnode/{tq => src}/inc/tqCommit.h | 0 source/dnode/vnode/{tq => src}/inc/tqInt.h | 0 .../dnode/vnode/{tq => src}/inc/tqMetaStore.h | 0 .../vnode/{tsdb => src}/inc/tsdbCommit.h | 0 .../vnode/{tsdb => src}/inc/tsdbCompact.h | 0 .../dnode/vnode/{tsdb => src}/inc/tsdbDef.h | 0 source/dnode/vnode/{tsdb => src}/inc/tsdbFS.h | 0 .../dnode/vnode/{tsdb => src}/inc/tsdbFile.h | 0 .../vnode/{tsdb => src}/inc/tsdbHealth.h | 0 .../dnode/vnode/{tsdb => src}/inc/tsdbLog.h | 0 .../vnode/{tsdb => src}/inc/tsdbMemTable.h | 0 .../vnode/{tsdb => src}/inc/tsdbMemory.h | 0 .../vnode/{tsdb => src}/inc/tsdbOptions.h | 0 .../vnode/{tsdb => src}/inc/tsdbReadImpl.h | 0 .../vnode/{tsdb => src}/inc/tsdbRowMergeBuf.h | 0 .../vnode/{impl => src}/inc/vnodeBufferPool.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeCfg.h | 0 .../vnode/{impl => src}/inc/vnodeCommit.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeDef.h | 2 +- .../dnode/vnode/{impl => src}/inc/vnodeFS.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeInt.h | 2 +- .../dnode/vnode/{impl => src}/inc/vnodeMAF.h | 0 .../{impl => src}/inc/vnodeMemAllocator.h | 0 .../vnode/{impl => src}/inc/vnodeQuery.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeRead.h | 0 .../vnode/{impl => src}/inc/vnodeRequest.h | 0 .../vnode/{impl => src}/inc/vnodeStateMgr.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeSync.h | 2 +- .../vnode/{impl => src}/inc/vnodeWrite.h | 0 .../dnode/vnode/{ => src}/meta/CMakeLists.txt | 0 .../{meta/src => src/meta}/metaBDBImpl.c | 0 .../vnode/{meta/src => src/meta}/metaCache.c | 0 .../vnode/{meta/src => src/meta}/metaCfg.c | 0 .../vnode/{meta/src => src/meta}/metaCommit.c | 0 .../vnode/{meta/src => src/meta}/metaIdx.c | 2 + .../vnode/{meta/src => src/meta}/metaMain.c | 0 .../vnode/{meta/src => src/meta}/metaQuery.c | 0 .../vnode/{meta/src => src/meta}/metaTable.c | 0 .../vnode/{meta/src => src/meta}/metaTbCfg.c | 0 .../vnode/{meta/src => src/meta}/metaTbTag.c | 0 .../vnode/{meta/src => src/meta}/metaTbUid.c | 0 .../dnode/vnode/{ => src}/tq/CMakeLists.txt | 0 source/dnode/vnode/{tq/src => src/tq}/tq.c | 0 .../dnode/vnode/{tq/src => src/tq}/tqCommit.c | 0 .../vnode/{tq/src => src/tq}/tqMetaStore.c | 0 .../dnode/vnode/{ => src}/tsdb/CMakeLists.txt | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbCommit.c | 0 .../{tsdb/src => src/tsdb}/tsdbCompact.c | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbFS.c | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbFile.c | 0 source/dnode/vnode/src/tsdb/tsdbHealth.c | 98 +++ .../vnode/{tsdb/src => src/tsdb}/tsdbMain.c | 0 .../{tsdb/src => src/tsdb}/tsdbMemTable.c | 0 .../{tsdb/src => src/tsdb}/tsdbOptions.c | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbRead.c | 2 +- .../{tsdb/src => src/tsdb}/tsdbReadImpl.c | 0 .../{tsdb/src => src/tsdb}/tsdbRowMergeBuf.c | 26 +- .../vnode/{tsdb/src => src/tsdb}/tsdbScan.c | 2 +- .../vnode/{tsdb/src => src/tsdb}/tsdbWrite.c | 0 .../vnode/{impl => src/vnd}/CMakeLists.txt | 0 .../{impl/src => src/vnd}/vnodeArenaMAImpl.c | 0 .../{impl/src => src/vnd}/vnodeBufferPool.c | 0 .../vnode/{impl/src => src/vnd}/vnodeCfg.c | 0 .../vnode/{impl/src => src/vnd}/vnodeCommit.c | 0 .../vnode/{impl/src => src/vnd}/vnodeFS.c | 0 .../vnode/{impl/src => src/vnd}/vnodeInt.c | 0 .../vnode/{impl/src => src/vnd}/vnodeMain.c | 0 .../vnode/{impl/src => src/vnd}/vnodeMgr.c | 0 .../vnode/{impl/src => src/vnd}/vnodeQuery.c | 0 .../vnode/{impl/src => src/vnd}/vnodeRead.c | 0 .../{impl/src => src/vnd}/vnodeRequest.c | 0 .../{impl/src => src/vnd}/vnodeStateMgr.c | 0 .../vnode/{impl/src => src/vnd}/vnodeSync.c | 0 .../vnode/{impl/src => src/vnd}/vnodeWrite.c | 0 .../dnode/vnode/{tq => }/test/CMakeLists.txt | 0 .../dnode/vnode/{tq => }/test/tqMetaTest.cpp | 0 .../vnode/{tq => }/test/tqSerializerTest.cpp | 0 source/dnode/vnode/tsdb/inc/tsdbint.h | 149 ---- source/dnode/vnode/tsdb/src/tsdbHealth.c | 98 --- source/dnode/vnode/tsdb/src/tsdbSync.c | 724 ------------------ source/dnode/vnode/tsdb/test/tsdbTests.cpp | 0 source/libs/executor/CMakeLists.txt | 2 +- 102 files changed, 152 insertions(+), 1634 deletions(-) delete mode 100644 source/dnode/vnode/impl/test/CMakeLists.txt delete mode 100644 source/dnode/vnode/impl/test/vBenchmarkTest.cpp delete mode 100644 source/dnode/vnode/impl/test/vnodeApiTests.cpp rename {include/dnode/vnode/meta => source/dnode/vnode/inc}/meta.h (100%) rename {include/dnode/vnode/tq => source/dnode/vnode/inc}/tq.h (100%) rename {include/dnode/vnode/tsdb => source/dnode/vnode/inc}/tsdb.h (100%) rename {include/dnode/vnode => source/dnode/vnode/inc}/vnode.h (100%) delete mode 100644 source/dnode/vnode/meta/src/metaSQLiteImpl.c delete mode 100644 source/dnode/vnode/meta/test/CMakeLists.txt delete mode 100644 source/dnode/vnode/meta/test/metaAPITest.cpp rename source/dnode/vnode/{meta => src}/inc/metaCache.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaCfg.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaDB.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaDef.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaIdx.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaQuery.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaTbCfg.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaTbTag.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaTbUid.h (100%) rename source/dnode/vnode/{tq => src}/inc/tqCommit.h (100%) rename source/dnode/vnode/{tq => src}/inc/tqInt.h (100%) rename source/dnode/vnode/{tq => src}/inc/tqMetaStore.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbCommit.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbCompact.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbDef.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbFS.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbFile.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbHealth.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbLog.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbMemTable.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbMemory.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbOptions.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbReadImpl.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbRowMergeBuf.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeBufferPool.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeCfg.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeCommit.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeDef.h (98%) rename source/dnode/vnode/{impl => src}/inc/vnodeFS.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeInt.h (98%) rename source/dnode/vnode/{impl => src}/inc/vnodeMAF.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeMemAllocator.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeQuery.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeRead.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeRequest.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeStateMgr.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeSync.h (97%) rename source/dnode/vnode/{impl => src}/inc/vnodeWrite.h (100%) rename source/dnode/vnode/{ => src}/meta/CMakeLists.txt (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaBDBImpl.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaCache.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaCfg.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaCommit.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaIdx.c (98%) rename source/dnode/vnode/{meta/src => src/meta}/metaMain.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaQuery.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTable.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTbCfg.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTbTag.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTbUid.c (100%) rename source/dnode/vnode/{ => src}/tq/CMakeLists.txt (100%) rename source/dnode/vnode/{tq/src => src/tq}/tq.c (100%) rename source/dnode/vnode/{tq/src => src/tq}/tqCommit.c (100%) rename source/dnode/vnode/{tq/src => src/tq}/tqMetaStore.c (100%) rename source/dnode/vnode/{ => src}/tsdb/CMakeLists.txt (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbCommit.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbCompact.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbFS.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbFile.c (100%) create mode 100644 source/dnode/vnode/src/tsdb/tsdbHealth.c rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbMain.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbMemTable.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbOptions.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbRead.c (99%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbReadImpl.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbRowMergeBuf.c (53%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbScan.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbWrite.c (100%) rename source/dnode/vnode/{impl => src/vnd}/CMakeLists.txt (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeArenaMAImpl.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeBufferPool.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeCfg.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeCommit.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeFS.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeInt.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeMain.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeMgr.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeQuery.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeRead.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeRequest.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeStateMgr.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeSync.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeWrite.c (100%) rename source/dnode/vnode/{tq => }/test/CMakeLists.txt (100%) rename source/dnode/vnode/{tq => }/test/tqMetaTest.cpp (100%) rename source/dnode/vnode/{tq => }/test/tqSerializerTest.cpp (100%) delete mode 100644 source/dnode/vnode/tsdb/inc/tsdbint.h delete mode 100644 source/dnode/vnode/tsdb/src/tsdbHealth.c delete mode 100644 source/dnode/vnode/tsdb/src/tsdbSync.c delete mode 100644 source/dnode/vnode/tsdb/test/tsdbTests.cpp diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a4a9cff002..9dc4bb1873 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,4 +1,33 @@ -add_subdirectory(meta) -add_subdirectory(tq) -add_subdirectory(tsdb) -add_subdirectory(impl) \ No newline at end of file +aux_source_directory(src/meta META_SRC) +aux_source_directory(src/tq TQ_SRC) +aux_source_directory(src/tsdb TSDB_SRC) +aux_source_directory(src/vnd VND_SRC) +list(APPEND + VNODE_SRC + ${META_SRC} + ${TQ_SRC} + ${TSDB_SRC} + ${VND_SRC} +) + +add_library(vnode STATIC ${VNODE_SRC}) +target_include_directories( + vnode + PUBLIC inc + PRIVATE src/inc +) +target_link_libraries( + vnode + PUBLIC os + PUBLIC util + PUBLIC common + PUBLIC transport + PUBLIC bdb + PUBLIC tfs + PUBLIC wal + PUBLIC qworker +) + +if(${BUILD_TEST}) + # add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt deleted file mode 100644 index e1226331e9..0000000000 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Vnode API test -add_executable(vnodeApiTests "") -target_sources(vnodeApiTests - PRIVATE - "vnodeApiTests.cpp" -) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) - -add_test( - NAME vnode_api_tests - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests - ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp deleted file mode 100644 index e218886231..0000000000 --- a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest -// https://github.com/google/benchmark \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp deleted file mode 100644 index 6ec58a1e9d..0000000000 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ /dev/null @@ -1,285 +0,0 @@ -/** - * @file vnodeApiTests.cpp - * @author hzcheng (hzcheng@taosdata.com) - * @brief VNODE module API tests - * @version 0.1 - * @date 2021-12-13 - * - * @copyright Copyright (c) 2021 - * - */ - -#include -#include - -#include "vnode.h" - -static STSchema *vtCreateBasicSchema() { - STSchemaBuilder sb; - STSchema * pSchema = NULL; - - tdInitTSchemaBuilder(&sb, 0); - - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - for (int i = 1; i < 10; i++) { - tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, i, 0); - } - - pSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static STSchema *vtCreateBasicTagSchema() { - STSchemaBuilder sb; - STSchema * pSchema = NULL; - - tdInitTSchemaBuilder(&sb, 0); - - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - for (int i = 10; i < 12; i++) { - tdAddColToSchema(&sb, TSDB_DATA_TYPE_BINARY, i, 20); - } - - pSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static SKVRow vtCreateBasicTag() { - SKVRowBuilder rb; - SKVRow pTag; - - tdInitKVRowBuilder(&rb); - - for (int i = 0; i < 2; i++) { - void *pVal = malloc(sizeof(VarDataLenT) + strlen("foo")); - varDataLen(pVal) = strlen("foo"); - memcpy(varDataVal(pVal), "foo", strlen("foo")); - - tdAddColToKVRow(&rb, i, TSDB_DATA_TYPE_BINARY, pVal); - free(pVal); - } - - pTag = tdGetKVRowFromBuilder(&rb); - tdDestroyKVRowBuilder(&rb); - - return pTag; -} - -static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { - SRpcMsg * pMsg; - STSchema *pSchema; - STSchema *pTagSchema; - int zs; - void * pBuf; - - pSchema = vtCreateBasicSchema(); - pTagSchema = vtCreateBasicTagSchema(); - - SVnodeReq vCreateSTbReq; - vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - zs = vnodeBuildReq(NULL, &vCreateSTbReq, TDMT_VND_CREATE_STB); - pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TDMT_VND_CREATE_STB; - pMsg->contLen = zs; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); - - pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, &vCreateSTbReq, TDMT_VND_CREATE_STB); - META_CLEAR_TB_CFG(&vCreateSTbReq); - - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - - *ppMsg = pMsg; -} - -static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { - SRpcMsg *pMsg; - int tz; - SKVRow pTag = vtCreateBasicTag(); - - SVnodeReq vCreateCTbReq; - vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - - tz = vnodeBuildReq(NULL, &vCreateCTbReq, TDMT_VND_CREATE_TABLE); - pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TDMT_VND_CREATE_TABLE; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateCTbReq, TDMT_VND_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateCTbReq); - free(pTag); - - *ppMsg = pMsg; -} - -static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { - // TODO -} - -static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - SRpcMsg * pMsg; - SSubmitMsg *pSubmitMsg; - SSubmitBlk *pSubmitBlk; - int tz = 1024; // TODO - - pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); - pMsg->msgType = TDMT_VND_SUBMIT; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - - // For submit msg header - pSubmitMsg = (SSubmitMsg *)(pMsg->pCont); - // pSubmitMsg->header.contLen = 0; - // pSubmitMsg->header.vgId = 0; - // pSubmitMsg->length = 0; - pSubmitMsg->numOfBlocks = 1; - - // For submit blk - pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks); - pSubmitBlk->uid = 0; - pSubmitBlk->tid = 0; - pSubmitBlk->padding = 0; - pSubmitBlk->sversion = 0; - pSubmitBlk->dataLen = 0; - pSubmitBlk->numOfRows = 0; - - // For row batch - - *ppMsg = pMsg; -} - -static void vtClearMsgBatch(SArray *pMsgArr) { - SRpcMsg *pMsg; - for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i); - free(pMsg); - } - - taosArrayClear(pMsgArr); -} - -static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) { - int rcode; - SRpcMsg *pReq; - SRpcMsg *pRsp; - - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); - - for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { - pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i); - rcode = vnodeApplyWMsg(pVnode, pReq, NULL); - GTEST_ASSERT_EQ(rcode, 0); - } -} - -TEST(vnodeApiTest, vnode_simple_create_table_test) { - tb_uid_t suid = 1638166374163; - SRpcMsg *pMsg; - SArray * pMsgArr = NULL; - SVnode * pVnode; - int rcode; - int ntables = 1000000; - int batch = 10; - char tbname[128]; - - pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg)); - - vnodeDestroy("vnode1"); - GTEST_ASSERT_GE(vnodeInit(2), 0); - - // CREATE AND OPEN A VNODE - pVnode = vnodeOpen("vnode1", NULL); - ASSERT_NE(pVnode, nullptr); - - // CREATE A SUPER TABLE - sprintf(tbname, "st"); - vtBuildCreateStbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // CREATE A LOT OF CHILD TABLES - for (int i = 0; i < ntables / batch; i++) { - // Build request batch - for (int j = 0; j < batch; j++) { - sprintf(tbname, "ct%d", i * batch + j + 1); - vtBuildCreateCtbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - } - - // Process request batch - vtProcessAndApplyReqs(pVnode, pMsgArr); - - // Clear request batch - vtClearMsgBatch(pMsgArr); - } - - // CLOSE THE VNODE - vnodeClose(pVnode); - vnodeCleanup(); - - taosArrayDestroy(pMsgArr); -} - -TEST(vnodeApiTest, vnode_simple_insert_test) { - const char *vname = "vnode2"; - char tbname[128]; - tb_uid_t suid = 1638166374163; - SRpcMsg * pMsg; - SArray * pMsgArr; - int rcode; - SVnode * pVnode; - int batch = 1; - int loop = 1000000; - - pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); - - vnodeDestroy(vname); - - GTEST_ASSERT_GE(vnodeInit(2), 0); - - // Open a vnode - pVnode = vnodeOpen(vname, NULL); - GTEST_ASSERT_NE(pVnode, nullptr); - - // 1. CREATE A SUPER TABLE - sprintf(tbname, "st"); - vtBuildCreateStbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // 2. CREATE A CHILD TABLE - sprintf(tbname, "t0"); - vtBuildCreateCtbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // 3. WRITE A LOT OF TIME-SERIES DATA - for (int j = 0; j < loop; j++) { - for (int i = 0; i < batch; i++) { - vtBuildSubmitReq(&pMsg); - taosArrayPush(pMsgArr, &pMsg); - } - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - } - - // Close the vnode - vnodeClose(pVnode); - vnodeCleanup(); - - taosArrayDestroy(pMsgArr); -} \ No newline at end of file diff --git a/include/dnode/vnode/meta/meta.h b/source/dnode/vnode/inc/meta.h similarity index 100% rename from include/dnode/vnode/meta/meta.h rename to source/dnode/vnode/inc/meta.h diff --git a/include/dnode/vnode/tq/tq.h b/source/dnode/vnode/inc/tq.h similarity index 100% rename from include/dnode/vnode/tq/tq.h rename to source/dnode/vnode/inc/tq.h diff --git a/include/dnode/vnode/tsdb/tsdb.h b/source/dnode/vnode/inc/tsdb.h similarity index 100% rename from include/dnode/vnode/tsdb/tsdb.h rename to source/dnode/vnode/inc/tsdb.h diff --git a/include/dnode/vnode/vnode.h b/source/dnode/vnode/inc/vnode.h similarity index 100% rename from include/dnode/vnode/vnode.h rename to source/dnode/vnode/inc/vnode.h diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c deleted file mode 100644 index fe9ef22fb3..0000000000 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "metaDef.h" -#include "sqlite3.h" - -struct SMetaDB { - sqlite3 *pDB; -}; - -int metaOpenDB(SMeta *pMeta) { - char dir[128]; - int rc; - char *err = NULL; - - pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); - if (pMeta->pDB == NULL) { - // TODO: handle error - return -1; - } - - sprintf(dir, "%s/meta.db", pMeta->path); - rc = sqlite3_open(dir, &(pMeta->pDB->pDB)); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to open meta.db\n"); - } - - // For all tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS tb (" - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_uid INTEGER NOT NULL UNIQUE " - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table tb since %s\n", err); - } - - // For super tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS stb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_schema BLOB NOT NULL," - " tag_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table stb since %s\n", err); - } - - // For normal tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS ntb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL," - " tb_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table ntb since %s\n", err); - } - - sqlite3_exec(pMeta->pDB->pDB, "BEGIN;", NULL, NULL, &err); - - tfree(err); - - return 0; -} - -void metaCloseDB(SMeta *pMeta) { - if (pMeta->pDB) { - sqlite3_exec(pMeta->pDB->pDB, "COMMIT;", NULL, NULL, NULL); - sqlite3_close(pMeta->pDB->pDB); - free(pMeta->pDB); - pMeta->pDB = NULL; - } - - // TODO -} - -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { - char sql[256]; - char * err = NULL; - int rc; - tb_uid_t uid; - sqlite3_stmt *stmt; - char buf[256]; - void * pBuf; - - switch (pTbCfg->type) { - case META_SUPER_TABLE: - uid = pTbCfg->stbCfg.suid; - sprintf(sql, - "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - ");" - "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - " (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256)," - " tag1 INTEGER);", - pTbCfg->name, uid, uid); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); - } - - sprintf(sql, "INSERT INTO stb VALUES (%" PRIu64 ", %s, ?, ?)", uid, pTbCfg->name); - sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); - - pBuf = buf; - tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); - sqlite3_bind_blob(stmt, 1, buf, POINTER_DISTANCE(pBuf, buf), NULL); - pBuf = buf; - tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); - sqlite3_bind_blob(stmt, 2, buf, POINTER_DISTANCE(pBuf, buf), NULL); - - sqlite3_step(stmt); - - sqlite3_finalize(stmt); - -#if 0 - sprintf(sql, - "INSERT INTO tb VALUES (?, ?);" - // "INSERT INTO stb VALUES (?, ?, ?, ?);" - // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - // " (" - // " tb_uid INTEGER NOT NULL UNIQUE," - // " tbname VARCHAR(256)," - // " tag1 INTEGER);" - , - uid); - rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); - if (rc != SQLITE_OK) { - return -1; - } - sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT); - sqlite3_bind_int64(stmt, 2, uid); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - - - // sqlite3_bind_int64(stmt, 3, uid); - // sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT); - // pBuf = buf; - // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); - // sqlite3_bind_blob(stmt, 5, buf, POINTER_DISTANCE(pBuf, buf), NULL); - // pBuf = buf; - // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); - // sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL); - - rc = sqliteVjj3_step(stmt); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB)); - } - sqlite3_finalize(stmt); -#endif - break; - case META_NORMAL_TABLE: - // uid = metaGenerateUid(pMeta); - // sprintf(sql, - // "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );", - // pTbCfg->name, uid, uid, pTbCfg->name, ); - - // rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - // if (rc != SQLITE_OK) { - // printf("failed to create normal table since %s\n", err); - // } - break; - case META_CHILD_TABLE: -#if 0 - uid = metaGenerateUid(pMeta); - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create child table since %s\n", err); - } -#endif - break; - default: - break; - } - - tfree(err); - - return 0; -} - -int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { - /* TODO */ - return 0; -} \ No newline at end of file diff --git a/source/dnode/vnode/meta/test/CMakeLists.txt b/source/dnode/vnode/meta/test/CMakeLists.txt deleted file mode 100644 index 625c07ad56..0000000000 --- a/source/dnode/vnode/meta/test/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# add_executable(metaTest "") -# target_sources(metaTest -# PRIVATE -# "../src/metaMain.c" -# "../src/metaUid.c" -# "metaTests.cpp" -# ) -# target_include_directories(metaTest -# PUBLIC -# "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" -# "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -# ) -# target_link_libraries(metaTest -# os -# util -# common -# gtest_main -# tkv -# ) -# enable_testing() -# add_test( -# NAME meta_test -# COMMAND metaTest -# ) diff --git a/source/dnode/vnode/meta/test/metaAPITest.cpp b/source/dnode/vnode/meta/test/metaAPITest.cpp deleted file mode 100644 index 0d79882018..0000000000 --- a/source/dnode/vnode/meta/test/metaAPITest.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#if 0 -#include -#include -#include - -#include "meta.h" - -static STSchema *metaGetSimpleSchema() { - STSchema * pSchema = NULL; - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, 0); - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 8); - tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, 1, 4); - - pSchema = tdGetSchemaFromBuilder(&sb); - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static SKVRow metaGetSimpleTags() { - SKVRowBuilder kvrb = {0}; - SKVRow row; - - tdInitKVRowBuilder(&kvrb); - int64_t ts = 1634287978000; - int32_t a = 10; - - tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_TIMESTAMP, (void *)(&ts)); - tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_INT, (void *)(&a)); - - row = tdGetKVRowFromBuilder(&kvrb); - - tdDestroyKVRowBuilder(&kvrb); - - return row; -} - -TEST(MetaTest, DISABLED_meta_create_1m_normal_tables_test) { - // Open Meta - SMeta *meta = metaOpen(NULL, NULL); - std::cout << "Meta is opened!" << std::endl; - - // Create 1000000 normal tables - META_TABLE_OPTS_DECLARE(tbOpts); - STSchema *pSchema = metaGetSimpleSchema(); - char tbname[128]; - - for (size_t i = 0; i < 1000000; i++) { - sprintf(tbname, "ntb%ld", i); - metaNormalTableOptsInit(&tbOpts, tbname, pSchema); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - } - - tdFreeSchema(pSchema); - - // Close Meta - metaClose(meta); - std::cout << "Meta is closed!" << std::endl; - - // Destroy Meta - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; -} - -TEST(MetaTest, meta_create_1m_child_tables_test) { - // Open Meta - SMeta *meta = metaOpen(NULL); - std::cout << "Meta is opened!" << std::endl; - - // Create a super tables - tb_uid_t uid = 477529885843758ul; - META_TABLE_OPTS_DECLARE(tbOpts); - STSchema *pSchema = metaGetSimpleSchema(); - STSchema *pTagSchema = metaGetSimpleSchema(); - - metaSuperTableOptsInit(&tbOpts, "st", uid, pSchema, pTagSchema); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - - // Create 1000000 child tables - char name[128]; - SKVRow row = metaGetSimpleTags(); - for (size_t i = 0; i < 1000000; i++) { - sprintf(name, "ctb%ld", i); - metaChildTableOptsInit(&tbOpts, name, uid, row); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - } - kvRowFree(row); - - // Close Meta - metaClose(meta); - std::cout << "Meta is closed!" << std::endl; - - // Destroy Meta - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaCache.h b/source/dnode/vnode/src/inc/metaCache.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaCache.h rename to source/dnode/vnode/src/inc/metaCache.h diff --git a/source/dnode/vnode/meta/inc/metaCfg.h b/source/dnode/vnode/src/inc/metaCfg.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaCfg.h rename to source/dnode/vnode/src/inc/metaCfg.h diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/src/inc/metaDB.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaDB.h rename to source/dnode/vnode/src/inc/metaDB.h diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaDef.h rename to source/dnode/vnode/src/inc/metaDef.h diff --git a/source/dnode/vnode/meta/inc/metaIdx.h b/source/dnode/vnode/src/inc/metaIdx.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaIdx.h rename to source/dnode/vnode/src/inc/metaIdx.h diff --git a/source/dnode/vnode/meta/inc/metaQuery.h b/source/dnode/vnode/src/inc/metaQuery.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaQuery.h rename to source/dnode/vnode/src/inc/metaQuery.h diff --git a/source/dnode/vnode/meta/inc/metaTbCfg.h b/source/dnode/vnode/src/inc/metaTbCfg.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbCfg.h rename to source/dnode/vnode/src/inc/metaTbCfg.h diff --git a/source/dnode/vnode/meta/inc/metaTbTag.h b/source/dnode/vnode/src/inc/metaTbTag.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbTag.h rename to source/dnode/vnode/src/inc/metaTbTag.h diff --git a/source/dnode/vnode/meta/inc/metaTbUid.h b/source/dnode/vnode/src/inc/metaTbUid.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbUid.h rename to source/dnode/vnode/src/inc/metaTbUid.h diff --git a/source/dnode/vnode/tq/inc/tqCommit.h b/source/dnode/vnode/src/inc/tqCommit.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqCommit.h rename to source/dnode/vnode/src/inc/tqCommit.h diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqInt.h rename to source/dnode/vnode/src/inc/tqInt.h diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqMetaStore.h rename to source/dnode/vnode/src/inc/tqMetaStore.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbCommit.h rename to source/dnode/vnode/src/inc/tsdbCommit.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbCompact.h b/source/dnode/vnode/src/inc/tsdbCompact.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbCompact.h rename to source/dnode/vnode/src/inc/tsdbCompact.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbDef.h rename to source/dnode/vnode/src/inc/tsdbDef.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbFS.h rename to source/dnode/vnode/src/inc/tsdbFS.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbFile.h rename to source/dnode/vnode/src/inc/tsdbFile.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbHealth.h b/source/dnode/vnode/src/inc/tsdbHealth.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbHealth.h rename to source/dnode/vnode/src/inc/tsdbHealth.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbLog.h rename to source/dnode/vnode/src/inc/tsdbLog.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/src/inc/tsdbMemTable.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbMemTable.h rename to source/dnode/vnode/src/inc/tsdbMemTable.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbMemory.h rename to source/dnode/vnode/src/inc/tsdbMemory.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbOptions.h b/source/dnode/vnode/src/inc/tsdbOptions.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbOptions.h rename to source/dnode/vnode/src/inc/tsdbOptions.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbReadImpl.h rename to source/dnode/vnode/src/inc/tsdbReadImpl.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h b/source/dnode/vnode/src/inc/tsdbRowMergeBuf.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h rename to source/dnode/vnode/src/inc/tsdbRowMergeBuf.h diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/src/inc/vnodeBufferPool.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeBufferPool.h rename to source/dnode/vnode/src/inc/vnodeBufferPool.h diff --git a/source/dnode/vnode/impl/inc/vnodeCfg.h b/source/dnode/vnode/src/inc/vnodeCfg.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeCfg.h rename to source/dnode/vnode/src/inc/vnodeCfg.h diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/src/inc/vnodeCommit.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeCommit.h rename to source/dnode/vnode/src/inc/vnodeCommit.h diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h similarity index 98% rename from source/dnode/vnode/impl/inc/vnodeDef.h rename to source/dnode/vnode/src/inc/vnodeDef.h index f9172dd351..1c534a8aeb 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -17,7 +17,7 @@ #define _TD_VNODE_DEF_H_ #include "mallocator.h" -#include "sync.h" +// #include "sync.h" #include "tcoding.h" #include "tlist.h" #include "tlockfree.h" diff --git a/source/dnode/vnode/impl/inc/vnodeFS.h b/source/dnode/vnode/src/inc/vnodeFS.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeFS.h rename to source/dnode/vnode/src/inc/vnodeFS.h diff --git a/source/dnode/vnode/impl/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h similarity index 98% rename from source/dnode/vnode/impl/inc/vnodeInt.h rename to source/dnode/vnode/src/inc/vnodeInt.h index 48977ff046..028798bc3e 100644 --- a/source/dnode/vnode/impl/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -19,7 +19,7 @@ #include "vnode.h" #include "meta.h" -#include "sync.h" +// #include "sync.h" #include "tlog.h" #include "tq.h" #include "tsdb.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMAF.h b/source/dnode/vnode/src/inc/vnodeMAF.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeMAF.h rename to source/dnode/vnode/src/inc/vnodeMAF.h diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/src/inc/vnodeMemAllocator.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeMemAllocator.h rename to source/dnode/vnode/src/inc/vnodeMemAllocator.h diff --git a/source/dnode/vnode/impl/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeQuery.h rename to source/dnode/vnode/src/inc/vnodeQuery.h diff --git a/source/dnode/vnode/impl/inc/vnodeRead.h b/source/dnode/vnode/src/inc/vnodeRead.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeRead.h rename to source/dnode/vnode/src/inc/vnodeRead.h diff --git a/source/dnode/vnode/impl/inc/vnodeRequest.h b/source/dnode/vnode/src/inc/vnodeRequest.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeRequest.h rename to source/dnode/vnode/src/inc/vnodeRequest.h diff --git a/source/dnode/vnode/impl/inc/vnodeStateMgr.h b/source/dnode/vnode/src/inc/vnodeStateMgr.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeStateMgr.h rename to source/dnode/vnode/src/inc/vnodeStateMgr.h diff --git a/source/dnode/vnode/impl/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h similarity index 97% rename from source/dnode/vnode/impl/inc/vnodeSync.h rename to source/dnode/vnode/src/inc/vnodeSync.h index a3eb004dfa..e82979551d 100644 --- a/source/dnode/vnode/impl/inc/vnodeSync.h +++ b/source/dnode/vnode/src/inc/vnodeSync.h @@ -16,7 +16,7 @@ #ifndef _TD_VNODE_SYNC_H_ #define _TD_VNODE_SYNC_H_ -#include "sync.h" +// #include "sync.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/impl/inc/vnodeWrite.h b/source/dnode/vnode/src/inc/vnodeWrite.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeWrite.h rename to source/dnode/vnode/src/inc/vnodeWrite.h diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/src/meta/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/meta/CMakeLists.txt rename to source/dnode/vnode/src/meta/CMakeLists.txt diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c similarity index 100% rename from source/dnode/vnode/meta/src/metaBDBImpl.c rename to source/dnode/vnode/src/meta/metaBDBImpl.c diff --git a/source/dnode/vnode/meta/src/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCache.c rename to source/dnode/vnode/src/meta/metaCache.c diff --git a/source/dnode/vnode/meta/src/metaCfg.c b/source/dnode/vnode/src/meta/metaCfg.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCfg.c rename to source/dnode/vnode/src/meta/metaCfg.c diff --git a/source/dnode/vnode/meta/src/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCommit.c rename to source/dnode/vnode/src/meta/metaCommit.c diff --git a/source/dnode/vnode/meta/src/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c similarity index 98% rename from source/dnode/vnode/meta/src/metaIdx.c rename to source/dnode/vnode/src/meta/metaIdx.c index 3da56fc394..d9abb4bb7b 100644 --- a/source/dnode/vnode/meta/src/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#ifdef USE_INVERTED_INDEX #include "index.h" +#endif #include "metaDef.h" struct SMetaIdx { diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c similarity index 100% rename from source/dnode/vnode/meta/src/metaMain.c rename to source/dnode/vnode/src/meta/metaMain.c diff --git a/source/dnode/vnode/meta/src/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c similarity index 100% rename from source/dnode/vnode/meta/src/metaQuery.c rename to source/dnode/vnode/src/meta/metaQuery.c diff --git a/source/dnode/vnode/meta/src/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTable.c rename to source/dnode/vnode/src/meta/metaTable.c diff --git a/source/dnode/vnode/meta/src/metaTbCfg.c b/source/dnode/vnode/src/meta/metaTbCfg.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbCfg.c rename to source/dnode/vnode/src/meta/metaTbCfg.c diff --git a/source/dnode/vnode/meta/src/metaTbTag.c b/source/dnode/vnode/src/meta/metaTbTag.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbTag.c rename to source/dnode/vnode/src/meta/metaTbTag.c diff --git a/source/dnode/vnode/meta/src/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbUid.c rename to source/dnode/vnode/src/meta/metaTbUid.c diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/src/tq/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tq/CMakeLists.txt rename to source/dnode/vnode/src/tq/CMakeLists.txt diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/src/tq/tq.c similarity index 100% rename from source/dnode/vnode/tq/src/tq.c rename to source/dnode/vnode/src/tq/tq.c diff --git a/source/dnode/vnode/tq/src/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c similarity index 100% rename from source/dnode/vnode/tq/src/tqCommit.c rename to source/dnode/vnode/src/tq/tqCommit.c diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c similarity index 100% rename from source/dnode/vnode/tq/src/tqMetaStore.c rename to source/dnode/vnode/src/tq/tqMetaStore.c diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/src/tsdb/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tsdb/CMakeLists.txt rename to source/dnode/vnode/src/tsdb/CMakeLists.txt diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbCommit.c rename to source/dnode/vnode/src/tsdb/tsdbCommit.c diff --git a/source/dnode/vnode/tsdb/src/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbCompact.c rename to source/dnode/vnode/src/tsdb/tsdbCompact.c diff --git a/source/dnode/vnode/tsdb/src/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbFS.c rename to source/dnode/vnode/src/tsdb/tsdbFS.c diff --git a/source/dnode/vnode/tsdb/src/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbFile.c rename to source/dnode/vnode/src/tsdb/tsdbFile.c diff --git a/source/dnode/vnode/src/tsdb/tsdbHealth.c b/source/dnode/vnode/src/tsdb/tsdbHealth.c new file mode 100644 index 0000000000..99c1b925b0 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbHealth.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// #include "os.h" +// #include "tmsg.h" +// #include "tarray.h" +// #include "query.h" +// #include "tglobal.h" +// #include "tlist.h" +// #include "tsdbint.h" +// #include "tsdbBuffer.h" +// #include "tsdbLog.h" +// #include "tsdbHealth.h" +// #include "ttimer.h" +// #include "tthread.h" + + +// // return malloc new block count +// int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { +// STsdbBufPool *pPool = pRepo->pPool; +// int32_t cnt = 0; + +// if(tsdbAllowNewBlock(pRepo)) { +// STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); +// if (pBufBlock) { +// if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { +// // append error +// tsdbFreeBufBlock(pBufBlock); +// } else { +// pPool->nElasticBlocks ++; +// cnt ++ ; +// } +// } +// } +// return cnt; +// } + +// // switch anther thread to run +// void* cbKillQueryFree(void* param) { +// STsdbRepo* pRepo = (STsdbRepo*)param; +// // vnode +// if(pRepo->appH.notifyStatus) { +// pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); +// } + +// // free +// if(pRepo->pthread){ +// void* p = pRepo->pthread; +// pRepo->pthread = NULL; +// free(p); +// } + +// return NULL; +// } + +// // return true do free , false do nothing +// bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { +// // check previous running +// if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { +// tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); +// return false; +// } +// // create new +// pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); +// if(pRepo->pthread == NULL) { +// tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); +// return false; +// } +// return true; +// } + +// bool tsdbAllowNewBlock(STsdbRepo* pRepo) { +// int32_t nMaxElastic = pRepo->config.totalBlocks/3; +// STsdbBufPool* pPool = pRepo->pPool; +// if(pPool->nElasticBlocks >= nMaxElastic) { +// tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); +// return false; +// } +// return true; +// } + +// bool tsdbNoProblem(STsdbRepo* pRepo) { +// if(listNEles(pRepo->pPool->bufBlockList) == 0) +// return false; +// return true; +// } \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbMain.c rename to source/dnode/vnode/src/tsdb/tsdbMain.c diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbMemTable.c rename to source/dnode/vnode/src/tsdb/tsdbMemTable.c diff --git a/source/dnode/vnode/tsdb/src/tsdbOptions.c b/source/dnode/vnode/src/tsdb/tsdbOptions.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbOptions.c rename to source/dnode/vnode/src/tsdb/tsdbOptions.c diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c similarity index 99% rename from source/dnode/vnode/tsdb/src/tsdbRead.c rename to source/dnode/vnode/src/tsdb/tsdbRead.c index 906046ed9a..a8296e8f8e 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -28,7 +28,7 @@ #include "taosdef.h" #include "tlosertree.h" -#include "tsdbint.h" +#include "tsdbDef.h" #include "tmsg.h" #define EXTRA_BYTES 2 diff --git a/source/dnode/vnode/tsdb/src/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbReadImpl.c rename to source/dnode/vnode/src/tsdb/tsdbReadImpl.c diff --git a/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c b/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c similarity index 53% rename from source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c rename to source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c index 5ce580f70f..1eebea22d4 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c +++ b/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c @@ -13,18 +13,18 @@ * along with this program. If not, see . */ -#include "tsdbRowMergeBuf.h" -#include "tdataformat.h" +// #include "tsdbRowMergeBuf.h" +// #include "tdataformat.h" -// row1 has higher priority -SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { - if(row2 == NULL) return row1; - if(row1 == NULL) return row2; - ASSERT(pSchema1->version == memRowVersion(row1)); - ASSERT(pSchema2->version == memRowVersion(row2)); +// // row1 has higher priority +// SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { +// if(row2 == NULL) return row1; +// if(row1 == NULL) return row2; +// ASSERT(pSchema1->version == memRowVersion(row1)); +// ASSERT(pSchema2->version == memRowVersion(row2)); - if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { - return NULL; - } - return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); -} +// if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { +// return NULL; +// } +// return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); +// } diff --git a/source/dnode/vnode/tsdb/src/tsdbScan.c b/source/dnode/vnode/src/tsdb/tsdbScan.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbScan.c rename to source/dnode/vnode/src/tsdb/tsdbScan.c index 382f7b11ae..c0e468e640 100644 --- a/source/dnode/vnode/tsdb/src/tsdbScan.c +++ b/source/dnode/vnode/src/tsdb/tsdbScan.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "tsdbint.h" #if 0 +#include "tsdbint.h" #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } diff --git a/source/dnode/vnode/tsdb/src/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbWrite.c rename to source/dnode/vnode/src/tsdb/tsdbWrite.c diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/src/vnd/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/impl/CMakeLists.txt rename to source/dnode/vnode/src/vnd/CMakeLists.txt diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeArenaMAImpl.c rename to source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeBufferPool.c rename to source/dnode/vnode/src/vnd/vnodeBufferPool.c diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeCfg.c rename to source/dnode/vnode/src/vnd/vnodeCfg.c diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeCommit.c rename to source/dnode/vnode/src/vnd/vnodeCommit.c diff --git a/source/dnode/vnode/impl/src/vnodeFS.c b/source/dnode/vnode/src/vnd/vnodeFS.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeFS.c rename to source/dnode/vnode/src/vnd/vnodeFS.c diff --git a/source/dnode/vnode/impl/src/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeInt.c rename to source/dnode/vnode/src/vnd/vnodeInt.c diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeMain.c rename to source/dnode/vnode/src/vnd/vnodeMain.c diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeMgr.c rename to source/dnode/vnode/src/vnd/vnodeMgr.c diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeQuery.c rename to source/dnode/vnode/src/vnd/vnodeQuery.c diff --git a/source/dnode/vnode/impl/src/vnodeRead.c b/source/dnode/vnode/src/vnd/vnodeRead.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeRead.c rename to source/dnode/vnode/src/vnd/vnodeRead.c diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/src/vnd/vnodeRequest.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeRequest.c rename to source/dnode/vnode/src/vnd/vnodeRequest.c diff --git a/source/dnode/vnode/impl/src/vnodeStateMgr.c b/source/dnode/vnode/src/vnd/vnodeStateMgr.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeStateMgr.c rename to source/dnode/vnode/src/vnd/vnodeStateMgr.c diff --git a/source/dnode/vnode/impl/src/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeSync.c rename to source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeWrite.c rename to source/dnode/vnode/src/vnd/vnodeWrite.c diff --git a/source/dnode/vnode/tq/test/CMakeLists.txt b/source/dnode/vnode/test/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tq/test/CMakeLists.txt rename to source/dnode/vnode/test/CMakeLists.txt diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp similarity index 100% rename from source/dnode/vnode/tq/test/tqMetaTest.cpp rename to source/dnode/vnode/test/tqMetaTest.cpp diff --git a/source/dnode/vnode/tq/test/tqSerializerTest.cpp b/source/dnode/vnode/test/tqSerializerTest.cpp similarity index 100% rename from source/dnode/vnode/tq/test/tqSerializerTest.cpp rename to source/dnode/vnode/test/tqSerializerTest.cpp diff --git a/source/dnode/vnode/tsdb/inc/tsdbint.h b/source/dnode/vnode/tsdb/inc/tsdbint.h deleted file mode 100644 index bdd1bd6f71..0000000000 --- a/source/dnode/vnode/tsdb/inc/tsdbint.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_INT_H_ -#define _TD_TSDB_INT_H_ - -#if 0 -// // TODO: remove the include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include - -#include "hash.h" -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tarray.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tcompression.h" -#include "tdataformat.h" -#include "tfs.h" -#include "tlist.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tskiplist.h" -#include "tsocket.h" - -#include "tsdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Log -#include "tsdbLog.h" -// Meta -#include "tsdbMeta.h" -// Buffer -#include "tsdbBuffer.h" -// MemTable -#include "tsdbMemTable.h" -// File -#include "tsdbFile.h" -// FS -#include "tsdbFS.h" -// ReadImpl -#include "tsdbReadImpl.h" -// Commit -#include "tsdbCommit.h" -// Compact -#include "tsdbCompact.h" -// Commit Queue -#include "tsdbCommitQueue.h" - -#include "tsdbRowMergeBuf.h" -// Main definitions -struct STsdbRepo { - uint8_t state; - - STsdbCfg config; - - STsdbCfg save_config; // save apply config - bool config_changed; // config changed flag - pthread_mutex_t save_mutex; // protect save config - - uint8_t hasCachedLastColumn; - - STsdbAppH appH; - STsdbStat stat; - STsdbMeta* tsdbMeta; - STsdbBufPool* pPool; - SMemTable* mem; - SMemTable* imem; - STsdbFS* fs; - SRtn rtn; - tsem_t readyToCommit; - pthread_mutex_t mutex; - bool repoLocked; - int32_t code; // Commit code - - SMergeBuf mergeBuf; //used when update=2 - int8_t compactState; // compact state: inCompact/noCompact/waitingCompact? - pthread_t* pthread; -}; - -#define REPO_ID(r) (r)->config.tsdbId -#define REPO_CFG(r) (&((r)->config)) -#define REPO_FS(r) ((r)->fs) -#define IS_REPO_LOCKED(r) (r)->repoLocked -#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) - -int tsdbLockRepo(STsdbRepo* pRepo); -int tsdbUnlockRepo(STsdbRepo* pRepo); -STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); -int tsdbCheckCommit(STsdbRepo* pRepo); -int tsdbRestoreInfo(STsdbRepo* pRepo); -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg); -void tsdbGetRootDir(int repoid, char dirName[]); -void tsdbGetDataDir(int repoid, char dirName[]); - -static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { - ASSERT(pRepo != NULL); - if (pRepo->mem == NULL) return NULL; - - SListNode* pNode = listTail(pRepo->mem->bufBlockList); - if (pNode == NULL) return NULL; - - STsdbBufBlock* pBufBlock = NULL; - tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); - - return pBufBlock; -} - -static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { - ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); - int maxTables = TSDB_INIT_NTABLES; - while (true) { - maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables) break; - maxTables *= 2; - } - - return maxTables + 1; -} - -#ifdef __cplusplus -} -#endif - -#endif -#endif /* _TD_TSDB_INT_H_ */ diff --git a/source/dnode/vnode/tsdb/src/tsdbHealth.c b/source/dnode/vnode/tsdb/src/tsdbHealth.c deleted file mode 100644 index 4205f3e90f..0000000000 --- a/source/dnode/vnode/tsdb/src/tsdbHealth.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tarray.h" -#include "query.h" -#include "tglobal.h" -#include "tlist.h" -#include "tsdbint.h" -#include "tsdbBuffer.h" -#include "tsdbLog.h" -#include "tsdbHealth.h" -#include "ttimer.h" -#include "tthread.h" - - -// return malloc new block count -int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { - STsdbBufPool *pPool = pRepo->pPool; - int32_t cnt = 0; - - if(tsdbAllowNewBlock(pRepo)) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock) { - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - // append error - tsdbFreeBufBlock(pBufBlock); - } else { - pPool->nElasticBlocks ++; - cnt ++ ; - } - } - } - return cnt; -} - -// switch anther thread to run -void* cbKillQueryFree(void* param) { - STsdbRepo* pRepo = (STsdbRepo*)param; - // vnode - if(pRepo->appH.notifyStatus) { - pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); - } - - // free - if(pRepo->pthread){ - void* p = pRepo->pthread; - pRepo->pthread = NULL; - free(p); - } - - return NULL; -} - -// return true do free , false do nothing -bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { - // check previous running - if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { - tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); - return false; - } - // create new - pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); - if(pRepo->pthread == NULL) { - tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); - return false; - } - return true; -} - -bool tsdbAllowNewBlock(STsdbRepo* pRepo) { - int32_t nMaxElastic = pRepo->config.totalBlocks/3; - STsdbBufPool* pPool = pRepo->pPool; - if(pPool->nElasticBlocks >= nMaxElastic) { - tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); - return false; - } - return true; -} - -bool tsdbNoProblem(STsdbRepo* pRepo) { - if(listNEles(pRepo->pPool->bufBlockList) == 0) - return false; - return true; -} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbSync.c b/source/dnode/vnode/tsdb/src/tsdbSync.c deleted file mode 100644 index edcb84d091..0000000000 --- a/source/dnode/vnode/tsdb/src/tsdbSync.c +++ /dev/null @@ -1,724 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tsdbint.h" - -// Sync handle -typedef struct { - STsdbRepo *pRepo; - SRtn rtn; - SOCKET socketFd; - void * pBuf; - bool mfChanged; - SMFile * pmf; - SMFile mf; - SDFileSet df; - SDFileSet *pdf; -} SSyncH; - -#define SYNC_BUFFER(sh) ((sh)->pBuf) - -static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd); -static void tsdbDestroySyncH(SSyncH *pSyncH); -static int32_t tsdbSyncSendMeta(SSyncH *pSynch); -static int32_t tsdbSyncRecvMeta(SSyncH *pSynch); -static int32_t tsdbSendMetaInfo(SSyncH *pSynch); -static int32_t tsdbRecvMetaInfo(SSyncH *pSynch); -static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend); -static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend); -static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch); -static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch); -static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2); -static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet); -static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet); -static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch); -static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged); - -int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) { - STsdbRepo *pRepo = (STsdbRepo *)tsdb; - SSyncH synch = {0}; - - tsdbInitSyncH(&synch, pRepo, socketFd); - // Disable TSDB commit - tsem_wait(&(pRepo->readyToCommit)); - - if (tsdbSyncSendMeta(&synch) < 0) { - tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbSyncSendDFileSetArray(&synch) < 0) { - tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - // Enable TSDB commit - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return 0; - -_err: - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return -1; -} - -int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { - STsdbRepo *pRepo = (STsdbRepo *)tsdb; - SSyncH synch = {0}; - - pRepo->state = TSDB_STATE_OK; - - tsdbInitSyncH(&synch, pRepo, socketFd); - tsem_wait(&(pRepo->readyToCommit)); - tsdbStartFSTxn(pRepo, 0, 0); - - if (tsdbSyncRecvMeta(&synch) < 0) { - tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbSyncRecvDFileSetArray(&synch) < 0) { - tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - tsdbEndFSTxn(pRepo); - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - - // Reload file change - tsdbReload(pRepo, synch.mfChanged); - - return 0; - -_err: - tsdbEndFSTxnWithError(REPO_FS(pRepo)); - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return -1; -} - -static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) { - pSyncH->pRepo = pRepo; - pSyncH->socketFd = socketFd; - tsdbGetRtnSnap(pRepo, &(pSyncH->rtn)); -} - -static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); } - -static int32_t tsdbSyncSendMeta(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - bool toSendMeta = false; - SMFile mf; - - // Send meta info to remote - tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo)); - if (tsdbSendMetaInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (pRepo->fs->cstatus->pmf == NULL) { - // No meta file, not need to wait to retrieve meta file - tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo)); - return 0; - } - - if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) { - tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (toSendMeta) { - tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf); - if (tsdbOpenMFile(&mf, O_RDONLY) < 0) { - tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - int64_t writeLen = mf.info.size; - tsdbInfo("vgId:%d, metafile:%s will be sent, size:%" PRId64, REPO_ID(pRepo), mf.f.aname, writeLen); - - int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send metafile since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - tstrerror(terrno), ret, writeLen); - tsdbCloseMFile(&mf); - return -1; - } - - tsdbCloseMFile(&mf); - tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo)); - } else { - tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo)); - } - - return 0; -} - -static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - SMFile * pLMFile = pRepo->fs->cstatus->pmf; - - // Recv meta info from remote - if (tsdbRecvMetaInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - // No meta file, do nothing (rm local meta file) - if (pSynch->pmf == NULL) { - if (pLMFile == NULL) { - pSynch->mfChanged = false; - } else { - pSynch->mfChanged = true; - } - tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo)); - return 0; - } - - if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size || - pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) { - // Local has no meta file or has a different meta file, need to copy from remote - pSynch->mfChanged = true; - - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo)); - - // Recv from remote - SMFile mf; - SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - if (tsdbCreateMFile(&mf, false) < 0) { - tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname); - - int64_t readLen = pSynch->pmf->info.size; - int64_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metafile since %s, ret:%" PRId64 " readLen:%" PRId64, REPO_ID(pRepo), - tstrerror(terrno), ret, readLen); - tsdbCloseMFile(&mf); - tsdbRemoveMFile(&mf); - return -1; - } - - tsdbInfo("vgId:%d, metafile is received, size:%" PRId64, REPO_ID(pRepo), readLen); - - mf.info = pSynch->pmf->info; - tsdbCloseMFile(&mf); - tsdbUpdateMFile(REPO_FS(pRepo), &mf); - } else { - pSynch->mfChanged = false; - tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo)); - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - tsdbUpdateMFile(REPO_FS(pRepo), pLMFile); - } - - return 0; -} - -static int32_t tsdbSendMetaInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - SMFile * pMFile = pRepo->fs->cstatus->pmf; - - if (pMFile) { - tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM); - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { - tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - void *ptr = SYNC_BUFFER(pSynch); - taosEncodeFixedU32(&ptr, tlen); - void *tptr = ptr; - if (pMFile) { - tsdbEncodeSMFileEx(&ptr, pMFile); - taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); - } - - int32_t writeLen = tlen + sizeof(uint32_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, - writeLen); - return -1; - } - - tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen); - return 0; -} - -static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - char buf[64] = {0}; - - int32_t readLen = sizeof(uint32_t); - int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); - return -1; - } - - taosDecodeFixedU32(buf, &tlen); - - tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen); - if (tlen == 0) { - pSynch->pmf = NULL; - return 0; - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { - tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); - if (ret != tlen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen); - return -1; - } - - tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen); - if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { - terrno = TSDB_CODE_TDB_MESSED_MSG; - tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - pSynch->pmf = &(pSynch->mf); - tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf); - - return 0; -} - -static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) { - STsdbRepo *pRepo = pSynch->pRepo; - uint8_t decision = toSend; - - int32_t writeLen = sizeof(uint8_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); - return -1; - } - - return 0; -} - -static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) { - STsdbRepo *pRepo = pSynch->pRepo; - uint8_t decision = 0; - - int32_t readLen = sizeof(uint8_t); - int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); - return -1; - } - - *toSend = decision; - return 0; -} - -static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - STsdbFS * pfs = REPO_FS(pRepo); - SFSIter fsiter; - SDFileSet *pSet; - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - - do { - pSet = tsdbFSIterNext(&fsiter); - if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) { - tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, - tstrerror(terrno)); - return -1; - } - - // No more file set to send, jut break - if (pSet == NULL) { - tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo)); - break; - } - } while (true); - - return 0; -} - -static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - STsdbFS * pfs = REPO_FS(pRepo); - SFSIter fsiter; - SDFileSet *pLSet; // Local file set - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - - pLSet = tsdbFSIterNext(&fsiter); - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - while (true) { - if (pLSet == NULL && pSynch->pdf == NULL) { - tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo)); - break; - } else { - tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1, - pSynch->pdf != NULL ? pSynch->pdf->fid : -1); - } - - if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) { - // remote not has pLSet->fid set, just remove local (do nothing to remote the fset) - tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid, - pSynch->pdf != NULL ? pSynch->pdf->fid : -1); - pLSet = tsdbFSIterNext(&fsiter); - } else { - if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) && - tsdbFSetIsOk(pLSet)) { - // Just keep local files and notify remote not to send - tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid); - - if (tsdbUpdateDFileSet(pfs, pLSet) < 0) { - tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } else { - // Need to copy from remote - int fidLevel = tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)); - if (fidLevel < 0) { // expired fileset - tsdbInfo("vgId:%d, fileset:%d will be skipped as expired", REPO_ID(pRepo), pSynch->pdf->fid); - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - // Move forward - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - if (pLSet) { - pLSet = tsdbFSIterNext(&fsiter); - } - // Next loop - continue; - } else { - tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); - // Notify remote to send there file here - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - // Create local files and copy from remote - SDiskID did; - SDFileSet fset; - - tfsAllocDisk(fidLevel, &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs)); - - // Create new FSET - if (tsdbCreateDFileSet(&fset, false) < 0) { - tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); // local file - SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype); // remote file - - tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo), - pDFile->f.aname, pDFile->info.size, pRDFile->info.size); - - int64_t writeLen = pRDFile->info.size; - int64_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - pDFile->f.aname, tstrerror(terrno), ret, writeLen); - tsdbCloseDFileSet(&fset); - tsdbRemoveDFileSet(&fset); - return -1; - } - - // Update new file info - pDFile->info = pRDFile->info; - tsdbInfo("vgId:%d, file:%s is received, size:%" PRId64, REPO_ID(pRepo), pDFile->f.aname, writeLen); - } - - tsdbCloseDFileSet(&fset); - if (tsdbUpdateDFileSet(pfs, &fset) < 0) { - tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid); - } - - // Move forward - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (pLSet) { - pLSet = tsdbFSIterNext(&fsiter); - } - } - -#if 0 - if (pLSet == NULL) { - // Copy from remote >>>>>>>>>>> - } else { - if (pSynch->pdf == NULL) { - // Remove local file, just ignore ++++++++++++++ - pLSet = tsdbFSIterNext(&fsiter); - } else { - if (pLSet->fid < pSynch->pdf->fid) { - // Remove local file, just ignore ++++++++++++ - pLSet = tsdbFSIterNext(&fsiter); - } else if (pLSet->fid > pSynch->pdf->fid){ - // Copy from remote >>>>>>>>>>>>>> - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - // TODO - return -1; - } - } else { - if (true/*TODO: is same fset*/) { - // No need to copy --------------------- - } else { - // copy from remote >>>>>>>>>>>>>. - } - } - } - } -#endif - } - - return 0; -} - -static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); - SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); - - if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) { - return false; - } - } - - return true; -} - -static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) { - STsdbRepo *pRepo = pSynch->pRepo; - bool toSend = false; - - // skip expired fileset - if (pSet && tsdbGetFidLevel(pSet->fid, &(pSynch->rtn)) < 0) { - tsdbInfo("vgId:%d, don't sync send since fileset:%d smaller than minFid:%d", REPO_ID(pRepo), pSet->fid, - pSynch->rtn.minFid); - return 0; - } - - if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) { - tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno)); - return -1; - } - - // No file any more, no need to send file, just return - if (pSet == NULL) { - return 0; - } - - if (tsdbRecvDecision(pSynch, &toSend) < 0) { - tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid, - tstrerror(terrno)); - return -1; - } - - if (toSend) { - tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid); - - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype); - - if (tsdbOpenDFile(&df, O_RDONLY) < 0) { - tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno)); - return -1; - } - - int64_t writeLen = df.info.size; - tsdbInfo("vgId:%d, file:%s will be sent, size:%" PRId64, REPO_ID(pRepo), df.f.aname, writeLen); - - int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - df.f.aname, tstrerror(terrno), ret, writeLen); - tsdbCloseDFile(&df); - return -1; - } - - tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname); - tsdbCloseDFile(&df); - } - - tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid); - } else { - tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid); - } - - return 0; -} - -static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - - if (pSet) { - tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM); - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { - tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - void *ptr = SYNC_BUFFER(pSynch); - taosEncodeFixedU32(&ptr, tlen); - void *tptr = ptr; - if (pSet) { - tsdbEncodeDFileSetEx(&ptr, pSet); - taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); - } - - int32_t writeLen = tlen + sizeof(uint32_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); - return -1; - } - - return 0; -} - -static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen; - char buf[64] = {0}; - - int32_t readLen = sizeof(uint32_t); - int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - taosDecodeFixedU32(buf, &tlen); - - tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen); - if (tlen == 0) { - pSynch->pdf = NULL; - return 0; - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { - tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); - if (ret != tlen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { - terrno = TSDB_CODE_TDB_MESSED_MSG; - tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - pSynch->pdf = &(pSynch->df); - tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf); - - return 0; -} - -static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) { - // TODO: may need to stop and restart stream - // if (isMfChanged) { - tsdbCloseMeta(pRepo); - tsdbFreeMeta(pRepo->tsdbMeta); - pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo)); - tsdbOpenMeta(pRepo); - tsdbLoadMetaCache(pRepo, true); - // } - - tsdbUnRefMemTable(pRepo, pRepo->mem); - tsdbUnRefMemTable(pRepo, pRepo->imem); - pRepo->mem = NULL; - pRepo->imem = NULL; - - if (tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - return 0; -} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/test/tsdbTests.cpp b/source/dnode/vnode/tsdb/test/tsdbTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 04b5fab4bf..9b53cc1fbb 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom tsdb + PRIVATE os util common function parser planner qcom vnode ) target_include_directories( From 5cf4edacbb6da2a5b554396f5fc7ccee6b80cd47 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 12 Jan 2022 18:40:22 +0800 Subject: [PATCH 24/63] feature/qnode --- include/common/tmsg.h | 7 + include/common/tmsgdef.h | 2 + include/libs/qworker/qworker.h | 19 +- source/dnode/vnode/impl/inc/vnodeQuery.h | 1 + source/dnode/vnode/impl/src/vnodeQuery.c | 17 +- source/libs/executor/src/executorMain.c | 6 +- source/libs/qworker/inc/qworkerInt.h | 76 +++-- source/libs/qworker/src/qworker.c | 393 ++++++++++++++++------ source/libs/qworker/test/qworkerTests.cpp | 13 +- 9 files changed, 389 insertions(+), 145 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2aaa2168cc..339db43374 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -993,6 +993,13 @@ typedef struct { uint64_t taskId; } SSinkDataReq; +typedef struct { + SMsgHead header; + uint64_t sId; + uint64_t queryId; + uint64_t taskId; +} SQueryContinueReq; + typedef struct { SMsgHead header; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 592672b32b..2f4bae2fa0 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -170,6 +170,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) + TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 9897467230..08b5fb98e7 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -22,9 +22,18 @@ extern "C" { #include "trpc.h" + +enum { + NODE_TYPE_VNODE = 1, + NODE_TYPE_QNODE, + NODE_TYPE_SNODE, +}; + + + typedef struct SQWorkerCfg { uint32_t maxSchedulerNum; - uint32_t maxResCacheNum; + uint32_t maxTaskNum; uint32_t maxSchTaskNum; } SQWorkerCfg; @@ -39,11 +48,17 @@ typedef struct { uint64_t numOfErrors; } SQWorkerStat; +typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); -int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt); + +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); +int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + +int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/impl/inc/vnodeQuery.h b/source/dnode/vnode/impl/inc/vnodeQuery.h index d43f5b1cf1..b51aafb313 100644 --- a/source/dnode/vnode/impl/inc/vnodeQuery.h +++ b/source/dnode/vnode/impl/inc/vnodeQuery.h @@ -22,6 +22,7 @@ extern "C" { #include "vnodeInt.h" #include "qworker.h" + typedef struct SQWorkerMgmt SQHandle; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 909b233efb..52b20f8411 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -19,11 +19,22 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } +int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vTrace("query message is processed"); - return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + vTrace("query message is processing"); + + switch (pMsg->msgType) { + case TDMT_VND_QUERY: + return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_QUERY_CONTINUE: + return qWorkerProcessQueryContinueMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_SCHEDULE_DATA_SINK: + return qWorkerProcessDataSinkMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + default: + vError("unknown msg type:%d in query queue", pMsg->msgType); + return TSDB_CODE_VND_APP_ERROR; + } } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 50f69fb567..daeefba253 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -178,8 +178,10 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = 0; - *handle = pTaskInfo->dsHandle; - + if (handle) { + *handle = pTaskInfo->dsHandle; + } + while(1) { st = taosGetTimestampUs(); SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 7883079fbe..b5ec50cc04 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -23,7 +23,7 @@ extern "C" { #include "tlockfree.h" #define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 -#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000 +#define QWORKER_DEFAULT_TASK_NUMBER 10000 #define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 enum { @@ -57,7 +57,6 @@ enum { QW_ADD_ACQUIRE, }; - typedef struct SQWTaskStatus { SRWLatch lock; int32_t code; @@ -67,12 +66,14 @@ typedef struct SQWTaskStatus { bool drop; } SQWTaskStatus; -typedef struct SQWorkerTaskHandlesCache { +typedef struct SQWTaskCtx { SRWLatch lock; + int8_t sinkScheduled; + int8_t queryScheduled; bool needRsp; qTaskInfo_t taskHandle; DataSinkHandle sinkHandle; -} SQWorkerTaskHandlesCache; +} SQWTaskCtx; typedef struct SQWSchStatus { int32_t lastAccessTs; // timestamp in second @@ -82,11 +83,15 @@ typedef struct SQWSchStatus { // Qnode/Vnode level task management typedef struct SQWorkerMgmt { - SQWorkerCfg cfg; - SRWLatch schLock; - SRWLatch resLock; - SHashObj *schHash; //key: schedulerId, value: SQWSchStatus - SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache + SQWorkerCfg cfg; + int8_t nodeType; + int32_t nodeId; + SRWLatch schLock; + SRWLatch ctxLock; + SHashObj *schHash; //key: schedulerId, value: SQWSchStatus + SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx + void *nodeObj; + putReqToQueryQFp putToQueueFp; } SQWorkerMgmt; #define QW_GOT_RES_DATA(data) (true) @@ -95,40 +100,63 @@ typedef struct SQWorkerMgmt { #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) #define QW_TASK_READY_RESP(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED) -#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0) -#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0) - +#define QW_SET_QTID(id, qId, tId) do { *(uint64_t *)(id) = (qId); *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); } while (0) +#define QW_GET_QTID(id, qId, tId) do { (qId) = *(uint64_t *)(id); (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); } while (0) +#define QW_IDS() sId, qId, tId #define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) #define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +#define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) +#define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) + +#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64 param, mgmt, sId, __VA_ARGS__) +#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64 param, mgmt, sId, __VA_ARGS__) + +#define QW_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) + + +#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 + #define QW_LOCK(type, _lock) do { \ if (QW_READ == (type)) { \ - if ((*(_lock)) < 0) assert(0); \ - taosRLockLatch(_lock); \ - qDebug("QW RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + qDebug("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + qDebug("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ } else { \ - if ((*(_lock)) < 0) assert(0); \ + assert(atomic_load_32((_lock)) >= 0); \ + qDebug("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ - qDebug("QW WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ } \ } while (0) - + #define QW_UNLOCK(type, _lock) do { \ if (QW_READ == (type)) { \ - if ((*(_lock)) <= 0) assert(0); \ + assert(atomic_load_32((_lock)) > 0); \ + qDebug("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ - qDebug("QW RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } else { \ - if ((*(_lock)) <= 0) assert(0); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + qDebug("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ - qDebug("QW WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } \ } while (0) -static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt); + + +static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); +static int32_t qwAddAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); #ifdef __cplusplus diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 2a395fcfe1..23d74ae91d 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -8,7 +8,7 @@ #include "tname.h" #include "dataSinkMgt.h" -int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { +int32_t qwValidateStatus(SQWorkerMgmt *mgmt, int8_t oriStatus, int8_t newStatus, uint64_t sId, uint64_t qId, uint64_t tId) { int32_t code = 0; if (oriStatus == newStatus) { @@ -62,7 +62,7 @@ int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { break; default: - qError("invalid task status:%d", oriStatus); + QW_TASK_ELOG("invalid task status:%d", oriStatus); return TSDB_CODE_QRY_APP_ERROR; } @@ -70,22 +70,27 @@ int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { _return: - qError("invalid task status, from %d to %d", oriStatus, newStatus); - QW_ERR_RET(code); + QW_TASK_ELOG("invalid task status update from %d to %d", oriStatus, newStatus); + QW_RET(code); } -int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) { +int32_t qwUpdateTaskInfo(SQWorkerMgmt *mgmt, SQWTaskStatus *task, int8_t type, void *data, uint64_t sId, uint64_t qId, uint64_t tId) { int32_t code = 0; + int8_t origStatus = 0; switch (type) { case QW_TASK_INFO_STATUS: { int8_t newStatus = *(int8_t *)data; - QW_ERR_RET(qwValidateStatus(task->status, newStatus)); + QW_ERR_RET(qwValidateStatus(mgmt, task->status, newStatus, QW_IDS())); + + origStatus = task->status; task->status = newStatus; + + QW_TASK_DLOG("task status updated from %d to %d", origStatus, newStatus); break; } default: - qError("uknown task info type:%d", type); + QW_TASK_ELOG("unknown task info, type:%d", type); return TSDB_CODE_QRY_APP_ERROR; } @@ -96,18 +101,18 @@ int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - SQWorkerTaskHandlesCache resCache = {0}; + SQWTaskCtx resCache = {0}; resCache.taskHandle = taskHandle; resCache.sinkHandle = sinkHandle; - QW_LOCK(QW_WRITE, &mgmt->resLock); - if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerTaskHandlesCache))) { - QW_UNLOCK(QW_WRITE, &mgmt->resLock); + QW_LOCK(QW_WRITE, &mgmt->ctxLock); + if (0 != taosHashPut(mgmt->ctxHash, id, sizeof(id), &resCache, sizeof(SQWTaskCtx))) { + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId); return TSDB_CODE_QRY_APP_ERROR; } - QW_UNLOCK(QW_WRITE, &mgmt->resLock); + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); return TSDB_CODE_SUCCESS; } @@ -116,7 +121,7 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { - qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); + QW_SCH_DLOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -126,14 +131,18 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, if (0 != code) { if (!HASH_NODE_EXIST(code)) { QW_UNLOCK(QW_WRITE, &mgmt->schLock); - qError("taosHashPut sId[%"PRIx64"] to scheduleHash failed", sId); + QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); return TSDB_CODE_QRY_APP_ERROR; } } QW_UNLOCK(QW_WRITE, &mgmt->schLock); - if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD)) { + if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch)) { + if (code) { + taosHashCleanup(newSch.tasksHash); + } + return TSDB_CODE_SUCCESS; } } @@ -141,7 +150,7 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, return TSDB_CODE_SUCCESS; } -static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { +static int32_t qwAcquireSchedulerImpl(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); if (NULL == (*sch)) { @@ -159,6 +168,14 @@ static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t s return TSDB_CODE_SUCCESS; } +static int32_t qwAddAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { + return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD); +} + +static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { + return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_RET_ERR); +} + static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } @@ -234,7 +251,7 @@ int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) { SQWSchStatus *tsch = NULL; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &tsch, QW_NOT_EXIST_ADD)); + QW_ERR_RET(qwAddAcquireScheduler(QW_READ, mgmt, sId, &tsch)); int32_t code = qwAddTaskToSch(QW_READ, tsch, qId, tId, status, eOpt, task); if (code) { @@ -250,14 +267,14 @@ static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_RET(code); } -static FORCE_INLINE int32_t qwAcquireTaskHandles(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerTaskHandlesCache **handles) { +static FORCE_INLINE int32_t qwAcquireTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWTaskCtx **handles) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); - QW_LOCK(rwType, &mgmt->resLock); - *handles = taosHashGet(mgmt->resHash, id, sizeof(id)); + QW_LOCK(rwType, &mgmt->ctxLock); + *handles = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*handles)) { - QW_UNLOCK(rwType, &mgmt->resLock); + QW_UNLOCK(rwType, &mgmt->ctxLock); return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; } @@ -265,7 +282,7 @@ static FORCE_INLINE int32_t qwAcquireTaskHandles(int32_t rwType, SQWorkerMgmt *m } static FORCE_INLINE void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) { - QW_UNLOCK(rwType, &mgmt->resLock); + QW_UNLOCK(rwType, &mgmt->ctxLock); } @@ -273,7 +290,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs SQWSchStatus *sch = NULL; int32_t taskNum = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); sch->lastAccessTs = taosGetTimestampSec(); @@ -319,7 +336,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId) { SQWSchStatus *sch = NULL; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); sch->lastAccessTs = taosGetTimestampSec(); @@ -333,12 +350,12 @@ int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); QW_UNLOCK(QW_WRITE, &task->lock); _return: @@ -355,7 +372,7 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint SQWTaskStatus *task = NULL; int32_t code = 0; - if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) { *taskStatus = JOB_TASK_STATUS_NULL; return TSDB_CODE_SUCCESS; } @@ -376,17 +393,17 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint } -int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { +int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD)); + QW_ERR_RET(qwAddAcquireScheduler(QW_READ, mgmt, sId, &sch)); - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(QW_READ, sch, qId, tId, &task)) { qwReleaseScheduler(QW_READ, mgmt); - code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); + code = qwAddTask(mgmt, sId, qId, tId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); if (code) { qwReleaseScheduler(QW_READ, mgmt); QW_ERR_RET(code); @@ -409,10 +426,10 @@ int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_ return TSDB_CODE_SUCCESS; } else if (task->status == JOB_TASK_STATUS_FAILED || task->status == JOB_TASK_STATUS_SUCCEED || task->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { newStatus = JOB_TASK_STATUS_CANCELLED; - QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS())); } else { newStatus = JOB_TASK_STATUS_CANCELLING; - QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS())); } QW_UNLOCK(QW_WRITE, &task->lock); @@ -441,50 +458,60 @@ _return: QW_RET(code); } -int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { +int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - char id[sizeof(queryId) + sizeof(taskId)] = {0}; - QW_SET_QTID(id, queryId, taskId); - - QW_LOCK(QW_WRITE, &mgmt->resLock); - if (mgmt->resHash) { - taosHashRemove(mgmt->resHash, id, sizeof(id)); - } - QW_UNLOCK(QW_WRITE, &mgmt->resLock); - if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { - qWarn("scheduler %"PRIx64" doesn't exist", sId); + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + QW_LOCK(QW_WRITE, &mgmt->ctxLock); + if (mgmt->ctxHash) { + if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { + QW_TASK_WLOG("taosHashRemove from ctx hash failed, id:%s", id); + } + } + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + + if (qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch)) { + QW_TASK_WLOG("scheduler does not exist, sch:%p", sch); return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_WRITE, sch, queryId, taskId, &task)) { + if (qwAcquireTask(QW_WRITE, sch, qId, tId, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); - qWarn("scheduler %"PRIx64" queryId %"PRIx64" taskId:%"PRIx64" doesn't exist", sId, queryId, taskId); + QW_TASK_WLOG("task does not exist, task:%p", task); return TSDB_CODE_SUCCESS; } - taosHashRemove(sch->tasksHash, id, sizeof(id)); + QW_TASK_DLOG("drop task, status:%d, code:%x, ready:%d, cancel:%d, drop:%d", task->status, task->code, task->ready, task->cancel, task->drop); + + if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { + QW_TASK_ELOG("taosHashRemove task from hash failed, task:%p", task); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + +_return: qwReleaseTask(QW_WRITE, sch); qwReleaseScheduler(QW_WRITE, mgmt); - return TSDB_CODE_SUCCESS; + QW_RET(code); } -int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { +int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD)); + QW_ERR_RET(qwAddAcquireScheduler(QW_READ, mgmt, sId, &sch)); - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(QW_READ, sch, qId, tId, &task)) { qwReleaseScheduler(QW_READ, mgmt); - code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); + code = qwAddTask(mgmt, sId, qId, tId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); if (code) { qwReleaseScheduler(QW_READ, mgmt); QW_ERR_RET(code); @@ -500,7 +527,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uin if (task->status == JOB_TASK_STATUS_EXECUTING) { newStatus = JOB_TASK_STATUS_DROPPING; - QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS())); } else if (task->status == JOB_TASK_STATUS_CANCELLING || task->status == JOB_TASK_STATUS_DROPPING || task->status == JOB_TASK_STATUS_NOT_START) { QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); @@ -512,7 +539,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uin qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(qwDropTask(mgmt, sId, queryId, taskId)); + QW_ERR_RET(qwDropTask(mgmt, sId, qId, tId)); return TSDB_CODE_SUCCESS; } @@ -743,7 +770,7 @@ int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); @@ -785,7 +812,7 @@ int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); @@ -816,7 +843,7 @@ _return: QW_RET(code); } -int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, bool *needStop) { +int32_t qwCheckTaskCancelDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needStop) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -824,11 +851,11 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI *needStop = false; - if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) { return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(QW_READ, sch, qId, tId, &task)) { qwReleaseScheduler(QW_READ, mgmt); return TSDB_CODE_SUCCESS; } @@ -836,9 +863,10 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI QW_LOCK(QW_READ, &task->lock); if ((!task->cancel) && (!task->drop)) { - qError("no cancel or drop, but task:%"PRIx64" exists", taskId); + QW_TASK_ELOG("no cancel or drop but task exists, status:%d", task->status); QW_UNLOCK(QW_READ, &task->lock); + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -851,17 +879,21 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI if (task->cancel) { QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + code = qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); QW_UNLOCK(QW_WRITE, &task->lock); + + QW_ERR_JRET(code); } if (task->drop) { qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - return qwDropTask(mgmt, sId, queryId, taskId); + QW_RET(qwDropTask(mgmt, sId, qId, tId)); } +_return: + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -875,7 +907,7 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 int32_t code = 0; int8_t newStatus = JOB_TASK_STATUS_CANCELLED; - code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); + code = qwAddAcquireScheduler(QW_READ, mgmt, sId, &sch); if (code) { qError("sId:%"PRIx64" not in cache", sId); QW_ERR_RET(code); @@ -895,7 +927,7 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 if (task->cancel) { QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); + qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS()); QW_UNLOCK(QW_WRITE, &task->lock); } @@ -910,7 +942,7 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 if (!(task->cancel || task->drop)) { QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); task->code = errCode; QW_UNLOCK(QW_WRITE, &task->lock); } @@ -921,6 +953,86 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } +int32_t qwScheduleDataSink(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + if (atomic_load_8(&handles->sinkScheduled)) { + qDebug("data sink already scheduled"); + return TSDB_CODE_SUCCESS; + } + + SSinkDataReq * req = (SSinkDataReq *)rpcMallocCont(sizeof(SSinkDataReq)); + if (NULL == req) { + qError("rpcMallocCont %d failed", (int32_t)sizeof(SSinkDataReq)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + req->header.vgId = mgmt->nodeId; + req->sId = sId; + req->queryId = queryId; + req->taskId = taskId; + + SRpcMsg pNewMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .msgType = TDMT_VND_SCHEDULE_DATA_SINK, + .pCont = req, + .contLen = sizeof(SSinkDataReq), + .code = 0, + }; + + int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); + if (TSDB_CODE_SUCCESS != code) { + qError("put data sink schedule msg to queue failed, code:%x", code); + rpcFreeCont(req); + QW_ERR_RET(code); + } + + qDebug("put data sink schedule msg to query queue"); + + return TSDB_CODE_SUCCESS; +} + +int32_t qwScheduleQuery(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + if (atomic_load_8(&handles->queryScheduled)) { + qDebug("query already scheduled"); + return TSDB_CODE_SUCCESS; + } + + QW_ERR_RET(qwUpdateTaskStatus(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_EXECUTING)); + + SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); + if (NULL == req) { + qError("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + req->header.vgId = mgmt->nodeId; + req->sId = sId; + req->queryId = queryId; + req->taskId = taskId; + + SRpcMsg pNewMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .msgType = TDMT_VND_QUERY_CONTINUE, + .pCont = req, + .contLen = sizeof(SQueryContinueReq), + .code = 0, + }; + + int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); + if (TSDB_CODE_SUCCESS != code) { + qError("put query continue msg to queue failed, code:%x", code); + rpcFreeCont(req); + QW_ERR_RET(code); + } + + + qDebug("put query continue msg to query queue"); + + return TSDB_CODE_SUCCESS; +} + + int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; @@ -932,11 +1044,15 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 int32_t dataLength = 0; SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; - SQWorkerTaskHandlesCache *handles = NULL; + SQWTaskCtx *handles = NULL; - QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, mgmt, queryId, taskId, &handles)); + if (atomic_load_8(&handles->needRsp)) { + qError("last fetch not responsed"); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } - QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); QW_LOCK(QW_READ, &task->lock); @@ -974,15 +1090,15 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) { rsp->completed = 1; + + QW_ERR_JRET(qwUpdateTaskStatus(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED)); } + // Note: schedule data sink firstly and will schedule query after it's done if (output.needSchedule) { - //TODO - } - - if ((!output.queryEnd) && DS_BUF_LOW == output.bufStatus) { - //TODO - //UPDATE STATUS TO EXECUTING + QW_ERR_JRET(qwScheduleDataSink(handles, mgmt, sId, queryId, taskId, pMsg)); + } else if ((!output.queryEnd) && (DS_BUF_LOW == output.bufStatus || DS_BUF_EMPTY == output.bufStatus)) { + QW_ERR_JRET(qwScheduleQuery(handles, mgmt, sId, queryId, taskId, pMsg)); } } else { if (dataLength < 0) { @@ -991,12 +1107,11 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 } if (queryEnd) { - QW_ERR_JRET(qwQueryPostProcess(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED, code)); + QW_ERR_JRET(qwUpdateTaskStatus(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED)); } else { - if (task->status != JOB_TASK_STATUS_EXECUTING) { - qError("invalid status %d for fetch without res", task->status); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } + assert(0 == handles->needRsp); + + qDebug("no res data in sink, need response later"); QW_LOCK(QW_WRITE, &handles->lock); handles->needRsp = true; @@ -1028,7 +1143,12 @@ _return: QW_RET(code); } -int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) { + if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp) { + qError("invalid param to init qworker"); + QW_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); @@ -1037,29 +1157,46 @@ int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { if (cfg) { mgmt->cfg = *cfg; + if (0 == mgmt->cfg.maxSchedulerNum) { + mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + } + if (0 == mgmt->cfg.maxTaskNum) { + mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; + } + if (0 == mgmt->cfg.maxSchTaskNum) { + mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + } } else { mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; - mgmt->cfg.maxResCacheNum = QWORKER_DEFAULT_RES_CACHE_NUMBER; + mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; } mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); if (NULL == mgmt->schHash) { tfree(mgmt); - QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler hash failed", mgmt->cfg.maxSchedulerNum); + qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - mgmt->resHash = taosHashInit(mgmt->cfg.maxResCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == mgmt->resHash) { + mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == mgmt->ctxHash) { taosHashCleanup(mgmt->schHash); mgmt->schHash = NULL; tfree(mgmt); - - QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d res cache hash failed", mgmt->cfg.maxResCacheNum); + qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + mgmt->nodeType = nodeType; + mgmt->nodeId = nodeId; + mgmt->nodeObj = nodeObj; + mgmt->putToQueueFp = fp; + *qWorkerMgmt = mgmt; + qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); + return TSDB_CODE_SUCCESS; } @@ -1069,25 +1206,31 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } int32_t code = 0; - SSubQueryMsg *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - qError("invalid query msg"); - QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - msg->sId = htobe64(msg->sId); - msg->queryId = htobe64(msg->queryId); - msg->taskId = htobe64(msg->taskId); - msg->contentLen = ntohl(msg->contentLen); - bool queryRsped = false; bool needStop = false; struct SSubplan *plan = NULL; + SSubQueryMsg *msg = pMsg->pCont; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + + if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen); + QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + msg->sId = be64toh(msg->sId); + msg->queryId = be64toh(msg->queryId); + msg->taskId = be64toh(msg->taskId); + msg->contentLen = ntohl(msg->contentLen); + + uint64_t sId = msg->sId; + uint64_t qId = msg->queryId; + uint64_t tId = msg->taskId; QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { qWarn("task need stop"); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); } code = qStringToSubplan(msg->msg, &plan); @@ -1144,30 +1287,59 @@ int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *p int32_t code = 0; int8_t status = 0; bool queryDone = false; - uint64_t sId, qId, tId; + SQueryContinueReq *req = (SQueryContinueReq *)pMsg->pCont; + bool needStop = false; + SQWTaskCtx *handles = NULL; - //TODO call executer to continue execute subquery - code = 0; - void *data = NULL; - queryDone = false; - //TODO call executer to continue execute subquery + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + + qTaskInfo_t taskHandle = handles->taskHandle; + DataSinkHandle sinkHandle = handles->sinkHandle; + bool needRsp = handles->needRsp; + + qwReleaseTaskResCache(QW_READ, qWorkerMgmt); + + QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, req->sId, req->queryId, req->taskId, &needStop)); + if (needStop) { + qWarn("task need stop"); + if (needRsp) { + qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); + } + QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + DataSinkHandle newHandle = NULL; + code = qExecTask(taskHandle, &newHandle); + if (code) { + qError("qExecTask failed, code:%x", code); + QW_ERR_JRET(code); + } + + if (sinkHandle != newHandle) { + qError("data sink mis-match"); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + +_return: + + if (needRsp) { + code = qwBuildAndSendQueryRsp(pMsg, code); + } if (TSDB_CODE_SUCCESS != code) { status = JOB_TASK_STATUS_FAILED; - } else if (queryDone) { - status = JOB_TASK_STATUS_SUCCEED; } else { status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } - code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code); - + code = qwQueryPostProcess(qWorkerMgmt, req->sId, req->queryId, req->taskId, status, code); + QW_RET(code); } -int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ +int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } @@ -1176,8 +1348,9 @@ int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid sink data msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } + //dsScheduleProcess(); //TODO return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index eaa79fd39a..4962eab460 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -42,6 +42,11 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { return 0; } +int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { + return 0; +} + + void qwtRpcSendResponse(const SRpcMsg *pRsp) { if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; @@ -258,7 +263,7 @@ TEST(seqTest, normalCase) { stubSetStringToPlan(); stubSetRpcSendResponse(); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); statusMsg.sId = htobe64(1); @@ -328,7 +333,7 @@ TEST(seqTest, cancelFirst) { stubSetStringToPlan(); stubSetRpcSendResponse(); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); statusMsg.sId = htobe64(1); @@ -402,7 +407,7 @@ TEST(seqTest, randCase) { srand(time(NULL)); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); int32_t t = 0; @@ -446,7 +451,7 @@ TEST(seqTest, multithreadRand) { srand(time(NULL)); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); pthread_attr_t thattr; From 5c6924e9e66f2f62ace4e57c731c77047974f0f1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 12 Jan 2022 23:36:15 +0800 Subject: [PATCH 25/63] add libuv --- source/libs/transport/src/rpcMain.c | 91 +++++++++++++++++++---------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 3095ddb9d2..542bde37b9 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -#ifdef USE_UV #include -#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" @@ -78,12 +76,15 @@ typedef struct SThreadObj { } SThreadObj; typedef struct SServerObj { + pthread_t thread; uv_tcp_t server; uv_loop_t* loop; int workerIdx; int numOfThread; SThreadObj** pThreadObj; uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; } SServerObj; typedef struct SConnCtx { @@ -93,33 +94,31 @@ typedef struct SConnCtx { int ref; } SConnCtx; -static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void onTimeout(uv_timer_t* handle); -static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -static void onWrite(uv_write_t* req, int status); -static void onAccept(uv_stream_t* stream, int status); -void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); -static void workerAsyncCB(uv_async_t* handle); +static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void onTimeout(uv_timer_t* handle); +static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void onWrite(uv_write_t* req, int status); +static void onAccept(uv_stream_t* stream, int status); +static void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void workerAsyncCB(uv_async_t* handle); + static void* workerThread(void* arg); +static void* acceptThread(void* arg); + +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); int32_t rpcInit() { return -1; } void rpcCleanup() { return; }; -void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); - if (pRpc == NULL) { - return NULL; - } - if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); - } - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SServerObj* srv = calloc(1, sizeof(SServerObj)); srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - srv->numOfThread = pRpc->numOfThreads; + srv->numOfThread = numOfThreads; srv->workerIdx = 0; srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); + srv->ip = ip; + srv->port = port; uv_loop_init(srv->loop); for (int i = 0; i < srv->numOfThread; i++) { @@ -136,24 +135,34 @@ void* rpcOpen(const SRpcInit* pInit) { srv->pThreadObj[i]->pipe = &(srv->pipe[i][1]); // init read int err = pthread_create(&(srv->pThreadObj[i]->thread), NULL, workerThread, (void*)(srv->pThreadObj[i])); if (err == 0) { - tError("sucess to create worker thread %d", i); + tDebug("sucess to create worker thread %d", i); // printf("thread %d create\n", i); } else { + // clear all resource later tError("failed to create worker thread %d", i); - return NULL; } } - uv_tcp_init(srv->loop, &srv->server); - struct sockaddr_in bind_addr; - uv_ip4_addr("0.0.0.0", pInit->localPort, &bind_addr); - uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); - int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { - tError("Listen error %s\n", uv_err_name(err)); + + int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + if (err == 0) { + tDebug("success to create accept thread"); + } else { + // clear all resource later + } + + return srv; +} +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { return NULL; } - uv_run(srv->loop, UV_RUN_DEFAULT); + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; } void rpcClose(void* arg) { return; } @@ -186,8 +195,11 @@ void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } void onWrite(uv_write_t* req, int status) { + if (status == 0) { + tDebug("data already was written on stream"); + } + // opt - if (req) tDebug("data already was written on stream"); } void workerAsyncCB(uv_async_t* handle) { @@ -207,7 +219,7 @@ void onAccept(uv_stream_t* stream, int status) { uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); uv_buf_t buf = uv_buf_init("a", 1); - // despatch to worker thread + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); } else { @@ -257,6 +269,23 @@ void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { } } +void* acceptThread(void* arg) { + // opt + SServerObj* srv = (SServerObj*)arg; + uv_tcp_init(srv->loop, &srv->server); + + struct sockaddr_in bind_addr; + + int port = 6030; + uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); +} void* workerThread(void* arg) { SThreadObj* pObj = (SThreadObj*)arg; int fd = pObj->fd; From 0db6b12a47e16d16d1768cd9de5e24e1d0d5869a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 01:40:33 +0000 Subject: [PATCH 26/63] more tkv --- .../tkv/src/inc/tkvBtree.h} | 19 ++++++++++++++++++- source/libs/tkv/src/inc/tkvDB.h | 16 +++++++++++++++- .../tkv/src/inc/tkvHash.h} | 13 ++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) rename source/{dnode/vnode/src/vnd/vnodeRead.c => libs/tkv/src/inc/tkvBtree.h} (72%) rename source/{dnode/vnode/src/inc/vnodeRead.h => libs/tkv/src/inc/tkvHash.h} (81%) diff --git a/source/dnode/vnode/src/vnd/vnodeRead.c b/source/libs/tkv/src/inc/tkvBtree.h similarity index 72% rename from source/dnode/vnode/src/vnd/vnodeRead.c rename to source/libs/tkv/src/inc/tkvBtree.h index 5e9f89ccd5..6360e62c79 100644 --- a/source/dnode/vnode/src/vnd/vnodeRead.c +++ b/source/libs/tkv/src/inc/tkvBtree.h @@ -13,4 +13,21 @@ * along with this program. If not, see . */ -#include "vnodeDef.h" \ No newline at end of file +#ifndef _TD_TKV_BTREE_H_ +#define _TD_TKV_BTREE_H_ + +#include "tkvDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + pgid_t root; // root page number +} STkvBtree; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h index 1a45702540..fc8219db3f 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -16,12 +16,26 @@ #ifndef _TD_TKV_DB_H_ #define _TD_TKV_DB_H_ +#include "tkvBtree.h" + #ifdef __cplusplus extern "C" { #endif +typedef enum { + TDB_BTREE = 0, + TDB_HASH, + TDB_HEAP, +} tdb_db_t; + struct TDB { - // TODO + pgsize_t pageSize; + + tdb_db_t type; // DB type + + union { + STkvBtree btree; + } dbimpl; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeRead.h b/source/libs/tkv/src/inc/tkvHash.h similarity index 81% rename from source/dnode/vnode/src/inc/vnodeRead.h rename to source/libs/tkv/src/inc/tkvHash.h index 5ce84b2ebf..06343b7ac9 100644 --- a/source/dnode/vnode/src/inc/vnodeRead.h +++ b/source/libs/tkv/src/inc/tkvHash.h @@ -13,18 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_VNODE_READ_H_ -#define _TD_VNODE_READ_H_ +#ifndef _TD_TKV_HAHS_H_ +#define _TD_TKV_HAHS_H_ + +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" -void vnodeProcessReadMsg(SVnode *pVnode, SVnodeMsg *pMsg); +typedef struct STkvHash { + // TODO +} STkvHash; #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_READ_H_*/ +#endif /*_TD_TKV_HAHS_H_*/ \ No newline at end of file From c31134216c9da4c07898b8bda25e9ff14419b1d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 01:50:42 +0000 Subject: [PATCH 27/63] more --- source/libs/tkv/src/inc/tkvDB.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h index fc8219db3f..6fe9dabf05 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -17,6 +17,7 @@ #define _TD_TKV_DB_H_ #include "tkvBtree.h" +#include "tkvHash.h" #ifdef __cplusplus extern "C" { @@ -30,11 +31,10 @@ typedef enum { struct TDB { pgsize_t pageSize; - - tdb_db_t type; // DB type - + tdb_db_t type; union { STkvBtree btree; + STkvhash hash; } dbimpl; }; From c8c09c8a8cbee993c3c944a62f47fa96d64067aa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 02:12:24 +0000 Subject: [PATCH 28/63] refact --- source/libs/CMakeLists.txt | 2 +- source/libs/{tkv => tdb}/CMakeLists.txt | 8 ++++---- source/libs/{tkv/inc/tkv.h => tdb/inc/tdb.h} | 8 ++++---- .../src/inc/tkvBtree.h => tdb/src/inc/tdbBtree.h} | 8 ++++---- .../inc/tkvBufPool.h => tdb/src/inc/tdbBufPool.h} | 12 ++++++------ .../{tkv/src/inc/tkvDB.h => tdb/src/inc/tdbDB.h} | 10 +++++----- .../{tkv/src/inc/tkvDef.h => tdb/src/inc/tdbDef.h} | 6 +++--- .../inc/tkvDiskMgr.h => tdb/src/inc/tdbDiskMgr.h} | 2 +- .../{tkv/src/inc/tkvEnv.h => tdb/src/inc/tdbEnv.h} | 6 +++--- .../src/inc/tkvHash.h => tdb/src/inc/tdbHash.h} | 6 +++--- .../src/inc/tkvPage.h => tdb/src/inc/tdbPage.h} | 14 ++++---------- .../{tkv/src/tkvBufPool.c => tdb/src/tdbBufPool.c} | 2 +- .../{tkv/src/tDiskMgr.c => tdb/src/tdbDiskMgr.c} | 2 +- source/libs/{tkv => tdb}/test/tDiskMgrTest.cpp | 0 source/libs/{tkv => tdb}/test/tkvTests.cpp | 0 15 files changed, 40 insertions(+), 46 deletions(-) rename source/libs/{tkv => tdb}/CMakeLists.txt (79%) rename source/libs/{tkv/inc/tkv.h => tdb/inc/tdb.h} (91%) rename source/libs/{tkv/src/inc/tkvBtree.h => tdb/src/inc/tdbBtree.h} (89%) rename source/libs/{tkv/src/inc/tkvBufPool.h => tdb/src/inc/tdbBufPool.h} (82%) rename source/libs/{tkv/src/inc/tkvDB.h => tdb/src/inc/tdbDB.h} (88%) rename source/libs/{tkv/src/inc/tkvDef.h => tdb/src/inc/tdbDef.h} (93%) rename source/libs/{tkv/src/inc/tkvDiskMgr.h => tdb/src/inc/tdbDiskMgr.h} (98%) rename source/libs/{tkv/src/inc/tkvEnv.h => tdb/src/inc/tdbEnv.h} (91%) rename source/libs/{tkv/src/inc/tkvHash.h => tdb/src/inc/tdbHash.h} (93%) rename source/libs/{tkv/src/inc/tkvPage.h => tdb/src/inc/tdbPage.h} (77%) rename source/libs/{tkv/src/tkvBufPool.c => tdb/src/tdbBufPool.c} (98%) rename source/libs/{tkv/src/tDiskMgr.c => tdb/src/tdbDiskMgr.c} (98%) rename source/libs/{tkv => tdb}/test/tDiskMgrTest.cpp (100%) rename source/libs/{tkv => tdb}/test/tkvTests.cpp (100%) diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 1dc16c74f7..1d23f333b2 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -add_subdirectory(tkv) +add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt similarity index 79% rename from source/libs/tkv/CMakeLists.txt rename to source/libs/tdb/CMakeLists.txt index fec3f37cd5..1832ea0b5d 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,17 +1,17 @@ -aux_source_directory(src TKV_SRC) -add_library(tkv STATIC ${TKV_SRC}) +aux_source_directory(src TDB_SRC) +add_library(tdb STATIC ${TDB_SRC}) # target_include_directories( # tkv # PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" # PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" # ) target_include_directories( - tkv + tdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc" ) target_link_libraries( - tkv + tdb PUBLIC os PUBLIC util ) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tdb/inc/tdb.h similarity index 91% rename from source/libs/tkv/inc/tkv.h rename to source/libs/tdb/inc/tdb.h index 00534d2827..2f47f545b1 100644 --- a/source/libs/tkv/inc/tkv.h +++ b/source/libs/tdb/inc/tdb.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_H_ -#define _TD_TKV_H_ +#ifndef _TD_TDB_H_ +#define _TD_TDB_H_ #include "os.h" @@ -28,7 +28,7 @@ typedef struct TDB_ENV TDB_ENV; // SKey typedef struct { - void * bdata; + void* bdata; uint32_t size; } TDB_KEY, TDB_VALUE; @@ -36,4 +36,4 @@ typedef struct { } #endif -#endif /*_TD_TKV_H_*/ \ No newline at end of file +#endif /*_TD_TDB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBtree.h b/source/libs/tdb/src/inc/tdbBtree.h similarity index 89% rename from source/libs/tkv/src/inc/tkvBtree.h rename to source/libs/tdb/src/inc/tdbBtree.h index 6360e62c79..c68f94bb48 100644 --- a/source/libs/tkv/src/inc/tkvBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_BTREE_H_ -#define _TD_TKV_BTREE_H_ +#ifndef _TD_TDB_BTREE_H_ +#define _TD_TDB_BTREE_H_ #include "tkvDef.h" @@ -24,10 +24,10 @@ extern "C" { typedef struct { pgid_t root; // root page number -} STkvBtree; +} TDB_BTREE; #ifdef __cplusplus } #endif -#endif /*_TD_TKV_BTREE_H_*/ \ No newline at end of file +#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h similarity index 82% rename from source/libs/tkv/src/inc/tkvBufPool.h rename to source/libs/tdb/src/inc/tdbBufPool.h index ec8d177a9a..7fda5bf2e6 100644 --- a/source/libs/tkv/src/inc/tkvBufPool.h +++ b/source/libs/tdb/src/inc/tdbBufPool.h @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_BUF_POOL_H_ -#define _TD_TKV_BUF_POOL_H_ +#ifndef _TD_TDB_BUF_POOL_H_ +#define _TD_TDB_BUF_POOL_H_ -#include "tkvPage.h" +#include "tdbPage.h" #ifdef __cplusplus extern "C" { @@ -26,9 +26,9 @@ typedef struct STkvBufPool STkvBufPool; int tbpOpen(STkvBufPool **ppTkvBufPool); int tbpClose(STkvBufPool *pTkvBufPool); -STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +STdbPage *tbpNewPage(STkvBufPool *pTkvBufPool); int tbpDelPage(STkvBufPool *pTkvBufPool); -STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +STdbPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); void tbpFlushPages(STkvBufPool *pTkvBufPool); @@ -36,4 +36,4 @@ void tbpFlushPages(STkvBufPool *pTkvBufPool); } #endif -#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tdb/src/inc/tdbDB.h similarity index 88% rename from source/libs/tkv/src/inc/tkvDB.h rename to source/libs/tdb/src/inc/tdbDB.h index 6fe9dabf05..40ddb1eb31 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DB_H_ -#define _TD_TKV_DB_H_ +#ifndef _TD_TDB_DB_H_ +#define _TD_TDB_DB_H_ -#include "tkvBtree.h" -#include "tkvHash.h" +#include "tdbBtree.h" +#include "tdbHash.h" #ifdef __cplusplus extern "C" { @@ -42,4 +42,4 @@ struct TDB { } #endif -#endif /*_TD_TKV_DB_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDef.h b/source/libs/tdb/src/inc/tdbDef.h similarity index 93% rename from source/libs/tkv/src/inc/tkvDef.h rename to source/libs/tdb/src/inc/tdbDef.h index cd418019be..a04b8cc402 100644 --- a/source/libs/tkv/src/inc/tkvDef.h +++ b/source/libs/tdb/src/inc/tdbDef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DEF_H_ -#define _TD_TKV_DEF_H_ +#ifndef _TD_TDB_DEF_H_ +#define _TD_TDB_DEF_H_ #include "os.h" @@ -39,4 +39,4 @@ typedef int32_t pgsize_t; } #endif -#endif /*_TD_TKV_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h similarity index 98% rename from source/libs/tkv/src/inc/tkvDiskMgr.h rename to source/libs/tdb/src/inc/tdbDiskMgr.h index 2ebe98ace2..b83a147437 100644 --- a/source/libs/tkv/src/inc/tkvDiskMgr.h +++ b/source/libs/tdb/src/inc/tdbDiskMgr.h @@ -22,7 +22,7 @@ extern "C" { #include "os.h" -#include "tkvDef.h" +#include "tdbDef.h" typedef struct STkvDiskMgr STkvDiskMgr; diff --git a/source/libs/tkv/src/inc/tkvEnv.h b/source/libs/tdb/src/inc/tdbEnv.h similarity index 91% rename from source/libs/tkv/src/inc/tkvEnv.h rename to source/libs/tdb/src/inc/tdbEnv.h index eba442e5a5..f3e4ef5888 100644 --- a/source/libs/tkv/src/inc/tkvEnv.h +++ b/source/libs/tdb/src/inc/tdbEnv.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_ENV_H_ -#define _TD_TKV_ENV_H_ +#ifndef _TD_TDB_ENV_H_ +#define _TD_TDB_ENV_H_ #ifdef __cplusplus extern "C" { @@ -28,4 +28,4 @@ struct TDB_ENV { } #endif -#endif /*_TD_TKV_ENV_H_*/ \ No newline at end of file +#endif /*_TD_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvHash.h b/source/libs/tdb/src/inc/tdbHash.h similarity index 93% rename from source/libs/tkv/src/inc/tkvHash.h rename to source/libs/tdb/src/inc/tdbHash.h index 06343b7ac9..fca19035f1 100644 --- a/source/libs/tkv/src/inc/tkvHash.h +++ b/source/libs/tdb/src/inc/tdbHash.h @@ -16,15 +16,15 @@ #ifndef _TD_TKV_HAHS_H_ #define _TD_TKV_HAHS_H_ -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STkvHash { +typedef struct { // TODO -} STkvHash; +} TDB_HASH; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/inc/tkvPage.h b/source/libs/tdb/src/inc/tdbPage.h similarity index 77% rename from source/libs/tkv/src/inc/tkvPage.h rename to source/libs/tdb/src/inc/tdbPage.h index d596d215cd..e7245b6c39 100644 --- a/source/libs/tkv/src/inc/tkvPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -17,30 +17,24 @@ #define _TD_TKV_PAGE_H_ #include "os.h" -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STkvPage { +typedef struct { pgid_t pgid; int32_t pinCount; bool idDirty; char* pData; -} STkvPage; +} STdbPage; typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} STkvPgHdr; - -// typedef struct { -// SPgHdr chdr; -// uint16_t used; // number of used slots -// uint16_t loffset; // the offset of the starting location of the last slot used -// } SSlottedPgHdr; +} STdbPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tdb/src/tdbBufPool.c similarity index 98% rename from source/libs/tkv/src/tkvBufPool.c rename to source/libs/tdb/src/tdbBufPool.c index 86bfa0ba3e..6ae3499064 100644 --- a/source/libs/tkv/src/tkvBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -26,7 +26,7 @@ struct SFrameIdWrapper { }; struct STkvBufPool { - STkvPage* pages; + STdbPage* pages; STkvDiskMgr* pDiskMgr; SHashObj* pgTb; // page_id_t --> frame_id_t TD_SLIST(SFrameIdWrapper) freeList; diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tdb/src/tdbDiskMgr.c similarity index 98% rename from source/libs/tkv/src/tDiskMgr.c rename to source/libs/tdb/src/tdbDiskMgr.c index fa8f6062d8..71ab5f2589 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tdb/src/tdbDiskMgr.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tkvDiskMgr.h" +#include "tdbDiskMgr.h" struct STkvDiskMgr { char * fname; diff --git a/source/libs/tkv/test/tDiskMgrTest.cpp b/source/libs/tdb/test/tDiskMgrTest.cpp similarity index 100% rename from source/libs/tkv/test/tDiskMgrTest.cpp rename to source/libs/tdb/test/tDiskMgrTest.cpp diff --git a/source/libs/tkv/test/tkvTests.cpp b/source/libs/tdb/test/tkvTests.cpp similarity index 100% rename from source/libs/tkv/test/tkvTests.cpp rename to source/libs/tdb/test/tkvTests.cpp From 3aae62f5dc55153df9081b01e54b9dfef3670294 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 02:39:12 +0000 Subject: [PATCH 29/63] more --- source/libs/tdb/src/inc/tdbBufPool.h | 16 ++++++++-------- source/libs/tdb/src/tdbBufPool.c | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h index 7fda5bf2e6..5200d22faa 100644 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ b/source/libs/tdb/src/inc/tdbBufPool.h @@ -22,15 +22,15 @@ extern "C" { #endif -typedef struct STkvBufPool STkvBufPool; +typedef struct STdbBufPool STdbBufPool; -int tbpOpen(STkvBufPool **ppTkvBufPool); -int tbpClose(STkvBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STkvBufPool *pTkvBufPool); -int tbpDelPage(STkvBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STkvBufPool *pTkvBufPool); +int tbpOpen(STdbBufPool **ppTkvBufPool); +int tbpClose(STdbBufPool *pTkvBufPool); +STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); +int tbpDelPage(STdbBufPool *pTkvBufPool); +STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STdbBufPool *pTkvBufPool); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/tdbBufPool.c index 6ae3499064..3eed100566 100644 --- a/source/libs/tdb/src/tdbBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -25,7 +25,7 @@ struct SFrameIdWrapper { frame_id_t id; }; -struct STkvBufPool { +struct STdbBufPool { STdbPage* pages; STkvDiskMgr* pDiskMgr; SHashObj* pgTb; // page_id_t --> frame_id_t From 4c5a6ccf946df9efaf661223babdc45cafc2e847 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 13 Jan 2022 11:36:58 +0800 Subject: [PATCH 30/63] feature/qnode --- source/libs/qworker/inc/qworkerInt.h | 15 +- source/libs/qworker/src/qworker.c | 286 ++++++++++++++------------- 2 files changed, 160 insertions(+), 141 deletions(-) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index b5ec50cc04..2f47c79065 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -114,9 +114,13 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64 param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64 param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64 param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64 param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64 param, mgmt, qId, tId, __VA_ARGS__) + +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 @@ -155,8 +159,9 @@ typedef struct SQWorkerMgmt { -static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); -static int32_t qwAddAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); +int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); +int32_t qwAcquireAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); +int32_t qwAcquireTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task); #ifdef __cplusplus diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 23d74ae91d..921298995f 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -108,7 +108,7 @@ int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, QW_LOCK(QW_WRITE, &mgmt->ctxLock); if (0 != taosHashPut(mgmt->ctxHash, id, sizeof(id), &resCache, sizeof(SQWTaskCtx))) { QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); - qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId); + QW_TASK_ELOG("taosHashPut task ctx to ctxHash failed, taskHandle:%p, sinkHandle:%p", taskHandle, sinkHandle); return TSDB_CODE_QRY_APP_ERROR; } @@ -117,7 +117,7 @@ int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, return TSDB_CODE_SUCCESS; } -static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { +int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { @@ -150,7 +150,7 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, return TSDB_CODE_SUCCESS; } -static int32_t qwAcquireSchedulerImpl(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); if (NULL == (*sch)) { @@ -168,42 +168,19 @@ static int32_t qwAcquireSchedulerImpl(int32_t rwType, SQWorkerMgmt *mgmt, uint64 return TSDB_CODE_SUCCESS; } -static int32_t qwAddAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { +int32_t qwAcquireAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD); } -static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { +int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_RET_ERR); } -static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { +void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } -static int32_t qwAcquireTaskImpl(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { - char id[sizeof(qId) + sizeof(tId)] = {0}; - QW_SET_QTID(id, qId, tId); - - QW_LOCK(rwType, &sch->tasksLock); - *task = taosHashGet(sch->tasksHash, id, sizeof(id)); - if (NULL == (*task)) { - QW_UNLOCK(rwType, &sch->tasksLock); - - return TSDB_CODE_QRY_TASK_NOT_EXIST; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t qwAcquireTask(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { - return qwAcquireTaskImpl(rwType, sch, qId, tId, task); -} - -static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { - QW_UNLOCK(rwType, &sch->tasksLock); -} - -int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int8_t status, int32_t eOpt, SQWTaskStatus **task) { +int32_t qwAddTaskImpl(SQWorkerMgmt *mgmt, SQWSchStatus *sch, int32_t rwType, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWTaskStatus **task) { int32_t code = 0; char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -212,62 +189,83 @@ int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t SQWTaskStatus ntask = {0}; ntask.status = status; - while (true) { - QW_LOCK(QW_WRITE, &sch->tasksLock); - int32_t code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask)); - if (0 != code) { - QW_UNLOCK(QW_WRITE, &sch->tasksLock); - if (HASH_NODE_EXIST(code)) { - if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { - if (qwAcquireTask(rwType, sch, qId, tId, task)) { - continue; - } - } else if (QW_EXIST_RET_ERR == eOpt) { - return TSDB_CODE_QRY_TASK_ALREADY_EXIST; - } else { - assert(0); - } - - break; - } else { - qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", qId, tId); - return TSDB_CODE_QRY_APP_ERROR; - } - } - + QW_LOCK(QW_WRITE, &sch->tasksLock); + code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask)); + if (0 != code) { QW_UNLOCK(QW_WRITE, &sch->tasksLock); - - if (rwType && task) { - if (TSDB_CODE_SUCCESS == qwAcquireTask(rwType, sch, qId, tId, task)) { - return TSDB_CODE_SUCCESS; + if (HASH_NODE_EXIST(code)) { + if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { + QW_ERR_RET(qwAcquireTask(mgmt, rwType, sch, qId, tId, task)); + } else if (QW_EXIST_RET_ERR == eOpt) { + return TSDB_CODE_QRY_TASK_ALREADY_EXIST; + } else { + assert(0); } } else { - break; + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", qId, tId); + return TSDB_CODE_QRY_APP_ERROR; } - } + } + + QW_UNLOCK(QW_WRITE, &sch->tasksLock); + + if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { + QW_ERR_RET(qwAcquireTask(mgmt, rwType, sch, qId, tId, task)); + } return TSDB_CODE_SUCCESS; } -static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) { +int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status) { SQWSchStatus *tsch = NULL; - QW_ERR_RET(qwAddAcquireScheduler(QW_READ, mgmt, sId, &tsch)); + int32_t code = 0; + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &tsch)); - int32_t code = qwAddTaskToSch(QW_READ, tsch, qId, tId, status, eOpt, task); - if (code) { - qwReleaseScheduler(QW_WRITE, mgmt); - } + QW_ERR_JRET(qwAddTaskImpl(mgmt, tsch, 0, qId, tId, JOB_TASK_STATUS_NOT_START, QW_EXIST_RET_ERR, NULL)); - if (NULL == task) { - qwReleaseScheduler(QW_READ, mgmt); - } else if (sch) { - *sch = tsch; - } +_return: - QW_RET(code); + qwReleaseScheduler(QW_READ, mgmt); + QW_ERR_RET(code); } -static FORCE_INLINE int32_t qwAcquireTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWTaskCtx **handles) { + +int32_t qwAcquireTaskImpl(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int32_t status, int32_t nOpt, SQWTaskStatus **task) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + QW_LOCK(rwType, &sch->tasksLock); + *task = taosHashGet(sch->tasksHash, id, sizeof(id)); + if (NULL == (*task)) { + QW_UNLOCK(rwType, &sch->tasksLock); + + if (QW_NOT_EXIST_ADD == nOpt) { + QW_ERR_RET(qwAddTaskImpl(mgmt, sch, rwType, qId, tId, status, QW_EXIST_ACQUIRE, task)); + } else if (QW_NOT_EXIST_RET_ERR == nOpt) { + return TSDB_CODE_QRY_TASK_NOT_EXIST; + } else { + assert(0); + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qwAcquireTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { + return qwAcquireTaskImpl(mgmt, rwType, sch, qId, tId, 0, QW_NOT_EXIST_RET_ERR, task); +} + +int32_t qwAcquireAddTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int32_t status, SQWTaskStatus **task) { + return qwAcquireTaskImpl(mgmt, rwType, sch, qId, tId, status, QW_NOT_EXIST_ADD, task); +} + + +void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { + QW_UNLOCK(rwType, &sch->tasksLock); +} + + +int32_t qwAcquireTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWTaskCtx **handles) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); @@ -281,7 +279,7 @@ static FORCE_INLINE int32_t qwAcquireTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt, return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) { +void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->ctxLock); } @@ -352,7 +350,7 @@ int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, qId, tId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); @@ -377,7 +375,7 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)) { qwReleaseScheduler(QW_READ, mgmt); *taskStatus = JOB_TASK_STATUS_NULL; @@ -398,17 +396,10 @@ int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tI SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAddAcquireScheduler(QW_READ, mgmt, sId, &sch)); + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch)); + + QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task)); - if (qwAcquireTask(QW_READ, sch, qId, tId, &task)) { - qwReleaseScheduler(QW_READ, mgmt); - - code = qwAddTask(mgmt, sId, qId, tId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); - if (code) { - qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(code); - } - } QW_LOCK(QW_WRITE, &task->lock); @@ -458,6 +449,42 @@ _return: QW_RET(code); } + +// caller should make sure task is not running +int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + QW_LOCK(QW_WRITE, &mgmt->ctxLock); + SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); + if (NULL == ctx) { + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; + } + + if (ctx->taskHandle) { + qDestroyTask(ctx->taskHandle); + ctx->taskHandle = NULL; + } + + if (ctx->sinkHandle) { + dsDestroyDataSinker(ctx->sinkHandle); + ctx->sinkHandle = NULL; + } + + if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { + QW_TASK_ELOG("taosHashRemove from ctx hash failed, id:%s", id); + + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; + } + + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + + return TSDB_CODE_SUCCESS; +} + + int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -466,20 +493,14 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - QW_LOCK(QW_WRITE, &mgmt->ctxLock); - if (mgmt->ctxHash) { - if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { - QW_TASK_WLOG("taosHashRemove from ctx hash failed, id:%s", id); - } - } - QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + qwDropTaskCtx(mgmt, sId, qId, tId); if (qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch)) { QW_TASK_WLOG("scheduler does not exist, sch:%p", sch); return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_WRITE, sch, qId, tId, &task)) { + if (qwAcquireTask(mgmt, QW_WRITE, sch, qId, tId, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); QW_TASK_WLOG("task does not exist, task:%p", task); @@ -506,17 +527,9 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAddAcquireScheduler(QW_READ, mgmt, sId, &sch)); + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch)); - if (qwAcquireTask(QW_READ, sch, qId, tId, &task)) { - qwReleaseScheduler(QW_READ, mgmt); - - code = qwAddTask(mgmt, sId, qId, tId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); - if (code) { - qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(code); - } - } + QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task)); QW_LOCK(QW_WRITE, &task->lock); @@ -772,7 +785,7 @@ int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)); QW_LOCK(QW_WRITE, &task->lock); @@ -814,7 +827,7 @@ int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)); QW_LOCK(QW_WRITE, &task->lock); if (QW_TASK_READY_RESP(task->status)) { @@ -843,7 +856,7 @@ _return: QW_RET(code); } -int32_t qwCheckTaskCancelDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needStop) { +int32_t qwCheckAndProcessTaskDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needStop) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -855,7 +868,7 @@ int32_t qwCheckTaskCancelDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, ui return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_READ, sch, qId, tId, &task)) { + if (qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)) { qwReleaseScheduler(QW_READ, mgmt); return TSDB_CODE_SUCCESS; } @@ -867,10 +880,7 @@ int32_t qwCheckTaskCancelDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, ui QW_UNLOCK(QW_READ, &task->lock); - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - QW_RET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } QW_UNLOCK(QW_READ, &task->lock); @@ -907,22 +917,16 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 int32_t code = 0; int8_t newStatus = JOB_TASK_STATUS_CANCELLED; - code = qwAddAcquireScheduler(QW_READ, mgmt, sId, &sch); + code = qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch); if (code) { - qError("sId:%"PRIx64" not in cache", sId); + QW_TASK_ELOG("sId:%"PRIx64" not in cache", sId); QW_ERR_RET(code); } - code = qwAcquireTask(QW_READ, sch, qId, tId, &task); + code = qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task); if (code) { - qwReleaseScheduler(QW_READ, mgmt); - - if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { - qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); - QW_ERR_RET(code); - } - - QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); + QW_TASK_ELOG("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); + QW_ERR_RET(code); } if (task->cancel) { @@ -940,7 +944,7 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } - if (!(task->cancel || task->drop)) { + if ((!(task->cancel || task->drop)) && status > 0) { QW_LOCK(QW_WRITE, &task->lock); qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); task->code = errCode; @@ -1053,7 +1057,7 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 } QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)); QW_LOCK(QW_READ, &task->lock); @@ -1208,6 +1212,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; bool queryRsped = false; bool needStop = false; + bool taskAdded = false; struct SSubplan *plan = NULL; SSubQueryMsg *msg = pMsg->pCont; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; @@ -1226,27 +1231,30 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); + QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { - qWarn("task need stop"); + QW_TASK_DLOG("task need stop, msgLen:%d", msg->contentLen); qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); } code = qStringToSubplan(msg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->sId, msg->queryId, msg->taskId, code); + QW_TASK_ELOG("string to subplan failed, code:%d", code); QW_ERR_JRET(code); } qTaskInfo_t pTaskInfo = NULL; code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { - qError("qCreateExecTask failed, code:%x", code); + QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); + QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_FAILED)); QW_ERR_JRET(code); - } else { - QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); } + + QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING)); + + taskAdded = true; QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); @@ -1256,12 +1264,12 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { code = qExecTask(pTaskInfo, &sinkHandle); if (code) { - qError("qExecTask failed, code:%x", code); + QW_TASK_ELOG("qExecTask failed, code:%x", code); QW_ERR_JRET(code); - } else { - QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); - QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); - } + } + + QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); + QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); _return: @@ -1278,6 +1286,12 @@ _return: status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } + if (!taskAdded) { + qwAddTask(qWorkerMgmt, sId, qId, tId, status); + + status = -1; + } + qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, code); QW_RET(code); @@ -1299,7 +1313,7 @@ int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *p qwReleaseTaskResCache(QW_READ, qWorkerMgmt); - QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, req->sId, req->queryId, req->taskId, &needStop)); + QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, req->sId, req->queryId, req->taskId, &needStop)); if (needStop) { qWarn("task need stop"); if (needRsp) { From f1df5eaabfe8df2f9e527f39a9d14c537f46409f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 05:37:12 +0000 Subject: [PATCH 31/63] more refact --- source/dnode/vnode/src/inc/vnodeDef.h | 13 ++- source/dnode/vnode/src/inc/vnodeMAF.h | 32 ------ source/dnode/vnode/src/inc/vnodeRequest.h | 33 ------ source/dnode/vnode/src/inc/vnodeSync.h | 33 ------ source/dnode/vnode/src/meta/CMakeLists.txt | 46 -------- source/dnode/vnode/src/tq/CMakeLists.txt | 20 ---- source/dnode/vnode/src/tsdb/CMakeLists.txt | 34 ------ source/dnode/vnode/src/vnd/CMakeLists.txt | 24 ----- source/dnode/vnode/src/vnd/vnodeMgr.c | 2 +- source/dnode/vnode/src/vnd/vnodeRequest.c | 119 --------------------- source/dnode/vnode/src/vnd/vnodeSync.c | 14 --- 11 files changed, 11 insertions(+), 359 deletions(-) delete mode 100644 source/dnode/vnode/src/inc/vnodeMAF.h delete mode 100644 source/dnode/vnode/src/inc/vnodeRequest.h delete mode 100644 source/dnode/vnode/src/inc/vnodeSync.h delete mode 100644 source/dnode/vnode/src/meta/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/tq/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/tsdb/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/vnd/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/vnd/vnodeRequest.c delete mode 100644 source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 1c534a8aeb..7054d5bc11 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -33,9 +33,7 @@ #include "vnodeFS.h" #include "vnodeMemAllocator.h" #include "vnodeQuery.h" -#include "vnodeRequest.h" #include "vnodeStateMgr.h" -#include "vnodeSync.h" #ifdef __cplusplus extern "C" { @@ -73,7 +71,6 @@ struct SVnode { STsdb* pTsdb; STQ* pTq; SWal* pWal; - SVnodeSync* pSync; SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; @@ -84,6 +81,16 @@ int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); +// For Log +extern int32_t vDebugFlag; + +#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeMAF.h b/source/dnode/vnode/src/inc/vnodeMAF.h deleted file mode 100644 index 7aa405103c..0000000000 --- a/source/dnode/vnode/src/inc/vnodeMAF.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_MAF_H_ -#define _TD_VNODE_MAF_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int vnodeOpenMAF(SVnode *pVnode); -void vnodeCloseMAF(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_MAF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeRequest.h b/source/dnode/vnode/src/inc/vnodeRequest.h deleted file mode 100644 index 52f4281eea..0000000000 --- a/source/dnode/vnode/src/inc/vnodeRequest.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_REQUEST_H_ -#define _TD_VNODE_REQUEST_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// SVDropTbReq -// int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq); -// void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_REQUEST_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h deleted file mode 100644 index e82979551d..0000000000 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_SYNC_H_ -#define _TD_VNODE_SYNC_H_ - -// #include "sync.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - /* data */ -} SVnodeSync; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_SYNC_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/CMakeLists.txt b/source/dnode/vnode/src/meta/CMakeLists.txt deleted file mode 100644 index 7041811617..0000000000 --- a/source/dnode/vnode/src/meta/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -set(META_DB_IMPL_LIST "BDB" "SQLITE") -set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") -set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) - -if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) - message(STATUS "META DB Impl: ${META_DB_IMPL}==============") -else() - message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============") -endif() - -aux_source_directory(src META_SRC) -if(${META_DB_IMPL} STREQUAL "BDB") - list(REMOVE_ITEM META_SRC "src/metaSQLiteImpl.c") -elseif(${META_DB_IMPL} STREQUAL "SQLITE") - list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c") -endif() - -add_library(meta STATIC ${META_SRC}) -target_include_directories( - meta - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/meta" - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - meta - PUBLIC common - PUBLIC index -) - -if(${META_DB_IMPL} STREQUAL "BDB") - target_link_libraries( - meta - PUBLIC bdb - ) -elseif(${META_DB_IMPL} STREQUAL "SQLITE") - target_link_libraries( - meta - PUBLIC sqlite - ) -endif() - - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/tq/CMakeLists.txt b/source/dnode/vnode/src/tq/CMakeLists.txt deleted file mode 100644 index 7cb7499d64..0000000000 --- a/source/dnode/vnode/src/tq/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -aux_source_directory(src TQ_SRC) -add_library(tq ${TQ_SRC}) -target_include_directories( - tq - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tq" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - tq - PUBLIC wal - PUBLIC os - PUBLIC util - PUBLIC common - PUBLIC transport -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/tsdb/CMakeLists.txt b/source/dnode/vnode/src/tsdb/CMakeLists.txt deleted file mode 100644 index e38ba1c466..0000000000 --- a/source/dnode/vnode/src/tsdb/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -aux_source_directory(src TSDB_SRC) -if(0) - add_library(tsdb ${TSDB_SRC}) -else(0) - add_library(tsdb STATIC "") - target_sources(tsdb - PRIVATE - "src/tsdbCommit.c" - "src/tsdbMain.c" - "src/tsdbMemTable.c" - "src/tsdbOptions.c" - "src/tsdbWrite.c" - "src/tsdbReadImpl.c" - "src/tsdbFile.c" - "src/tsdbFS.c" - "src/tsdbRead.c" - ) -endif(0) - -target_include_directories( - tsdb - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - tsdb - PUBLIC os - PUBLIC util - PUBLIC common - PUBLIC tkv - PUBLIC tfs - PUBLIC meta -) \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/CMakeLists.txt b/source/dnode/vnode/src/vnd/CMakeLists.txt deleted file mode 100644 index 944a4276db..0000000000 --- a/source/dnode/vnode/src/vnd/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -aux_source_directory(src VNODE_SRC) -add_library(vnode STATIC ${VNODE_SRC}) -target_include_directories( - vnode - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode" - private "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - vnode - PUBLIC os - PUBLIC transport - PUBLIC meta - PUBLIC tq - PUBLIC tsdb - PUBLIC wal - PUBLIC sync - PUBLIC cjson - PUBLIC qworker -) - -# test -if(${BUILD_TEST}) -# add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index fdb96e52e2..ce9e487076 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -41,7 +41,7 @@ int vnodeInit(const SVnodeOpt *pOption) { for (uint16_t i = 0; i < pOption->nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); - pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); + // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } } else { // TODO: if no commit thread is set, then another mechanism should be diff --git a/source/dnode/vnode/src/vnd/vnodeRequest.c b/source/dnode/vnode/src/vnd/vnodeRequest.c deleted file mode 100644 index 5367c9e091..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeRequest.c +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -#if 0 - -static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq); -static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq); - -int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type) { - int tsize = 0; - - tsize += taosEncodeFixedU64(buf, pReq->ver); - switch (type) { - case TDMT_VND_CREATE_STB: - tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); - break; - case TDMT_VND_SUBMIT: - /* code */ - break; - default: - break; - } - /* TODO */ - return tsize; -} - -void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); - - switch (type) { - case TDMT_VND_CREATE_STB: - buf = vnodeParseCreateTableReq(buf, &(pReq->ctReq)); - break; - - default: - break; - } - - // TODO - return buf; -} - -static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq) { - int tsize = 0; - - tsize += taosEncodeString(buf, pReq->name); - tsize += taosEncodeFixedU32(buf, pReq->ttl); - tsize += taosEncodeFixedU32(buf, pReq->keep); - tsize += taosEncodeFixedU8(buf, pReq->type); - - switch (pReq->type) { - case META_SUPER_TABLE: - tsize += taosEncodeFixedU64(buf, pReq->stbCfg.suid); - tsize += tdEncodeSchema(buf, pReq->stbCfg.pSchema); - tsize += tdEncodeSchema(buf, pReq->stbCfg.pTagSchema); - break; - case META_CHILD_TABLE: - tsize += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); - tsize += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); - break; - case META_NORMAL_TABLE: - tsize += tdEncodeSchema(buf, pReq->ntbCfg.pSchema); - break; - default: - break; - } - - return tsize; -} - -static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq) { - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeFixedU32(buf, &(pReq->ttl)); - buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->type)); - - switch (pReq->type) { - case META_SUPER_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); - buf = tdDecodeSchema(buf, &(pReq->stbCfg.pSchema)); - buf = tdDecodeSchema(buf, &(pReq->stbCfg.pTagSchema)); - break; - case META_CHILD_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->ctbCfg.suid)); - buf = tdDecodeKVRow(buf, &(pReq->ctbCfg.pTag)); - break; - case META_NORMAL_TABLE: - buf = tdDecodeSchema(buf, &(pReq->ntbCfg.pSchema)); - break; - default: - break; - } - - return buf; -} - -int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq) { - // TODO - return 0; -} - -void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq) { - // TODO -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file From 093b982fa01b4d13349ed2315118f115de6b2550 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 05:45:03 +0000 Subject: [PATCH 32/63] more refact --- source/dnode/vnode/src/inc/vnodeDef.h | 2 -- source/dnode/vnode/src/inc/vnodeFS.h | 47 ------------------------- source/dnode/vnode/src/inc/vnodeQuery.h | 3 +- source/dnode/vnode/src/vnd/vnodeFS.c | 16 --------- source/libs/tdb/src/tdbBufPool.c | 6 ++-- 5 files changed, 4 insertions(+), 70 deletions(-) delete mode 100644 source/dnode/vnode/src/inc/vnodeFS.h delete mode 100644 source/dnode/vnode/src/vnd/vnodeFS.c diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 7054d5bc11..85563890fa 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -30,7 +30,6 @@ #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" -#include "vnodeFS.h" #include "vnodeMemAllocator.h" #include "vnodeQuery.h" #include "vnodeStateMgr.h" @@ -71,7 +70,6 @@ struct SVnode { STsdb* pTsdb; STQ* pTq; SWal* pWal; - SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; SDnode* pDnode; diff --git a/source/dnode/vnode/src/inc/vnodeFS.h b/source/dnode/vnode/src/inc/vnodeFS.h deleted file mode 100644 index dbec985695..0000000000 --- a/source/dnode/vnode/src/inc/vnodeFS.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_FS_H_ -#define _TD_VNODE_FS_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { -} SDir; - -typedef struct { -} SFile; - -typedef struct SFS { - void *pImpl; - int (*startEdit)(struct SFS *); - int (*endEdit)(struct SFS *); -} SFS; - -typedef struct { -} SVnodeFS; - -int vnodeOpenFS(SVnode *pVnode); -void vnodeCloseFS(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_FS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h index d43f5b1cf1..9d40c34f00 100644 --- a/source/dnode/vnode/src/inc/vnodeQuery.h +++ b/source/dnode/vnode/src/inc/vnodeQuery.h @@ -19,12 +19,11 @@ #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" #include "qworker.h" +#include "vnode.h" typedef struct SQWorkerMgmt SQHandle; - int vnodeQueryOpen(SVnode *pVnode); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/vnd/vnodeFS.c b/source/dnode/vnode/src/vnd/vnodeFS.c deleted file mode 100644 index 5e9f89ccd5..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeFS.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" \ No newline at end of file diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/tdbBufPool.c index 3eed100566..bc3c386b0f 100644 --- a/source/libs/tdb/src/tdbBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -16,9 +16,9 @@ #include "thash.h" #include "tlist.h" -#include "tkvBufPool.h" -#include "tkvDiskMgr.h" -#include "tkvPage.h" +#include "tdbBufPool.h" +#include "tdbDiskMgr.h" +#include "tdbPage.h" struct SFrameIdWrapper { TD_SLIST_NODE(SFrameIdWrapper); From c9118a52e6776db03c854afadc9d4c77f0c9fa82 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 06:13:10 +0000 Subject: [PATCH 33/63] more tkv and refact --- source/dnode/mgmt/impl/src/dndEnv.c | 2 +- source/dnode/vnode/inc/vnode.h | 18 ---------------- source/libs/tdb/inc/tdb.h | 15 +++++++++++-- source/libs/tdb/src/inc/tdbDB.h | 11 +++------- source/libs/tdb/src/inc/tdbEnv.h | 31 --------------------------- source/libs/tdb/test/tDiskMgrTest.cpp | 10 --------- source/libs/tdb/test/tdbTest.cpp | 5 +++++ source/libs/tdb/test/tkvTests.cpp | 0 8 files changed, 22 insertions(+), 70 deletions(-) delete mode 100644 source/libs/tdb/src/inc/tdbEnv.h delete mode 100644 source/libs/tdb/test/tDiskMgrTest.cpp create mode 100644 source/libs/tdb/test/tdbTest.cpp delete mode 100644 source/libs/tdb/test/tkvTests.cpp diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 1bf1ea2b92..23fc643abe 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -293,7 +293,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode since %s", terrstr()); dndCleanup(); - return NULL; + return -1; } memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6113d0a536..7ff02309ec 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -37,36 +37,18 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; - - /** vnode buffer pool options */ struct { - /** write buffer size */ uint64_t wsize; uint64_t ssize; uint64_t lsize; - /** use heap allocator or arena allocator */ bool isHeapAllocator; }; - - /** time to live of tables in this vnode */ uint32_t ttl; - - /** data to keep in this vnode */ uint32_t keep; - - /** if TS data is eventually consistency */ bool isWeak; - - /** TSDB config */ STsdbCfg tsdbCfg; - - /** META config */ SMetaCfg metaCfg; - - /** TQ config */ STqCfg tqCfg; - - /** WAL config */ SWalCfg walCfg; } SVnodeCfg; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 2f47f545b1..40d79de821 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,9 +22,15 @@ extern "C" { #endif +typedef enum { + TDB_BTREE = 0, + TDB_HASH, + TDB_HEAP, +} tdb_db_t; + // Forward declaration -typedef struct TDB TDB; -typedef struct TDB_ENV TDB_ENV; +typedef struct TDB TDB; +typedef struct TDB_CURSOR TDB_CURSOR; // SKey typedef struct { @@ -32,6 +38,11 @@ typedef struct { uint32_t size; } TDB_KEY, TDB_VALUE; +// TDB Operations +int tdbCreateDB(TDB** dbpp); +int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags); +int tdbCloseDB(TDB* dbp, uint32_t flags); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 40ddb1eb31..479ef77711 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -23,19 +23,14 @@ extern "C" { #endif -typedef enum { - TDB_BTREE = 0, - TDB_HASH, - TDB_HEAP, -} tdb_db_t; struct TDB { pgsize_t pageSize; tdb_db_t type; union { - STkvBtree btree; - STkvhash hash; - } dbimpl; + TDB_BTREE btree; + TDB_HASH hash; + } dbam; // Different access methods }; #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h deleted file mode 100644 index f3e4ef5888..0000000000 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_ENV_H_ -#define _TD_TDB_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -struct TDB_ENV { - char *homeDir; -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/tDiskMgrTest.cpp b/source/libs/tdb/test/tDiskMgrTest.cpp deleted file mode 100644 index a735fdb33d..0000000000 --- a/source/libs/tdb/test/tDiskMgrTest.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "gtest/gtest.h" - -#include "iostream" - -#include "tDiskMgr.h" - -TEST(tDiskMgrTest, simple_test) { - // TODO - std::cout << "This is in tDiskMgrTest::simple_test" << std::endl; -} \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp new file mode 100644 index 0000000000..ee91c7669b --- /dev/null +++ b/source/libs/tdb/test/tdbTest.cpp @@ -0,0 +1,5 @@ +#include "gtest/gtest.h" + +TEST(tdb_api_test, tdb_create_open_close_db_test) { + +} \ No newline at end of file diff --git a/source/libs/tdb/test/tkvTests.cpp b/source/libs/tdb/test/tkvTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 From ac87dcf9661c1b2c2793add1219a15dbec2b8ffa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:04:43 +0000 Subject: [PATCH 34/63] more --- source/libs/tdb/CMakeLists.txt | 6 +++++- source/libs/tdb/test/CMakeLists.txt | 0 source/libs/tdb/test/tdbTest.cpp | 11 ++++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 source/libs/tdb/test/CMakeLists.txt diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 1832ea0b5d..68aa40c072 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -14,4 +14,8 @@ target_link_libraries( tdb PUBLIC os PUBLIC util -) \ No newline at end of file +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index ee91c7669b..3ff43fdd69 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,5 +1,14 @@ #include "gtest/gtest.h" -TEST(tdb_api_test, tdb_create_open_close_db_test) { +#include "tdb.h" +TEST(tdb_api_test, tdb_create_open_close_db_test) { + int ret; + TDB *dbp; + + tdbCreateDB(&dbp); + + tdbOpenDB(dbp, TDB_BTREE, 0); + + tdbCloseDB(dbp, 0); } \ No newline at end of file From a20724f46eda6f4dcb9bfeedf5e9c8c93a2370fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:15:28 +0000 Subject: [PATCH 35/63] more --- source/dnode/vnode/src/vnd/vnodeWrite.c | 1 + source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/test/CMakeLists.txt | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 185487757f..01619b5c77 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,6 +83,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } + vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { free(pCreateTbReq->stbCfg.pSchema); free(pCreateTbReq->stbCfg.pTagSchema); diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 68aa40c072..647771fd2d 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -17,5 +17,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - add_subdirectory(test) + # add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index e69de29bb2..2d77c1f4e9 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -0,0 +1,3 @@ +# tdbTest +add_executable(tdbTest "tdbTest.cpp") +target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file From 254e144f048076f7bd051805de397d76b67fa747 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:52:37 +0000 Subject: [PATCH 36/63] more --- source/dnode/vnode/src/tsdb/tsdbWrite.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 570e821af0..5f937f17e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -17,9 +17,11 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp) { // Check if mem is there. If not, create one. - pTsdb->mem = tsdbNewMemTable(pTsdb); if (pTsdb->mem == NULL) { - return -1; + pTsdb->mem = tsdbNewMemTable(pTsdb); + if (pTsdb->mem == NULL) { + return -1; + } } return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); } \ No newline at end of file From 7dbc861b4c130370df8e15f1e0053ee26c7416ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:57:18 +0000 Subject: [PATCH 37/63] more tdb --- source/dnode/vnode/inc/vnode.h | 26 +++++++-------- source/libs/tdb/CMakeLists.txt | 15 +++++---- source/libs/tdb/inc/tdb.h | 16 +++++---- source/libs/tdb/src/db/tdbDB.c | 34 ++++++++++++++++++++ source/libs/tdb/src/{ => dmgr}/tdbDiskMgr.c | 0 source/libs/tdb/src/inc/tdbBtree.h | 2 +- source/libs/tdb/src/inc/tdbDB.h | 2 +- source/libs/tdb/src/inc/tdbDef.h | 9 +++--- source/libs/tdb/src/{ => mpool}/tdbBufPool.c | 0 source/libs/tdb/test/tdbTest.cpp | 2 +- 10 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 source/libs/tdb/src/db/tdbDB.c rename source/libs/tdb/src/{ => dmgr}/tdbDiskMgr.c (100%) rename source/libs/tdb/src/{ => mpool}/tdbBufPool.c (100%) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7ff02309ec..d8de94b204 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -35,28 +35,26 @@ typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { - int32_t vgId; - SDnode *pDnode; - struct { - uint64_t wsize; - uint64_t ssize; - uint64_t lsize; - bool isHeapAllocator; - }; + int32_t vgId; + SDnode * pDnode; + uint64_t wsize; + uint64_t ssize; + uint64_t lsize; + bool isHeapAllocator; uint32_t ttl; uint32_t keep; - bool isWeak; + bool isWeak; STsdbCfg tsdbCfg; SMetaCfg metaCfg; - STqCfg tqCfg; - SWalCfg walCfg; + STqCfg tqCfg; + SWalCfg walCfg; } SVnodeCfg; typedef struct { int32_t sver; - char *timezone; - char *locale; - char *charset; + char * timezone; + char * locale; + char * charset; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeOpt; diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 647771fd2d..eb63f2b144 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,10 +1,11 @@ -aux_source_directory(src TDB_SRC) + +set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr") +foreach(TDB_SUBDIR ${TDB_SUBDIRS}) + aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) +endforeach() + add_library(tdb STATIC ${TDB_SRC}) -# target_include_directories( -# tkv -# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" -# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -# ) + target_include_directories( tdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" @@ -17,5 +18,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - # add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 40d79de821..b804fcfd5e 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,10 +22,14 @@ extern "C" { #endif +#define TDB_EXTERN +#define TDB_PUBLIC +#define TDB_STATIC static + typedef enum { - TDB_BTREE = 0, - TDB_HASH, - TDB_HEAP, + TDB_BTREE_T = 0, + TDB_HASH_T, + TDB_HEAP_T, } tdb_db_t; // Forward declaration @@ -39,9 +43,9 @@ typedef struct { } TDB_KEY, TDB_VALUE; // TDB Operations -int tdbCreateDB(TDB** dbpp); -int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags); -int tdbCloseDB(TDB* dbp, uint32_t flags); +TDB_EXTERN int tdbCreateDB(TDB** dbpp); +TDB_EXTERN int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags); +TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c new file mode 100644 index 0000000000..ac83fc993b --- /dev/null +++ b/source/libs/tdb/src/db/tdbDB.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbDB.h" +#include "tdb.h" + +TDB_EXTERN int tdbCreateDB(TDB** dbpp) { + TDB* dbp; + +// dbp = malloc + return 0; +} + +int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags) { + // TODO + return 0; +} + +int tdbCloseDB(TDB* dbp, uint32_t flags) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/tdbDiskMgr.c b/source/libs/tdb/src/dmgr/tdbDiskMgr.c similarity index 100% rename from source/libs/tdb/src/tdbDiskMgr.c rename to source/libs/tdb/src/dmgr/tdbDiskMgr.c diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index c68f94bb48..f1551e9c1a 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -16,7 +16,7 @@ #ifndef _TD_TDB_BTREE_H_ #define _TD_TDB_BTREE_H_ -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 479ef77711..04f4ba3f33 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -16,6 +16,7 @@ #ifndef _TD_TDB_DB_H_ #define _TD_TDB_DB_H_ +#include "tdb.h" #include "tdbBtree.h" #include "tdbHash.h" @@ -23,7 +24,6 @@ extern "C" { #endif - struct TDB { pgsize_t pageSize; tdb_db_t type; diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdbDef.h index a04b8cc402..4b5e54368b 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdbDef.h @@ -24,16 +24,17 @@ extern "C" { // pgid_t typedef int32_t pgid_t; -#define TKV_IVLD_PGID ((pgid_t)-1) +#define TDB_IVLD_PGID ((pgid_t)-1) // framd_id_t typedef int32_t frame_id_t; // pgsize_t typedef int32_t pgsize_t; -#define TKV_MIN_PGSIZE 512 -#define TKV_MAX_PGSIZE 16384 -#define TKV_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) +#define TDB_MIN_PGSIZE 512 +#define TDB_MAX_PGSIZE 16384 +#define TDB_DEFAULT_PGSIZE 4096 +#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) #ifdef __cplusplus } diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/mpool/tdbBufPool.c similarity index 100% rename from source/libs/tdb/src/tdbBufPool.c rename to source/libs/tdb/src/mpool/tdbBufPool.c diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3ff43fdd69..a456c33149 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -8,7 +8,7 @@ TEST(tdb_api_test, tdb_create_open_close_db_test) { tdbCreateDB(&dbp); - tdbOpenDB(dbp, TDB_BTREE, 0); + tdbOpenDB(dbp, TDB_BTREE_T, 0); tdbCloseDB(dbp, 0); } \ No newline at end of file From b1980e8f44f71d51c7c4184693878ad06f3ab2a9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 13 Jan 2022 17:30:37 +0800 Subject: [PATCH 38/63] feature/qnode --- source/libs/qworker/inc/qworkerInt.h | 19 +-- source/libs/qworker/src/qworker.c | 203 ++++++++++++++++----------- 2 files changed, 130 insertions(+), 92 deletions(-) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 2f47c79065..4030ad82ad 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -70,6 +70,7 @@ typedef struct SQWTaskCtx { SRWLatch lock; int8_t sinkScheduled; int8_t queryScheduled; + bool needRsp; qTaskInfo_t taskHandle; DataSinkHandle sinkHandle; @@ -99,7 +100,7 @@ typedef struct SQWorkerMgmt { #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) -#define QW_TASK_READY_RESP(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED) +#define QW_TASK_READY(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED) #define QW_SET_QTID(id, qId, tId) do { *(uint64_t *)(id) = (qId); *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); } while (0) #define QW_GET_QTID(id, qId, tId) do { (qId) = *(uint64_t *)(id); (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); } while (0) #define QW_IDS() sId, qId, tId @@ -111,16 +112,16 @@ typedef struct SQWorkerMgmt { #define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) #define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) -#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64 param, mgmt, sId, __VA_ARGS__) -#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64 param, mgmt, sId, __VA_ARGS__) +#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) +#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64 param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64 param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64 param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64 param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 921298995f..1be190065a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -221,7 +221,7 @@ int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t code = 0; QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &tsch)); - QW_ERR_JRET(qwAddTaskImpl(mgmt, tsch, 0, qId, tId, JOB_TASK_STATUS_NOT_START, QW_EXIST_RET_ERR, NULL)); + QW_ERR_JRET(qwAddTaskImpl(mgmt, tsch, 0, qId, tId, status, QW_EXIST_RET_ERR, NULL)); _return: @@ -557,6 +557,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ } QW_UNLOCK(QW_WRITE, &task->lock); + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -778,30 +779,35 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe return TSDB_CODE_SUCCESS; } -int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) { +int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); if (QW_READY_NOT_RECEIVED == task->ready) { + QW_SCH_TASK_DLOG("ready not received, ready:%d", task->ready); + goto _return; + } else if (QW_READY_RECEIVED == task->ready) { + task->ready = QW_READY_RESPONSED; + int32_t rspCode = task->code; + QW_UNLOCK(QW_WRITE, &task->lock); - qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - return TSDB_CODE_SUCCESS; - } else if (QW_READY_RECEIVED == task->ready) { - QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, rspCode)); + QW_ERR_RET(qwBuildAndSendReadyRsp(pMsg, rspCode)); + + QW_SCH_TASK_DLOG("ready response sent, ready:%d", task->ready); - task->ready = QW_READY_RESPONSED; + return TSDB_CODE_SUCCESS; } else if (QW_READY_RESPONSED == task->ready) { - qError("query response already send"); + QW_SCH_TASK_ELOG("ready response already send, ready:%d", task->ready); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } else { assert(0); @@ -812,7 +818,6 @@ _return: if (task) { QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); - } qwReleaseScheduler(QW_READ, mgmt); @@ -820,34 +825,39 @@ _return: QW_RET(code); } -int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { +int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); - if (QW_TASK_READY_RESP(task->status)) { - QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, task->code)); + int8_t status = task->status; + int32_t errCode = task->code; + + if (QW_TASK_READY(status)) { task->ready = QW_READY_RESPONSED; + + QW_UNLOCK(QW_WRITE, &task->lock); + + QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, errCode)); + + QW_SCH_TASK_DLOG("task ready responsed, status:%d", status); } else { task->ready = QW_READY_RECEIVED; + QW_UNLOCK(QW_WRITE, &task->lock); - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - return TSDB_CODE_SUCCESS; + QW_SCH_TASK_DLOG("task ready NOT responsed, status:%d", status); } _return: if (task) { - QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); } @@ -872,22 +882,15 @@ int32_t qwCheckAndProcessTaskDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId qwReleaseScheduler(QW_READ, mgmt); return TSDB_CODE_SUCCESS; } - - QW_LOCK(QW_READ, &task->lock); - if ((!task->cancel) && (!task->drop)) { - QW_TASK_ELOG("no cancel or drop but task exists, status:%d", task->status); - - QW_UNLOCK(QW_READ, &task->lock); - + if ((!atomic_load_8(&task->cancel)) && (!atomic_load_8(&task->drop))) { + QW_TASK_ELOG("no cancel or drop but task exists, status:%d", atomic_load_8(&task->status)); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - QW_UNLOCK(QW_READ, &task->lock); - *needStop = true; - if (task->cancel) { + if (atomic_load_8(&task->cancel)) { QW_LOCK(QW_WRITE, &task->lock); code = qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); QW_UNLOCK(QW_WRITE, &task->lock); @@ -929,13 +932,15 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 QW_ERR_RET(code); } + QW_LOCK(QW_WRITE, &task->lock); + if (task->cancel) { - QW_LOCK(QW_WRITE, &task->lock); qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS()); - QW_UNLOCK(QW_WRITE, &task->lock); } if (task->drop) { + QW_UNLOCK(QW_WRITE, &task->lock); + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -944,12 +949,12 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } - if ((!(task->cancel || task->drop)) && status > 0) { - QW_LOCK(QW_WRITE, &task->lock); + if (!(task->cancel || task->drop)) { qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); task->code = errCode; - QW_UNLOCK(QW_WRITE, &task->lock); } + + QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -995,24 +1000,24 @@ int32_t qwScheduleDataSink(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId return TSDB_CODE_SUCCESS; } -int32_t qwScheduleQuery(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { +int32_t qwScheduleQuery(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { if (atomic_load_8(&handles->queryScheduled)) { - qDebug("query already scheduled"); + QW_SCH_TASK_ELOG("query already scheduled, queryScheduled:%d", handles->queryScheduled); return TSDB_CODE_SUCCESS; } - QW_ERR_RET(qwUpdateTaskStatus(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_EXECUTING)); + QW_ERR_RET(qwUpdateTaskStatus(mgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING)); SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { - qError("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); + QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } req->header.vgId = mgmt->nodeId; req->sId = sId; - req->queryId = queryId; - req->taskId = taskId; + req->queryId = qId; + req->taskId = tId; SRpcMsg pNewMsg = { .handle = pMsg->handle, @@ -1025,20 +1030,21 @@ int32_t qwScheduleQuery(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, u int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); if (TSDB_CODE_SUCCESS != code) { - qError("put query continue msg to queue failed, code:%x", code); + QW_SCH_TASK_ELOG("put query continue msg to queue failed, code:%x", code); rpcFreeCont(req); QW_ERR_RET(code); } + handles->queryScheduled = true; - qDebug("put query continue msg to query queue"); + QW_SCH_TASK_DLOG("put query continue msg to query queue, vgId:%d", mgmt->nodeId); return TSDB_CODE_SUCCESS; } -int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { +int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -1049,25 +1055,29 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; SQWTaskCtx *handles = NULL; + int8_t status = 0; - QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, mgmt, queryId, taskId, &handles)); - if (atomic_load_8(&handles->needRsp)) { - qError("last fetch not responsed"); + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, mgmt, qId, tId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + + if (handles->needRsp) { + QW_UNLOCK(QW_WRITE, &handles->lock); + QW_SCH_TASK_ELOG("last fetch not responsed, needRsp:%d", handles->needRsp); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)); + QW_UNLOCK(QW_WRITE, &handles->lock); - QW_LOCK(QW_READ, &task->lock); + QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); if (task->cancel || task->drop) { - qError("task is already cancelled or dropped"); + QW_SCH_TASK_ELOG("task is already cancelled or dropped, cancel:%d, drop:%d", task->cancel, task->drop); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - qError("invalid status %d for fetch", task->status); + QW_SCH_TASK_ELOG("invalid status %d for fetch", task->status); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -1075,6 +1085,9 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 if (dataLength > 0) { SOutputData output = {0}; + + QW_SCH_TASK_DLOG("task got data in sink, dataLength:%d", dataLength); + QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp)); output.pData = rsp->data; @@ -1095,27 +1108,38 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) { rsp->completed = 1; - QW_ERR_JRET(qwUpdateTaskStatus(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED)); + status = JOB_TASK_STATUS_SUCCEED; + + QW_SCH_TASK_DLOG("task all fetched, status:%d", status); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS())); } // Note: schedule data sink firstly and will schedule query after it's done if (output.needSchedule) { - QW_ERR_JRET(qwScheduleDataSink(handles, mgmt, sId, queryId, taskId, pMsg)); - } else if ((!output.queryEnd) && (DS_BUF_LOW == output.bufStatus || DS_BUF_EMPTY == output.bufStatus)) { - QW_ERR_JRET(qwScheduleQuery(handles, mgmt, sId, queryId, taskId, pMsg)); + QW_SCH_TASK_DLOG("sink need schedule, queryEnd:%d", output.queryEnd); + QW_ERR_JRET(qwScheduleDataSink(handles, mgmt, sId, qId, tId, pMsg)); + } else if ((!output.queryEnd) && (DS_BUF_LOW == output.bufStatus || DS_BUF_EMPTY == output.bufStatus)) { + QW_SCH_TASK_DLOG("task not end, need to continue, bufStatus:%d", output.bufStatus); + QW_ERR_JRET(qwScheduleQuery(handles, mgmt, sId, qId, tId, pMsg)); } } else { if (dataLength < 0) { - qError("invalid length from dsGetDataLength, length:%d", dataLength); + QW_SCH_TASK_ELOG("invalid length from dsGetDataLength, length:%d", dataLength); QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } if (queryEnd) { - QW_ERR_JRET(qwUpdateTaskStatus(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED)); + status = JOB_TASK_STATUS_SUCCEED; + + QW_SCH_TASK_DLOG("no data in sink and query end, dataLength:%d", dataLength); + + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS())); } else { assert(0 == handles->needRsp); + + // MUST IN SCHEDULE OR IN SINK SCHEDULE - qDebug("no res data in sink, need response later"); + QW_SCH_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd); QW_LOCK(QW_WRITE, &handles->lock); handles->needRsp = true; @@ -1128,7 +1152,6 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 _return: if (task) { - QW_UNLOCK(QW_READ, &task->lock); qwReleaseTask(QW_READ, sch); } @@ -1212,10 +1235,10 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; bool queryRsped = false; bool needStop = false; - bool taskAdded = false; struct SSubplan *plan = NULL; SSubQueryMsg *msg = pMsg->pCont; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + int32_t rspCode = 0; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen); @@ -1237,6 +1260,8 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); } + + QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING)); code = qStringToSubplan(msg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { @@ -1248,53 +1273,49 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); - QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_FAILED)); QW_ERR_JRET(code); } - QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING)); - - taskAdded = true; - QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); queryRsped = true; DataSinkHandle sinkHandle = NULL; code = qExecTask(pTaskInfo, &sinkHandle); - if (code) { QW_TASK_ELOG("qExecTask failed, code:%x", code); QW_ERR_JRET(code); } QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); - QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); _return: - if (queryRsped) { - code = qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg, code); - } else { - code = qwBuildAndSendQueryRsp(pMsg, code); + if (code) { + rspCode = code; + } + + if (!queryRsped) { + code = qwBuildAndSendQueryRsp(pMsg, rspCode); + if (TSDB_CODE_SUCCESS == rspCode && code) { + rspCode = code; + } } int8_t status = 0; - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != rspCode) { status = JOB_TASK_STATUS_FAILED; } else { status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } - if (!taskAdded) { - qwAddTask(qWorkerMgmt, sId, qId, tId, status); - - status = -1; - } + qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, rspCode); - qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, code); + if (queryRsped) { + qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg); + } - QW_RET(code); + QW_RET(rspCode); } int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { @@ -1306,19 +1327,27 @@ int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *p SQWTaskCtx *handles = NULL; QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); qTaskInfo_t taskHandle = handles->taskHandle; DataSinkHandle sinkHandle = handles->sinkHandle; - bool needRsp = handles->needRsp; + QW_UNLOCK(QW_WRITE, &handles->lock); qwReleaseTaskResCache(QW_READ, qWorkerMgmt); QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, req->sId, req->queryId, req->taskId, &needStop)); if (needStop) { qWarn("task need stop"); - if (needRsp) { + + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + if (handles->needRsp) { qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); + handles->needRsp = false; } + QW_UNLOCK(QW_WRITE, &handles->lock); + qwReleaseTaskResCache(QW_READ, qWorkerMgmt); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); } @@ -1336,10 +1365,18 @@ int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *p _return: - if (needRsp) { + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + + if (handles->needRsp) { code = qwBuildAndSendQueryRsp(pMsg, code); + handles->needRsp = false; } - + handles->queryScheduled = false; + + QW_UNLOCK(QW_WRITE, &handles->lock); + qwReleaseTaskResCache(QW_READ, qWorkerMgmt); + if (TSDB_CODE_SUCCESS != code) { status = JOB_TASK_STATUS_FAILED; } else { From c8557afd447afb6d8f0d3e8fc0cdd0ef5a517ccc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 09:31:36 +0000 Subject: [PATCH 39/63] more tdb --- source/libs/tdb/inc/tdb.h | 4 +-- source/libs/tdb/src/db/tdbDB.c | 41 +++++++++++++++++++++++++++--- source/libs/tdb/src/inc/tdbBtree.h | 2 ++ source/libs/tdb/src/inc/tdbDB.h | 6 +++-- source/libs/tdb/src/inc/tdbHash.h | 8 +++--- source/libs/tdb/src/inc/tdbHeap.h | 35 +++++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 4 +-- 7 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 source/libs/tdb/src/inc/tdbHeap.h diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index b804fcfd5e..905b08ee0b 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -43,8 +43,8 @@ typedef struct { } TDB_KEY, TDB_VALUE; // TDB Operations -TDB_EXTERN int tdbCreateDB(TDB** dbpp); -TDB_EXTERN int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags); +TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type); +TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags); TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); #ifdef __cplusplus diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c index ac83fc993b..2af40d8642 100644 --- a/source/libs/tdb/src/db/tdbDB.c +++ b/source/libs/tdb/src/db/tdbDB.c @@ -16,19 +16,52 @@ #include "tdbDB.h" #include "tdb.h" -TDB_EXTERN int tdbCreateDB(TDB** dbpp) { +TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) { TDB* dbp; + int ret; -// dbp = malloc + dbp = calloc(1, sizeof(*dbp)); + if (dbp == NULL) { + return -1; + } + + dbp->pageSize = TDB_DEFAULT_PGSIZE; + dbp->type = type; + + switch (type) { + case TDB_BTREE_T: + // ret = tdbInitBtreeDB(dbp); + // if (ret < 0) goto _err; + break; + case TDB_HASH_T: + // ret = tdbInitHashDB(dbp); + // if (ret < 0) goto _err; + break; + case TDB_HEAP_T: + // ret = tdbInitHeapDB(dbp); + // if (ret < 0) goto _err; + break; + default: + break; + } + + *dbpp = dbp; + return 0; + +_err: + if (dbp) { + free(dbp); + } + *dbpp = NULL; return 0; } -int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags) { +TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags) { // TODO return 0; } -int tdbCloseDB(TDB* dbp, uint32_t flags) { +TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index f1551e9c1a..28258b8e60 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -26,6 +26,8 @@ typedef struct { pgid_t root; // root page number } TDB_BTREE; +TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 04f4ba3f33..e6b49c94ec 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -19,6 +19,7 @@ #include "tdb.h" #include "tdbBtree.h" #include "tdbHash.h" +#include "tdbHeap.h" #ifdef __cplusplus extern "C" { @@ -28,8 +29,9 @@ struct TDB { pgsize_t pageSize; tdb_db_t type; union { - TDB_BTREE btree; - TDB_HASH hash; + TDB_BTREE *btree; + TDB_HASH * hash; + TDB_HEAP * heap; } dbam; // Different access methods }; diff --git a/source/libs/tdb/src/inc/tdbHash.h b/source/libs/tdb/src/inc/tdbHash.h index fca19035f1..8219bda2f8 100644 --- a/source/libs/tdb/src/inc/tdbHash.h +++ b/source/libs/tdb/src/inc/tdbHash.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_HAHS_H_ -#define _TD_TKV_HAHS_H_ +#ifndef _TD_TDB_HASH_H_ +#define _TD_TDB_HASH_H_ #include "tdbDef.h" @@ -26,8 +26,10 @@ typedef struct { // TODO } TDB_HASH; +TDB_PUBLIC int tdbInitHashDB(TDB *dbp); + #ifdef __cplusplus } #endif -#endif /*_TD_TKV_HAHS_H_*/ \ No newline at end of file +#endif /*_TD_TDB_HASH_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHeap.h b/source/libs/tdb/src/inc/tdbHeap.h new file mode 100644 index 0000000000..25a812fa5f --- /dev/null +++ b/source/libs/tdb/src/inc/tdbHeap.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_HEAP_H_ +#define _TD_TDB_HEAP_H_ + +#include "tdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + // TODO +} TDB_HEAP; + +TDB_PUBLIC int tdbInitHeapDB(TDB *dbp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_HEAP_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index a456c33149..38c3b0b917 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,9 +6,9 @@ TEST(tdb_api_test, tdb_create_open_close_db_test) { int ret; TDB *dbp; - tdbCreateDB(&dbp); + tdbCreateDB(&dbp, TDB_BTREE_T); - tdbOpenDB(dbp, TDB_BTREE_T, 0); + tdbOpenDB(dbp, 0); tdbCloseDB(dbp, 0); } \ No newline at end of file From 4d32cfffb90ce4ba681a6608c1178e8dc747e7f1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 09:47:45 +0000 Subject: [PATCH 40/63] more config --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b258a2a252..cb05c84d72 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -26,7 +26,8 @@ "eamodio.gitlens", "matepek.vscode-catch2-test-adapter", "spmeesseman.vscode-taskexplorer", - "cschlosser.doxdocgen" + "cschlosser.doxdocgen", + "urosvujosevic.explorer-manager" ], // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], From 5d1238010ce87c78fa519a18c8c3d0ee67f4ed62 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jan 2022 19:01:28 +0800 Subject: [PATCH 41/63] [td-11818] fix bug in select * from super_table. --- include/libs/planner/planner.h | 1 + source/client/test/clientTests.cpp | 1054 ++++++++++---------- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 98 +- source/libs/parser/src/astValidate.c | 2 + source/libs/planner/src/physicalPlan.c | 4 +- source/libs/planner/src/physicalPlanJson.c | 32 +- source/libs/planner/src/planner.c | 6 +- 8 files changed, 631 insertions(+), 568 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index d98c1ca595..4f1d98d1e3 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -95,6 +95,7 @@ typedef struct SScanPhyNode { int8_t tableType; int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC int32_t count; // repeat count + int32_t reverse; // reverse scan count } SScanPhyNode; typedef SScanPhyNode SSystemTableScanPhyNode; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index f6bda9abcf..7086da0d55 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -48,477 +48,226 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); } -TEST(testCase, connect_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); - } - taos_close(pConn); -} - -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop database, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database if not exists abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("failed to create database abc1, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - pRes = taos_query(pConn, "drop dnode 4"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - - TEST(testCase, drop_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - showDB(pConn); - - TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - showDB(pConn); - - pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("create to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - taos_close(pConn); - return; - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - for (int32_t i = 0; i < 20; ++i) { - char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - (i + 1) * 30, (i + 2) * 40); - TAOS_RES* pres = taos_query(pConn, sql); - if (taos_errno(pres) != 0) { - printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - } - taos_free_result(pres); - } - - taos_close(pConn); -} - -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, generated_request_id_test) { - SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - for (int32_t i = 0; i < 50000; ++i) { - uint64_t v = generateRequestId(); - void* result = taosHashGet(phash, &v, sizeof(v)); - if (result != nullptr) { - printf("0x%lx, index:%d\n", v, i); - } - assert(result == nullptr); - taosHashPut(phash, &v, sizeof(v), NULL, 0); - } - - taosHashCleanup(phash); -} - -// TEST(testCase, create_topic_Test) { +//TEST(testCase, connect_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// if (pConn == NULL) { +// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop database, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database if not exists abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create database abc1, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// pRes = taos_query(pConn, "drop dnode 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} +// +// TEST(testCase, drop_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// showDB(pConn); +// +// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// showDB(pConn); +// +// pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); // if (taos_errno(pRes) != 0) { // printf("error in create db, reason:%s\n", taos_errstr(pRes)); // } @@ -542,61 +291,340 @@ TEST(testCase, generated_request_id_test) { // ASSERT_EQ(numOfFields, 0); // // taos_free_result(pRes); +// taos_close(pConn); +//} // -// char* sql = "select * from st1"; -// tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// for (int32_t i = 0; i < 20; ++i) { +// char sql[512] = {0}; +// snprintf(sql, tListLen(sql), +// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, +// (i + 1) * 30, (i + 2) * 40); +// TAOS_RES* pres = taos_query(pConn, sql); +// if (taos_errno(pres) != 0) { +// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); +// } +// taos_free_result(pres); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, show_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, generated_request_id_test) { +// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +// +// for (int32_t i = 0; i < 50000; ++i) { +// uint64_t v = generateRequestId(); +// void* result = taosHashGet(phash, &v, sizeof(v)); +// if (result != nullptr) { +// printf("0x%lx, index:%d\n", v, i); +// } +// assert(result == nullptr); +// taosHashPut(phash, &v, sizeof(v), NULL, 0); +// } +// +// taosHashCleanup(phash); +//} +// +//// TEST(testCase, create_topic_Test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "use abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +//// if (taos_errno(pRes) != 0) { +//// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +//// } +//// +//// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +//// ASSERT_TRUE(pFields == NULL); +//// +//// int32_t numOfFields = taos_num_fields(pRes); +//// ASSERT_EQ(numOfFields, 0); +//// +//// taos_free_result(pRes); +//// +//// char* sql = "select * from st1"; +//// tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +//// taos_close(pConn); +////} +// +//TEST(testCase, insert_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, projection_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// for(int32_t i = 0; i < 100; ++i) { +// char sql[512] = {0}; +// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); +// TAOS_RES* p = taos_query(pConn, sql); +// if (taos_errno(p) != 0) { +// printf("failed to insert data, reason:%s\n", taos_errstr(p)); +// } +// +// taos_free_result(p); +// } +// +// pRes = taos_query(pConn, "select * from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); // taos_close(pConn); //} -TEST(testCase, insert_test) { +TEST(testCase, projection_query_stables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, projection_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - for(int32_t i = 0; i < 10000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } - - taos_free_result(p); - } - - pRes = taos_query(pConn, "select * from tu"); + pRes = taos_query(pConn, "select ts,k from m1"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d76f270392..9ccc7992f3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -542,7 +542,7 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0dfcc9b1d8..b781615bae 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4844,7 +4844,16 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + if (pInfo == NULL || pOperator == NULL) { + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; @@ -4852,7 +4861,6 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in pInfo->current = 0; pInfo->scanFlag = MAIN_SCAN; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableScanOperator"; pOperator->operatorType = OP_TableScan; pOperator->blockingOptr = false; @@ -4866,6 +4874,39 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in return pOperator; } +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { + assert(repeatTime > 0); + + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + tfree(pInfo); + tfree(pOperator); + + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pInfo->pTsdbReadHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; + + pOperator->name = "DataBlocksOptimizedScanOperator"; + pOperator->operatorType = OP_DataBlocksOptScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = NULL; + pOperator->exec = doTableScan; + pOperator->pTaskInfo = pTaskInfo; + + return pOperator; +} + SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -4973,27 +5014,6 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf } -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { - assert(repeatTime > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->current = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - - SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); - pOptr->name = "DataBlocksOptimizedScanOperator"; -// pOptr->operatorType = OP_DataBlocksOptScan; - pOptr->pRuntimeEnv = pRuntimeEnv; - pOptr->blockingOptr = false; - pOptr->info = pInfo; - pOptr->exec = doTableScan; - - return pOptr; -} - SArray* getOrderCheckColumns(STaskAttr* pQuery) { int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); @@ -7187,10 +7207,13 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*) pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - SOperatorInfo* pOperatorInfo = createTableScanOperator(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); - pTaskInfo->pRoot = pOperatorInfo; + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + return createTableScanOperator(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); + } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + return createDataBlocksOptScanInfo(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else { assert(0); } @@ -7199,21 +7222,20 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { STsdbQueryCond cond = {.loadExternalRows = false}; - cond.twindow.skey = INT64_MIN; - cond.twindow.ekey = INT64_MAX; uint64_t uid = 0; SPhyNode* pPhyNode = pPlan->pNode; - if (pPhyNode->info.type == OP_TableScan) { + if (pPhyNode->info.type == OP_TableScan || pPhyNode->info.type == OP_DataBlocksOptScan) { - SScanPhyNode* pScanNode = (SScanPhyNode*) pPhyNode; - uid = pScanNode->uid; - cond.order = pScanNode->order; - cond.numOfCols = taosArrayGetSize(pScanNode->node.pTargets); + STableScanPhyNode* pTableScanNode = (STableScanPhyNode*) pPhyNode; + uid = pTableScanNode->scan.uid; + cond.order = pTableScanNode->scan.order; + cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + cond.twindow = pTableScanNode->window; for(int32_t i = 0; i < cond.numOfCols; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pScanNode->node.pTargets, i); + SExprInfo* pExprInfo = taosArrayGetP(pTableScanNode->scan.node.pTargets, i); assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE); SSchema* pSchema = pExprInfo->pExpr->pSchema; @@ -7235,7 +7257,11 @@ int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* r *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); - doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + if ((*pTaskInfo)->pRoot == NULL) { + return terrno; + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 5d56a88c14..512204b40b 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -917,6 +917,8 @@ int32_t validateLimitNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBu return buildInvalidOperationMsg(pMsgBuf, msg1); } } + + return TSDB_CODE_SUCCESS; } int32_t validateOrderbyNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf* pMsgBuf) { diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 71e69c67d2..7371ec6cfa 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -262,8 +262,8 @@ static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) { } static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { - SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList; - for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { + SVgroupsInfo* pVgroupList = pTable->pMeta->vgroupList; + for (int32_t i = 0; i < pVgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); subplan->msgType = TDMT_VND_QUERY; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 7e36e0e124..857df63b5a 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -530,32 +530,38 @@ static const char* jkScanNodeTableId = "TableId"; static const char* jkScanNodeTableType = "TableType"; static const char* jkScanNodeTableOrder = "Order"; static const char* jkScanNodeTableCount = "Count"; +static const char* jkScanNodeTableRevCount = "Reverse"; static bool scanNodeToJson(const void* obj, cJSON* json) { - const SScanPhyNode* scan = (const SScanPhyNode*)obj; - bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); + const SScanPhyNode* pNode = (const SScanPhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, pNode->uid); if (res) { - res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); + res = cJSON_AddNumberToObject(json, jkScanNodeTableType, pNode->tableType); } if (res) { - res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); + res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, pNode->order); } if (res) { - res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); + res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, pNode->count); + } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableRevCount, pNode->reverse); } return res; } static bool scanNodeFromJson(const cJSON* json, void* obj) { - SScanPhyNode* scan = (SScanPhyNode*)obj; - scan->uid = getNumber(json, jkScanNodeTableId); - scan->tableType = getNumber(json, jkScanNodeTableType); - scan->count = getNumber(json, jkScanNodeTableCount); - scan->order = getNumber(json, jkScanNodeTableOrder); + SScanPhyNode* pNode = (SScanPhyNode*)obj; + pNode->uid = getNumber(json, jkScanNodeTableId); + pNode->tableType = getNumber(json, jkScanNodeTableType); + pNode->count = getNumber(json, jkScanNodeTableCount); + pNode->order = getNumber(json, jkScanNodeTableOrder); + pNode->reverse = getNumber(json, jkScanNodeTableRevCount); return true; } @@ -910,7 +916,7 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); + size_t size = MAX(sizeof(SPhyNode), sizeof(STableScanPhyNode)); res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); } if (res) { @@ -940,8 +946,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); -// printf("====Physical plan:====\n") -// printf("%s\n", *str); + printf("====Physical plan:====\n"); + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 21f57d95d5..3047ef4f5a 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From 751512f4567fc5846eec2f7d191507e542138e1d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jan 2022 19:05:11 +0800 Subject: [PATCH 42/63] [td-11818] refactor. --- source/libs/planner/src/physicalPlanJson.c | 4 ++-- source/libs/planner/src/planner.c | 6 +++--- source/libs/transport/src/rpcTcp.c | 7 +++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 857df63b5a..e249ca4066 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -946,8 +946,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); - printf("====Physical plan:====\n"); - printf("%s\n", *str); +// printf("====Physical plan:====\n"); +// printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 3047ef4f5a..21f57d95d5 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 9fa51a6fdc..59d223153a 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -413,8 +413,11 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, - ip, port, localPort, pFdObj, pThreadObj->numOfFds); + + char ipport[40] = {0}; + taosIpPort2String(ip, port, ipport); + tDebug("%s %p TCP connection to %s is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, + ipport, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); taosCloseSocket(fd); From 6a0b6671cdb576ac40b2c49217d2b2893357a83a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 13 Jan 2022 06:24:22 -0500 Subject: [PATCH 43/63] TD-12678 single table aggregation physical plan code --- include/libs/planner/planner.h | 19 +++ include/libs/planner/plannerOp.h | 2 +- source/libs/planner/src/physicalPlan.c | 25 +++- source/libs/planner/src/physicalPlanJson.c | 153 ++++++++++++++++----- source/libs/planner/test/phyPlanTests.cpp | 32 ++++- 5 files changed, 192 insertions(+), 39 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index d98c1ca595..4f1db37f79 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -119,6 +119,25 @@ typedef struct SExchangePhyNode { SArray *pSrcEndPoints; // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; +typedef enum EAggAlgo { + AGG_ALGO_PLAIN = 1, // simple agg across all input rows + AGG_ALGO_SORTED, // grouped agg, input must be sorted + AGG_ALGO_HASHED // grouped agg, use internal hashtable +} EAggAlgo; + +typedef enum EAggSplit { + AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result + AGG_SPLIT_FINAL // second level agg, must calculate the final result +} EAggSplit; + +typedef struct SAggPhyNode { + SPhyNode node; + EAggAlgo aggAlgo; // algorithm used by agg operator + EAggSplit aggSplit; // distributed splitting mode + SArray *pExprs; // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function + SArray *pGroupByList; // SColIndex list, but these must be column node +} SAggPhyNode; + typedef struct SSubplanId { uint64_t queryId; uint64_t templateId; diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 41d6e028cf..5cc896f1c2 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -30,7 +30,7 @@ OP_ENUM_MACRO(TagScan) OP_ENUM_MACRO(SystemTableScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) -OP_ENUM_MACRO(Groupby) +// OP_ENUM_MACRO(Groupby) OP_ENUM_MACRO(Limit) OP_ENUM_MACRO(SLimit) OP_ENUM_MACRO(TimeWindow) diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 71e69c67d2..65c97110ce 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -189,7 +189,6 @@ static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableI return (SPhyNode*)node; } - static bool isSystemTable(SQueryTableInfo* pTable) { // todo return false; @@ -295,13 +294,31 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; - if (needMultiNodeScan(pTable)) { return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan); } +static SPhyNode* createSingleTableAgg(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SAggPhyNode* node = (SAggPhyNode*)initPhyNode(pPlanNode, OP_Aggregate, sizeof(SAggPhyNode)); + SGroupbyExpr* pGroupBy = (SGroupbyExpr*)pPlanNode->pExtInfo; + node->aggAlgo = AGG_ALGO_PLAIN; + node->aggSplit = AGG_SPLIT_FINAL; + if (NULL != pGroupBy) { + node->aggAlgo = AGG_ALGO_HASHED; + node->pGroupByList = validPointer(taosArrayDup(pGroupBy->columnInfo)); + } + return (SPhyNode*)node; +} + +static SPhyNode* createAggNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + // if (needMultiNodeAgg(pPlanNode)) { + + // } + return createSingleTableAgg(pCxt, pPlanNode); +} + static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SPhyNode* node = NULL; switch (pPlanNode->info.type) { @@ -311,6 +328,10 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; + case QNODE_AGGREGATE: + case QNODE_GROUPBY: + node = createAggNode(pCxt, pPlanNode); + break; case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 7e36e0e124..123364134d 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -20,6 +20,19 @@ typedef bool (*FToJson)(const void* obj, cJSON* json); typedef bool (*FFromJson)(const cJSON* json, void* obj); +static char* getString(const cJSON* json, const char* name) { + char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); + return strdup(p); +} + +static void copyString(const cJSON* json, const char* name, char* dst) { + strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); +} + +static int64_t getNumber(const cJSON* json, const char* name) { + return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); +} + static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) { if (NULL == obj) { return true; @@ -62,6 +75,39 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f return func(jObj, *obj); } +static const char* jkPnodeType = "Type"; +static int32_t getPnodeTypeSize(cJSON* json) { + switch (getNumber(json, jkPnodeType)) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return sizeof(STableScanPhyNode); + case OP_TagScan: + return sizeof(STagScanPhyNode); + case OP_SystemTableScan: + return sizeof(SSystemTableScanPhyNode); + case OP_Aggregate: + return sizeof(SAggPhyNode); + case OP_Exchange: + return sizeof(SExchangePhyNode); + default: + break; + }; + return -1; +} + +static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void** obj) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return true; + } + *obj = calloc(1, getPnodeTypeSize(jObj)); + if (NULL == *obj) { + return false; + } + return func(jObj, *obj); +} + static bool addTarray(cJSON* json, const char* name, FToJson func, const SArray* array, bool isPoint) { size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); if (size > 0) { @@ -154,26 +200,9 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - if (*array == NULL) { - *array = calloc(*size, itemSize); - } - - return fromItem(jArray, func, *array, itemSize, *size); -} - -static char* getString(const cJSON* json, const char* name) { - char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); - return strdup(p); -} - -static void copyString(const cJSON* json, const char* name, char* dst) { - strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); -} - -static int64_t getNumber(const cJSON* json, const char* name) { - return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); + return fromItem(jArray, func, array, itemSize, *size); } static const char* jkSchemaType = "Type"; @@ -221,7 +250,7 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->precision = getNumber(json, jkDataBlockSchemaPrecision); - return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); + return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); } static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; @@ -534,19 +563,15 @@ static const char* jkScanNodeTableCount = "Count"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* scan = (const SScanPhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); } - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); } - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); } - return res; } @@ -559,6 +584,66 @@ static bool scanNodeFromJson(const cJSON* json, void* obj) { return true; } +static const char* jkColIndexColId = "ColId"; +static const char* jkColIndexColIndex = "ColIndex"; +static const char* jkColIndexFlag = "Flag"; +static const char* jkColIndexName = "Name"; + +static bool colIndexToJson(const void* obj, cJSON* json) { + const SColIndex* col = (const SColIndex*)obj; + bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId); + if (res) { + res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex); + } + if (res) { + res = cJSON_AddNumberToObject(json, jkColIndexFlag, col->flag); + } + if (res) { + res = cJSON_AddStringToObject(json, jkColIndexName, col->name); + } + return res; +} + +static bool colIndexFromJson(const cJSON* json, void* obj) { + SColIndex* col = (SColIndex*)obj; + col->colId = getNumber(json, jkColIndexColId); + col->colIndex = getNumber(json, jkColIndexColIndex); + col->flag = getNumber(json, jkColIndexFlag); + copyString(json, jkColIndexName, col->name); + return true; +} + +static const char* jkAggNodeAggAlgo = "AggAlgo"; +static const char* jkAggNodeAggSplit = "AggSplit"; +static const char* jkAggNodeExprs = "Exprs"; +static const char* jkAggNodeGroupByList = "GroupByList"; + +static bool aggNodeToJson(const void* obj, cJSON* json) { + const SAggPhyNode* agg = (const SAggPhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo); + if (res) { + res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit); + } + if (res) { + res = addArray(json, jkAggNodeExprs, exprInfoToJson, agg->pExprs); + } + if (res) { + res = addArray(json, jkAggNodeGroupByList, colIndexToJson, agg->pGroupByList); + } + return res; +} + +static bool aggNodeFromJson(const cJSON* json, void* obj) { + SAggPhyNode* agg = (SAggPhyNode*)obj; + agg->aggAlgo = getNumber(json, jkAggNodeAggAlgo); + agg->aggSplit = getNumber(json, jkAggNodeAggSplit); + bool res = fromArray(json, jkAggNodeExprs, exprInfoFromJson, &agg->pExprs, sizeof(SExprInfo)); + if (res) { + res = fromArray(json, jkAggNodeGroupByList, colIndexFromJson, &agg->pGroupByList, sizeof(SExprInfo)); + } + return res; +} + static const char* jkTableScanNodeFlag = "Flag"; static const char* jkTableScanNodeWindow = "Window"; static const char* jkTableScanNodeTagsConditions = "TagsConditions"; @@ -667,10 +752,10 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { case OP_SystemTableScan: return scanNodeToJson(obj, json); case OP_Aggregate: - break; // todo + return aggNodeToJson(obj, json); case OP_Project: return true; - case OP_Groupby: + // case OP_Groupby: case OP_Limit: case OP_SLimit: case OP_TimeWindow: @@ -708,7 +793,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { break; // todo case OP_Project: return true; - case OP_Groupby: + // case OP_Groupby: case OP_Limit: case OP_SLimit: case OP_TimeWindow: @@ -735,12 +820,15 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; -static const char* jkPnodeSchema = "InputSchema"; +static const char* jkPnodeSchema = "TargetSchema"; static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { const SPhyNode* phyNode = (const SPhyNode*)obj; - bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type); + if (res) { + res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + } if (res) { res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets); } @@ -762,8 +850,8 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { static bool phyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* node = (SPhyNode*) obj; - node->info.name = getString(json, jkPnodeName); - node->info.type = opNameToOpType(node->info.name); + node->info.type = getNumber(json, jkPnodeType); + node->info.name = opTypeToOpName(node->info.type); bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); if (res) { @@ -910,8 +998,7 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); - res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); + res = fromPnode(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode); } if (res) { res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 29f6e48dc7..6d9e08e829 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -30,6 +30,21 @@ void* myCalloc(size_t nmemb, size_t size) { class PhyPlanTest : public Test { protected: + void pushAgg(int32_t aggOp) { + unique_ptr agg((SQueryPlanNode*)myCalloc(1, sizeof(SQueryPlanNode))); + agg->info.type = aggOp; + agg->pExpr = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + unique_ptr expr((SExprInfo*)myCalloc(1, sizeof(SExprInfo))); + expr->base.resSchema.type = TSDB_DATA_TYPE_INT; + expr->base.resSchema.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; + expr->pExpr = (tExprNode*)myCalloc(1, sizeof(tExprNode)); + expr->pExpr->nodeType = TEXPR_FUNCTION_NODE; + strcpy(expr->pExpr->_function.functionName, "Count"); + SExprInfo* item = expr.release(); + taosArrayPush(agg->pExpr, &item); + pushNode(agg.release()); + } + void pushScan(const string& db, const string& table, int32_t scanOp) { shared_ptr meta = mockCatalogService->getTableMeta(db, table); EXPECT_TRUE(meta); @@ -95,10 +110,11 @@ protected: private: void pushNode(SQueryPlanNode* node) { if (logicPlan_) { - // todo - } else { - logicPlan_.reset(node); + node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + SQueryPlanNode* child = logicPlan_.release(); + taosArrayPush(node->pChildren, &child); } + logicPlan_.reset(node); } void copySchemaMeta(STableMeta** dst, const STableMeta* src) { @@ -174,6 +190,16 @@ TEST_F(PhyPlanTest, superTableScanTest) { // todo check } +// select count(*) from table +TEST_F(PhyPlanTest, simpleAggTest) { + pushScan("test", "t1", QNODE_TABLESCAN); + pushAgg(QNODE_AGGREGATE); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = result(); + // todo check +} + // insert into t values(...) TEST_F(PhyPlanTest, insertTest) { ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS); From a00a8dd90d1cb19453b4b0ee7b06deed682d8407 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 13 Jan 2022 20:59:41 +0800 Subject: [PATCH 44/63] add libuv test --- source/libs/transport/CMakeLists.txt | 7 ++ source/libs/transport/inc/transportInt.h | 53 ++++++++++- source/libs/transport/src/rpcMain.c | 88 +++++++++++++------ source/libs/transport/test/CMakeLists.txt | 21 +++++ source/libs/transport/test/transportTests.cc | 35 ++++++++ source/libs/transport/test/transportTests.cpp | 0 6 files changed, 174 insertions(+), 30 deletions(-) create mode 100644 source/libs/transport/test/CMakeLists.txt create mode 100644 source/libs/transport/test/transportTests.cc delete mode 100644 source/libs/transport/test/transportTests.cpp diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index c4eeef5df2..61d781210c 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -27,4 +27,11 @@ if (${BUILD_WITH_UV}) add_definitions(-DUSE_UV) endif(${BUILD_WITH_UV}) +if (${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) + + + + diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 9809f7ee1a..067b371b84 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -22,9 +22,58 @@ extern "C" { #ifdef USE_UV -#else +#include +typedef void *queue[2]; + +/* Private macros. */ +#define QUEUE_NEXT(q) (*(queue **)&((*(q))[0])) +#define QUEUE_PREV(q) (*(queue **)&((*(q))[1])) + +#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q))) +#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) + +/* Initialize an empty queue. */ +#define QUEUE_INIT(q) \ + { \ + QUEUE_NEXT(q) = (q); \ + QUEUE_PREV(q) = (q); \ + } + +/* Return true if the queue has no element. */ +#define QUEUE_IS_EMPTY(q) ((const queue *)(q) == (const queue *)QUEUE_NEXT(q)) + +/* Insert an element at the back of a queue. */ +#define QUEUE_PUSH(q, e) \ + { \ + QUEUE_NEXT(e) = (q); \ + QUEUE_PREV(e) = QUEUE_PREV(q); \ + QUEUE_PREV_NEXT(e) = (e); \ + QUEUE_PREV(q) = (e); \ + } + +/* Remove the given element from the queue. Any element can be removed at any * + * time. */ +#define QUEUE_REMOVE(e) \ + { \ + QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \ + QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \ + } + +/* Return the element at the front of the queue. */ +#define QUEUE_HEAD(q) (QUEUE_NEXT(q)) + +/* Return the element at the back of the queue. */ +#define QUEUE_TAIL(q) (QUEUE_PREV(q)) + +/* Iterate over the element of a queue. * Mutating the queue while iterating + * results in undefined behavior. */ +#define QUEUE_FOREACH(q, e) for ((q) = QUEUE_NEXT(e); (q) != (e); (q) = QUEUE_NEXT(q)) + +/* Return the structure holding the given element. */ +#define QUEUE_DATA(e, type, field) ((type *)((void *)((char *)(e)-offsetof(type, field)))) + +#endif // USE_LIBUV -#endif #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 542bde37b9..818d129032 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -28,6 +28,7 @@ #include "tmd5.h" #include "tmempool.h" #include "tmsg.h" +#include "transportInt.h" #include "tref.h" #include "trpc.h" #include "ttimer.h" @@ -68,11 +69,13 @@ typedef struct { #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) typedef struct SThreadObj { - pthread_t thread; - uv_pipe_t* pipe; - uv_loop_t* loop; - uv_async_t* workerAsync; // - int fd; + pthread_t thread; + uv_pipe_t* pipe; + uv_loop_t* loop; + uv_async_t* workerAsync; // + int fd; + queue conn; + pthread_mutex_t connMtx; } SThreadObj; typedef struct SServerObj { @@ -88,10 +91,12 @@ typedef struct SServerObj { } SServerObj; typedef struct SConnCtx { - uv_tcp_t* pClient; + uv_tcp_t* pTcp; uv_timer_t* pTimer; uv_async_t* pWorkerAsync; + queue queue; int ref; + int persist; // persist connection or not } SConnCtx; static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -110,6 +115,9 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, int32_t rpcInit() { return -1; } void rpcCleanup() { return; }; +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + // opte +} void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SServerObj* srv = calloc(1, sizeof(SServerObj)); srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); @@ -122,30 +130,32 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_loop_init(srv->loop); for (int i = 0; i < srv->numOfThread; i++) { - srv->pThreadObj[i] = (SThreadObj*)calloc(1, sizeof(SThreadObj)); - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); + int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { return NULL; } + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - srv->pThreadObj[i]->fd = fds[0]; - srv->pThreadObj[i]->pipe = &(srv->pipe[i][1]); // init read - int err = pthread_create(&(srv->pThreadObj[i]->thread), NULL, workerThread, (void*)(srv->pThreadObj[i])); + thrd->fd = fds[0]; + thrd->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { - tDebug("sucess to create worker thread %d", i); + tDebug("sucess to create worker-thread %d", i); // printf("thread %d create\n", i); } else { // clear all resource later - tError("failed to create worker thread %d", i); + tError("failed to create worker-thread %d", i); } + srv->pThreadObj[i] = thrd; } int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); if (err == 0) { - tDebug("success to create accept thread"); + tDebug("success to create accept-thread"); } else { // clear all resource later } @@ -158,7 +168,7 @@ void* rpcOpen(const SRpcInit* pInit) { return NULL; } if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; @@ -198,29 +208,45 @@ void onWrite(uv_write_t* req, int status) { if (status == 0) { tDebug("data already was written on stream"); } + free(req); // opt } void workerAsyncCB(uv_async_t* handle) { - // opt SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); + SConnCtx* conn = NULL; + + // opt later + pthread_mutex_lock(&pObj->connMtx); + if (!QUEUE_IS_EMPTY(&pObj->conn)) { + queue* head = QUEUE_HEAD(&pObj->conn); + conn = QUEUE_DATA(head, SConnCtx, queue); + QUEUE_REMOVE(&conn->queue); + } + pthread_mutex_unlock(&pObj->connMtx); + if (conn == NULL) { + tError("except occurred, do nothing"); + return; + } } + void onAccept(uv_stream_t* stream, int status) { if (status == -1) { return; } SServerObj* pObj = container_of(stream, SServerObj, server); - tDebug("new conntion accepted by main server, dispatch to one worker thread"); uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, cli); + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); uv_buf_t buf = uv_buf_init("a", 1); pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); } else { uv_close((uv_handle_t*)cli, NULL); @@ -250,21 +276,21 @@ void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer = malloc(sizeof(uv_timer_t)); uv_timer_init(pObj->loop, pConn->pTimer); - pConn->pClient = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); pConn->pWorkerAsync = pObj->workerAsync; // thread safty - uv_tcp_init(pObj->loop, pConn->pClient); + uv_tcp_init(pObj->loop, pConn->pTcp); - if (uv_accept(q, (uv_stream_t*)(pConn->pClient)) == 0) { + if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; - uv_fileno((const uv_handle_t*)pConn->pClient, &fd); + uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tDebug("new connection created: %d", fd); uv_timer_start(pConn->pTimer, onTimeout, 10, 0); - uv_read_start((uv_stream_t*)(pConn->pClient), allocBuffer, onRead); + uv_read_start((uv_stream_t*)(pConn->pTcp), allocBuffer, onRead); } else { uv_timer_stop(pConn->pTimer); free(pConn->pTimer); - uv_close((uv_handle_t*)pConn->pClient, NULL); - free(pConn->pClient); + uv_close((uv_handle_t*)pConn->pTcp, NULL); + free(pConn->pTcp); free(pConn); } } @@ -276,7 +302,6 @@ void* acceptThread(void* arg) { struct sockaddr_in bind_addr; - int port = 6030; uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); int err = 0; @@ -288,16 +313,22 @@ void* acceptThread(void* arg) { } void* workerThread(void* arg) { SThreadObj* pObj = (SThreadObj*)arg; - int fd = pObj->fd; + pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pObj->loop); uv_pipe_init(pObj->loop, pObj->pipe, 1); - uv_pipe_open(pObj->pipe, fd); + uv_pipe_open(pObj->pipe, pObj->fd); + + QUEUE_INIT(&pObj->conn); pObj->workerAsync = malloc(sizeof(uv_async_t)); uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); + + // pObj->workerAsync->data = (void*)pObj; + uv_read_start((uv_stream_t*)pObj->pipe, allocBuffer, onConnection); + uv_run(pObj->loop, UV_RUN_DEFAULT); } #else @@ -471,7 +502,8 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; - if (pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + if (pInit->label) tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + pRpc->connType = pInit->connType; if (pRpc->connType == TAOS_CONN_CLIENT) { pRpc->numOfThreads = pInit->numOfThreads; diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt new file mode 100644 index 0000000000..9e58bf08cd --- /dev/null +++ b/source/libs/transport/test/CMakeLists.txt @@ -0,0 +1,21 @@ +add_executable(transportTest "") +target_sources(transportTest + PRIVATE + "transportTests.cc" +) + +target_include_directories(transportTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries (transportTest + os + util + common + gtest_main + transport +) + + diff --git a/source/libs/transport/test/transportTests.cc b/source/libs/transport/test/transportTests.cc new file mode 100644 index 0000000000..468aeba8a9 --- /dev/null +++ b/source/libs/transport/test/transportTests.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#include "transportInt.h" +#include "trpc.h" + +using namespace std; + +int main() { + SRpcInit init = {.localPort = 6030, .label = "rpc", .numOfThreads = 5}; + void* p = rpcOpen(&init); + + while (1) { + std::cout << "cron task" << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(10 * 1000)); + } +} diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 From 9ea9444b36aabf5b39e0bc81c5a044bb56638853 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jan 2022 10:20:55 +0800 Subject: [PATCH 45/63] [td-11818] code refactor. --- source/client/inc/clientInt.h | 15 +- source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 6 +- source/libs/executor/inc/executorimpl.h | 44 ++--- source/libs/executor/src/executorimpl.c | 209 ++++++++++++++------- source/libs/function/src/tscalarfunction.c | 1 - 6 files changed, 175 insertions(+), 101 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 39f84ffd86..e452c6dea7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -84,7 +84,7 @@ typedef struct STscObj { uint64_t id; // ref ID returned by taosAddRef void *pTransporter; pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj from this tscObj + int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo *pAppInfo; } STscObj; @@ -109,12 +109,13 @@ typedef struct SShowReqInfo { } SShowReqInfo; typedef struct SRequestSendRecvBody { - tsem_t rspSem; // not used now - void* fp; - SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. - struct SSchJob *pQueryJob; // query job, created according to sql query DAG. - SDataBuf requestMsg; - SReqResultInfo resInfo; + tsem_t rspSem; // not used now + void* fp; + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + SDataBuf requestMsg; + struct SSchJob *pQueryJob; // query job, created according to sql query DAG. + struct SQueryDag *pDag; // the query dag, generated according to the sql statement. + SReqResultInfo resInfo; } SRequestSendRecvBody; #define ERROR_MSG_BUF_DEFAULT_SIZE 512 diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 0e3afb60c0..f89442ad97 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -199,6 +199,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest->pInfo); doFreeReqResultInfo(&pRequest->body.resInfo); + qDestroyQueryDag(pRequest->body.pDag); deregisterRequest(pRequest); tfree(pRequest); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 16961799b9..5ecdcb0dbf 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -331,7 +331,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQuery = NULL; - SQueryDag *pDag = NULL; terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -340,14 +339,13 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQuery)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return); + CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return); pRequest->code = terrno; } _return: qDestroyQuery(pQuery); - qDestroyQueryDag(pDag); if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9ccc7992f3..42c0f2a6e4 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -542,34 +542,34 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperator(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); +SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); +SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows, void* merger); -SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); -SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, +SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); +SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); +SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); +SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); //SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); //SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b781615bae..031900d2bd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -294,7 +294,6 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); res->info.numOfCols = numOfOutput; - res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData idata = {{0}}; @@ -1815,7 +1814,7 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return TSDB_CODE_SUCCESS; } -static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1919,6 +1918,104 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } +static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + if (pFuncCtx == NULL) { + return NULL; + } + + *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + if (*rowCellInfoOffset == 0) { + tfree(pFuncCtx); + return NULL; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + SSqlExpr *pSqlExpr = &pExpr[i].base; + SQLFunctionCtx* pCtx = &pFuncCtx[i]; +#if 0 + SColIndex *pIndex = &pSqlExpr->colInfo; + + if (TSDB_COL_REQ_NULL(pIndex->flag)) { + pCtx->requireNull = true; + pIndex->flag &= ~(TSDB_COL_NULL); + } else { + pCtx->requireNull = false; + } +#endif +// pCtx->inputBytes = pSqlExpr->colBytes; +// pCtx->inputType = pSqlExpr->colType; + + pCtx->ptsOutputBuf = NULL; + + pCtx->resDataInfo.bytes = pSqlExpr->resSchema.bytes; + pCtx->resDataInfo.type = pSqlExpr->resSchema.type; + +// pCtx->order = pQueryAttr->order.order; +// pCtx->functionId = pSqlExpr->functionId; +// pCtx->stableQuery = pQueryAttr->stableQuery; + pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; + + pCtx->numOfParams = pSqlExpr->numOfParams; + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + int16_t type = pSqlExpr->param[j].nType; + int16_t bytes = pSqlExpr->param[j].nLen; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + taosVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type); + } else { + taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i, bytes, type); + } + } + + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + int32_t f = getExprFunctionId(&pExpr[0]); + assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); + +// pCtx->param[2].i = pQueryAttr->order.order; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[3].i = functionId; + pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + +// pCtx->param[1].i = pQueryAttr->order.col.info.colId; + } else if (functionId == FUNCTION_INTERP) { +// pCtx->param[2].i = (int8_t)pQueryAttr->fillType; +// if (pQueryAttr->fillVal != NULL) { +// if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { +// pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; +// } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value +// if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { +// taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); +// } +// } +// } + } else if (functionId == FUNCTION_TS_COMP) { +// pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == FUNCTION_TWA) { +// pCtx->param[1].i = pQueryAttr->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; +// pCtx->param[2].i = pQueryAttr->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == FUNCTION_ARITHM) { +// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); + } + } + +// for(int32_t i = 1; i < numOfOutput; ++i) { +// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes); +// } + + setCtxTagColumnInfo(pFuncCtx, numOfOutput); + + return pFuncCtx; +} + static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; @@ -4452,7 +4549,7 @@ void queryCostStatis(SQInfo *pQInfo) { // return true; //} -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { +void appendDownstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { if (p->pDownstream == NULL) { assert(p->numOfDownstream == 0); } @@ -4545,28 +4642,6 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr pRuntimeEnv->cur.vgroupIndex = -1; setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); - switch(tbScanner) { -// case OP_TableBlockInfoScan: { -// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); -// break; -// } -// case OP_TableSeqScan: { -// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); -// break; -// } -// case OP_DataBlocksOptScan: { -// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); -// break; -// } -// case OP_TableScan: { -// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); -// break; -// } - default: { // do nothing - break; - } - } - if (sourceOptr != NULL) { assert(pRuntimeEnv->proot == NULL); pRuntimeEnv->proot = sourceOptr; @@ -4841,7 +4916,7 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperator(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -4854,7 +4929,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in return NULL; } - pInfo->pTsdbReadHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = order; @@ -4874,7 +4949,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in return pOperator; } -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -4887,7 +4962,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, int32_t order return NULL; } - pInfo->pTsdbReadHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->order = order; @@ -4907,10 +4982,10 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, int32_t order return pOperator; } -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -4931,10 +5006,10 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn return pOperator; } -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pTsdbReadHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -5118,7 +5193,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pInfo->bufCapacity = 4096; pInfo->udfInfo = pUdfInfo; pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); @@ -5167,7 +5242,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S // pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -5316,7 +5391,7 @@ SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->cleanup = destroyOrderOperatorInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6248,33 +6323,33 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); +// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t numOfRows = 1;//(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN); +// setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; -// pOperator->operatorType = OP_Aggregate; + pOperator->operatorType = OP_Aggregate; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->pRuntimeEnv = NULL; pOperator->exec = doAggregate; pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6354,7 +6429,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -6369,7 +6444,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp pOperator->exec = doSTableAggregate; pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6382,7 +6457,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator SOptrBasicInfo* pBInfo = &pInfo->binfo; pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); - pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); + pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); @@ -6399,7 +6474,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = doProjectOperation; pOperator->cleanup = destroyProjectOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6457,7 +6532,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroyConditionOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6475,7 +6550,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->exec = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6483,7 +6558,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -6500,7 +6575,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe pOperator->exec = doIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6508,7 +6583,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -6525,7 +6600,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->exec = doAllIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6533,7 +6608,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); pInfo->colIndex = -1; pInfo->reptScan = false; - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -6549,13 +6624,13 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->exec = doStateWindowAgg; pOperator->cleanup = destroyStateWindowOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -6574,14 +6649,14 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = doSessionWindowAgg; pOperator->cleanup = destroySWindowOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -6598,14 +6673,14 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->exec = doSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); @@ -6622,7 +6697,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->exec = doAllSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6633,7 +6708,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pInfo->colIndex = -1; // group by column index - pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6655,7 +6730,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = hashGroupbyAggregate; pOperator->cleanup = destroyGroupbyOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6694,7 +6769,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->exec = doFill; pOperator->cleanup = destroySFillOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -6742,7 +6817,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } @@ -7040,7 +7115,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->pExpr = pExpr; pOperator->cleanup = destroyDistinctOperatorInfo; - appendUpstream(pOperator, downstream); + appendDownstream(pOperator, downstream); return pOperator; } diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 1e7e3ef155..90d25b3e4a 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -2,7 +2,6 @@ #include "tbinoperator.h" #include "tunaryoperator.h" - static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { dst->type = src->type; dst->bytes = src->bytes; From de6fa2081099d064eba589fa855f9e950887d615 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jan 2022 10:36:26 +0800 Subject: [PATCH 46/63] [td-11818] refactor. --- source/client/inc/clientInt.h | 1 - source/client/src/clientEnv.c | 11 +++-------- source/client/src/clientImpl.c | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index e452c6dea7..8b530ab6cf 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -82,7 +82,6 @@ typedef struct STscObj { int32_t acctId; uint32_t connId; uint64_t id; // ref ID returned by taosAddRef - void *pTransporter; pthread_mutex_t mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo *pAppInfo; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f89442ad97..f747ccf3b6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -89,13 +89,12 @@ static void tscInitLogFile() { // todo close the transporter properly void closeTransporter(STscObj* pTscObj) { - if (pTscObj == NULL || pTscObj->pTransporter == NULL) { + if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); - rpcClose(pTscObj->pTransporter); - pTscObj->pTransporter = NULL; + tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); + rpcClose(pTscObj->pAppInfo->pTransporter); } // TODO refactor @@ -140,10 +139,6 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI } pObj->pAppInfo = pAppInfo; - if (pAppInfo != NULL) { - pObj->pTransporter = pAppInfo->pTransporter; - } - tstrncpy(pObj->user, user, sizeof(pObj->user)); memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5ecdcb0dbf..3090955f69 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -158,7 +158,7 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, - .pTransporter = pTscObj->pTransporter, + .pTransporter = pTscObj->pAppInfo->pTransporter, }; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); @@ -191,10 +191,10 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { pShowReqInfo->pArray = pDcl->pExtension; } } - asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } else { SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; - asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, pSendMsg); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, pSendMsg); } tsem_wait(&pRequest->body.rspSem); @@ -241,7 +241,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); + int32_t code = scheduleExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { @@ -255,7 +255,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return pRequest->code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); + return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); } TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { @@ -305,7 +305,7 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); @@ -406,7 +406,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con SMsgSendInfo* body = buildConnectMsg(pRequest); int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); if (pRequest->code != TSDB_CODE_SUCCESS) { @@ -417,7 +417,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con taos_close(pTscObj); pTscObj = NULL; } else { - tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p, reqId:0x%"PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pTransporter, pRequest->requestId); + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p, reqId:0x%"PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId); destroyRequest(pRequest); } @@ -585,7 +585,7 @@ void* doFetchRow(SRequestObj* pRequest) { int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; - asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; @@ -595,7 +595,7 @@ void* doFetchRow(SRequestObj* pRequest) { int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; - asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); From 3b69bde0cec2dcddd9a4e216007a5abc66934c1c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 14 Jan 2022 10:48:05 +0800 Subject: [PATCH 47/63] add mq defination --- include/client/taos.h | 3 +- include/common/tmsg.h | 34 +++++++ include/common/tmsgdef.h | 2 +- source/client/inc/clientHb.h | 78 --------------- source/client/inc/clientInt.h | 74 +++++++++++++-- source/client/src/clientHb.c | 43 ++++++--- source/client/src/clientImpl.c | 115 +++++++++++++++++++++++ source/client/src/clientMsgHandler.c | 5 +- source/dnode/mnode/impl/src/mndProfile.c | 18 ++++ source/libs/wal/src/walMeta.c | 6 +- 10 files changed, 276 insertions(+), 102 deletions(-) delete mode 100644 source/client/inc/clientHb.h diff --git a/include/client/taos.h b/include/client/taos.h index 4669ca51f7..db33dbf8a2 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -193,8 +193,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); - -DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); +DLL_EXPORT TAOS_RES* tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fdf64b7af2..9f6d268ab2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -68,6 +68,14 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 +typedef enum { + HEARTBEAT_TYPE_MQ = 0, + HEARTBEAT_TYPE_QUERY = 1, + // types can be added here + // + HEARTBEAT_TYPE_MAX +} EHbType; + typedef enum _mgmt_table { TSDB_MGMT_TABLE_START, TSDB_MGMT_TABLE_ACCT, @@ -220,6 +228,7 @@ static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) return buf; } + typedef struct { int32_t vgId; char* dbName; @@ -359,6 +368,31 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { return buf; } +typedef struct SMqHbRsp { + int8_t status; //idle or not + int8_t vnodeChanged; + int8_t epChanged; // should use new epset + int8_t reserved; + SEpSet epSet; +} SMqHbRsp; + +static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { + int tlen = 0; + tlen += taosEncodeFixedI8(buf, pRsp->status); + tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); + tlen += taosEncodeFixedI8(buf, pRsp->epChanged); + tlen += taosEncodeSEpSet(buf, &pRsp->epSet); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { + buf = taosDecodeFixedI8(buf, &pRsp->status); + buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged); + buf = taosDecodeFixedI8(buf, &pRsp->epChanged); + buf = taosDecodeSEpSet(buf, &pRsp->epSet); + return buf; +} + typedef struct { int32_t acctId; int64_t clusterId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 592672b32b..654174966d 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -129,7 +129,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp) TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h deleted file mode 100644 index 7bc4311b29..0000000000 --- a/source/client/inc/clientHb.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tarray.h" -#include "thash.h" -#include "tmsg.h" - -#define HEARTBEAT_INTERVAL 1500 // ms - -typedef enum { - HEARTBEAT_TYPE_MQ = 0, - // types can be added here - // - HEARTBEAT_TYPE_MAX -} EHbType; - -typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); - -typedef struct SAppHbMgr { - // statistics - int32_t reportCnt; - int32_t connKeyCnt; - int64_t reportBytes; // not implemented - int64_t startTime; - // ctl - SRWLatch lock; // lock is used in serialization - // connection - void* transporter; - SEpSet epSet; - // info - SHashObj* activeInfo; // hash - SHashObj* getInfoFuncs; // hash -} SAppHbMgr; - -typedef struct SClientHbMgr { - int8_t inited; - // ctl - int8_t threadStop; - pthread_t thread; - pthread_mutex_t lock; // used when app init and cleanup - SArray* appHbMgrs; // SArray one for each cluster - FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; -} SClientHbMgr; - -// TODO: embed param into function -// return type: SArray -typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param); - -// global, called by mgmt -int hbMgrInit(); -void hbMgrCleanUp(); -int hbHandleRsp(SClientHbBatchRsp* hbRsp); - -// cluster level -SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet); -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); - -// conn level -int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); - -int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); - -// mq -void hbMgrInitMqHbRspHandle(); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 26afe237c9..71c441419d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -31,6 +31,41 @@ extern "C" { #include "trpc.h" #include "query.h" +#define HEARTBEAT_INTERVAL 1500 // ms + +typedef struct SAppInstInfo SAppInstInfo; + +typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); + +typedef struct SAppHbMgr { + // statistics + int32_t reportCnt; + int32_t connKeyCnt; + int64_t reportBytes; // not implemented + int64_t startTime; + // ctl + SRWLatch lock; // lock is used in serialization + // connection + SAppInstInfo* pAppInstInfo; + // info + SHashObj* activeInfo; // hash + SHashObj* getInfoFuncs; // hash +} SAppHbMgr; + +typedef struct SClientHbMgr { + int8_t inited; + // ctl + int8_t threadStop; + pthread_t thread; + pthread_mutex_t lock; // used when app init and cleanup + SArray* appHbMgrs; // SArray one for each cluster + FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; +} SClientHbMgr; + +// TODO: embed param into function +// return type: SArray +typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param); + typedef struct SQueryExecMetric { int64_t start; // start timestamp int64_t parsed; // start to parse @@ -55,15 +90,15 @@ typedef struct SHeartBeatInfo { void *pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; -typedef struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; +struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; SList *pConnList; // STscObj linked list - int64_t clusterId; + int64_t clusterId; void *pTransporter; - SHeartBeatInfo hb; -} SAppInstInfo; + struct SAppHbMgr *pAppHbMgr; +}; typedef struct SAppInfo { int64_t startTime; @@ -81,6 +116,7 @@ typedef struct STscObj { char db[TSDB_DB_FNAME_LEN]; int32_t acctId; uint32_t connId; + int32_t connType; uint64_t id; // ref ID returned by taosAddRef void *pTransporter; pthread_mutex_t mutex; // used to protect the operation on db @@ -88,6 +124,10 @@ typedef struct STscObj { SAppInstInfo *pAppInfo; } STscObj; +typedef struct SMqConsumer { + STscObj* pTscObj; +} SMqConsumer; + typedef struct SReqResultInfo { const char *pRspMsg; const char *pData; @@ -169,6 +209,26 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void *doFetchRow(SRequestObj* pRequest); void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +// --- heartbeat +// global, called by mgmt +int hbMgrInit(); +void hbMgrCleanUp(); +int hbHandleRsp(SClientHbBatchRsp* hbRsp); + +// cluster level +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo); +void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); + +// conn level +int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); + +int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); + +// --- mq +void hbMgrInitMqHbRspHandle(); + + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9bbd62c1d9..0d343b1b77 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "clientInt.h" #include "clientHb.h" #include "trpc.h" @@ -21,10 +22,18 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbRspHandle(SClientHbRsp* pReq) { +static int32_t hbMqHbRspHandle(SClientHbRsp* pRsp) { return 0; } +static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) { + if (code != 0) { + return -1; + } + SClientHbRsp* pRsp = (SClientHbRsp*) pMsg->pData; + return hbMqHbRspHandle(pRsp); +} + void hbMgrInitMqHbRspHandle() { clientHbMgr.handle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle; } @@ -77,18 +86,31 @@ static void* hbThreadFunc(void* param) { for(int i = 0; i < sz; i++) { SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i); SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr); - void* reqStr = NULL; - int tlen = tSerializeSClientHbBatchReq(&reqStr, pReq); + int tlen = tSerializeSClientHbBatchReq(NULL, pReq); + void *buf = malloc(tlen); + if (buf == NULL) { + //TODO: error handling + break; + } + tSerializeSClientHbBatchReq(buf, pReq); SMsgSendInfo info; - /*info.fp = hbHandleRsp;*/ + info.fp = hbMqAsyncCallBack; + info.msgInfo.pData = buf; + info.msgInfo.len = tlen; + info.msgType = TDMT_MND_HEARTBEAT; + info.param = NULL; + info.requestId = generateRequestId(); + info.requestObjRefId = -1; + SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo; int64_t transporterId = 0; - asyncSendMsgToServer(pAppHbMgr->transporter, &pAppHbMgr->epSet, &transporterId, &info); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, &info); tFreeClientHbBatchReq(pReq); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); - taosMsleep(HEARTBEAT_INTERVAL); } + taosMsleep(HEARTBEAT_INTERVAL); } return NULL; } @@ -110,7 +132,8 @@ static void hbStopThread() { atomic_store_8(&clientHbMgr.threadStop, 1); } -SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) { +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { + hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -119,9 +142,8 @@ SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) { // init stat pAppHbMgr->startTime = taosGetTimestampMs(); - // init connection info - pAppHbMgr->transporter = transporter; - pAppHbMgr->epSet = epSet; + // init app info + pAppHbMgr->pAppInstInfo = pAppInstInfo; // init hash info pAppHbMgr->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); @@ -171,7 +193,6 @@ void hbMgrCleanUp() { if (old == 0) return; taosArrayDestroy(clientHbMgr.appHbMgrs); - } int hbHandleRsp(SClientHbBatchRsp* hbRsp) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d18142cebf..5ee71fc95a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -113,6 +113,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); + p->pAppHbMgr = appHbMgrInit(p); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -220,6 +221,101 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL /*todo appInfo.xxx*/, pDag, pJob); } +typedef struct tmq_t tmq_t; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + // statistics + int64_t consumeCnt; + // offset + int64_t committedOffset; + int64_t currentOffset; + //connection info + int32_t vgId; + SEpSet epSet; +} SMqClientTopic; + +typedef struct tmq_resp_err_t { + int32_t code; +} tmq_resp_err_t; + +typedef struct tmq_topic_vgroup_list_t { + char* topicName; + int32_t vgId; + int64_t committedOffset; +} tmq_topic_vgroup_list_t; + +typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); + +typedef struct tmq_conf_t{ + char* clientId; + char* groupId; + char* ip; + uint16_t port; + tmq_commit_cb* commit_cb; +} tmq_conf_t; + +struct tmq_t { + char groupId[256]; + char clientId[256]; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + SArray* clientTopics; // SArray +}; + +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { + conf->commit_cb = cb; +} + +SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { + tmq_t* pTmq = (void*)param; + SArray* pArray = taosArrayInit(0, sizeof(SKv)); + if (pArray == NULL) { + return NULL; + } + SKv kv = {0}; + kv.key = malloc(256); + if (kv.key == NULL) { + taosArrayDestroy(pArray); + return NULL; + } + strcpy(kv.key, "groupId"); + kv.keyLen = strlen("groupId") + 1; + kv.value = malloc(256); + if (kv.value == NULL) { + free(kv.key); + taosArrayDestroy(pArray); + return NULL; + } + strcpy(kv.value, pTmq->groupId); + kv.valueLen = strlen(pTmq->groupId) + 1; + + taosArrayPush(pArray, &kv); + strcpy(kv.key, "clientUid"); + kv.keyLen = strlen("clientUid") + 1; + *(uint32_t*)kv.value = pTmq->pTscObj->connId; + kv.valueLen = sizeof(uint32_t); + + return NULL; +} + +tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { + tmq_t* pTmq = malloc(sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + strcpy(pTmq->groupId, conf->groupId); + strcpy(pTmq->clientId, conf->clientId); + pTmq->pTscObj = (STscObj*)conn; + pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; + + return pTmq; +} + TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { STscObj* pTscObj = (STscObj*)taos; SRequestObj* pRequest = NULL; @@ -281,6 +377,25 @@ _return: return pRequest; } +typedef struct tmq_message_t { + int32_t numOfRows; + char* topicName; + TAOS_ROW row[]; +} tmq_message_t; + +tmq_message_t* tmq_consume_poll(tmq_t* mq, int64_t blocking_time) { + return NULL; +} + +tmq_resp_err_t* tmq_commit(tmq_t* mq, void* callback, int32_t async) { + return NULL; +} + +void tmq_message_destroy(tmq_message_t* mq_message) { + +} + + TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; if (sqlLen > (size_t) tsMaxSQLStringLen) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 85f3fb06a7..94c5c230f7 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -71,6 +71,9 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); + // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); @@ -382,4 +385,4 @@ void initMsgHandleFp() { handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = processShowRsp; handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = processRetrieveVndRsp; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fced3facbe..d856d3a473 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -258,6 +258,23 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { } static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + char *batchReqStr = pReq->rpcMsg.pCont; + SClientHbBatchReq batchReq = {0}; + tDeserializeClientHbBatchReq(batchReqStr, &batchReq); + SArray *pArray = batchReq.reqs; + int sz = taosArrayGetSize(pArray); + for (int i = 0; i < sz; i++) { + SClientHbReq* pReq = taosArrayGet(pArray, i); + if (pReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { + + } else if (pReq->connKey.hbType == HEARTBEAT_TYPE_MQ) { + + } + } + return 0; + +#if 0 SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -327,6 +344,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { pReq->contLen = sizeof(SConnectRsp); pReq->pCont = pRsp; return 0; +#endif } static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index dea178eb54..d630080086 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -199,8 +199,10 @@ int walCheckAndRepairMeta(SWal* pWal) { } int walCheckAndRepairIdx(SWal* pWal) { - // iterate all idx files - // check first and last entry of each idx file valid + // TODO: iterate all log files + // if idx not found, scan log and write idx + // if found, check complete by first and last entry of each idx file + // if idx incomplete, binary search last valid entry, and then build other part return 0; } From ae3c70a6ec1c2ab750e302576c4986d66f9be060 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 14 Jan 2022 10:53:16 +0800 Subject: [PATCH 48/63] fix compile error --- source/client/src/clientHb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0d343b1b77..9ed688f101 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -14,7 +14,6 @@ */ #include "clientInt.h" -#include "clientHb.h" #include "trpc.h" static SClientHbMgr clientHbMgr = {0}; From 8cd25b8e48e1a782cc2dbca2ea9e6590acb9ca9f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 14 Jan 2022 13:22:46 +0800 Subject: [PATCH 49/63] fix --- source/client/src/clientImpl.c | 4 ++-- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 84bb404764..b53c455cbd 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -116,7 +116,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - p->pAppHbMgr = appHbMgrInit(p); + /*p->pAppHbMgr = appHbMgrInit(p);*/ taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -788,4 +788,4 @@ void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* p pResultInfo->completed = (pRsp->completed == 1); setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); -} \ No newline at end of file +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 497ef1ac95..ba29b81132 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); + /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/ // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d856d3a473..71b61730c3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -258,6 +258,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { } static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { +#if 0 SMnode *pMnode = pReq->pMnode; char *batchReqStr = pReq->rpcMsg.pCont; SClientHbBatchReq batchReq = {0}; @@ -273,8 +274,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } } return 0; +#else -#if 0 SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; From 790054bce97909d8a7a646bd98418cfaa2f758c5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 14 Jan 2022 05:42:37 +0000 Subject: [PATCH 50/63] fix create table stuck --- source/dnode/vnode/src/inc/vnodeDef.h | 12 ++++++------ source/dnode/vnode/src/vnd/vnodeWrite.c | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 85563890fa..1333c9dce7 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -82,12 +82,12 @@ int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); // For Log extern int32_t vDebugFlag; -#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) -#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) -#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) -#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) -#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }} while(0) +#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }} while(0) +#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }} while(0) +#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("VND ", 255, __VA_ARGS__); }} while(0) +#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 01619b5c77..472f8e0fe0 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -116,7 +116,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // Check if it needs to commit if (vnodeShouldCommit(pVnode)) { - tsem_wait(&(pVnode->canCommit)); + // tsem_wait(&(pVnode->canCommit)); if (vnodeAsyncCommit(pVnode) < 0) { // TODO: handle error } From aa9a5b6f67168269da569d3d7f8c10e3eed85a16 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 14 Jan 2022 15:54:07 +0800 Subject: [PATCH 51/63] add test for new heartbeat --- include/common/tmsg.h | 12 ++- source/client/src/clientHb.c | 10 +-- source/client/test/clientTests.cpp | 76 +++++++++---------- source/common/src/tmsg.c | 74 ++++++++++++++++-- source/dnode/mnode/impl/src/mndProfile.c | 29 +++++-- .../dnode/mnode/impl/test/profile/profile.cpp | 39 ++++++++++ 6 files changed, 179 insertions(+), 61 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7bce917701..e98446fbdf 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -155,7 +155,7 @@ typedef struct { typedef struct { SClientHbKey connKey; - SHashObj* info; // hash + SHashObj* info; // hash } SClientHbReq; typedef struct { @@ -181,7 +181,10 @@ static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { } int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq); +void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); + +int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); +void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); static FORCE_INLINE void tFreeClientHbReq(void *pReq) { SClientHbReq* req = (SClientHbReq*)pReq; @@ -190,7 +193,7 @@ static FORCE_INLINE void tFreeClientHbReq(void *pReq) { } int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); -void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); +void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; @@ -198,6 +201,9 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { free(pReq); } +int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); +void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); + static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { int tlen = 0; tlen += taosEncodeFixedI32(buf, pKv->keyLen); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9ed688f101..bd94bc05ec 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -43,17 +43,17 @@ static FORCE_INLINE void hbMgrInitHandle() { } SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq* pReq = malloc(sizeof(SClientHbBatchReq)); - if (pReq == NULL) { + SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq)); + if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); - pReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { - taosArrayPush(pReq->reqs, pIter); + taosArrayPush(pBatchReq->reqs, pIter); SClientHbReq* pOneReq = pIter; taosHashClear(pOneReq->info); @@ -70,7 +70,7 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } - return pReq; + return pBatchReq; } static void* hbThreadFunc(void* param) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d1093bb1a6..108f126e15 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -147,29 +147,29 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} // -//TEST(testCase, create_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create database abc1 vgroups 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_close(pConn); -//} +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_close(pConn); +} // //TEST(testCase, create_dnode_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -293,24 +293,24 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +//TEST(testCase, create_ctable_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } + //pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + //} - taos_free_result(pRes); - taos_close(pConn); -} + //taos_free_result(pRes); + //taos_close(pConn); +//} //TEST(testCase, show_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 53f59c7d57..4d1a07be21 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -89,7 +89,7 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { int tlen = 0; tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); - int kvNum = taosHashGetSize(pReq->info); + int32_t kvNum = taosHashGetSize(pReq->info); tlen += taosEncodeFixedI32(buf, kvNum); SKv kv; void* pIter = taosHashIterate(pReq->info, pIter); @@ -104,14 +104,15 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { return tlen; } -void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) { - ASSERT(pReq->info != NULL); +void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { buf = taosDecodeSClientHbKey(buf, &pReq->connKey); // TODO: error handling - int kvNum; - taosDecodeFixedI32(buf, &kvNum); - pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int32_t kvNum; + buf = taosDecodeFixedI32(buf, &kvNum); + if (pReq->info == NULL) { + pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } for(int i = 0; i < kvNum; i++) { SKv kv; buf = taosDecodeSKv(buf, &kv); @@ -121,12 +122,69 @@ void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) { return buf; } -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq) { +int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) { int tlen = 0; + tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); + tlen += taosEncodeFixedI32(buf, pRsp->status); + tlen += taosEncodeFixedI32(buf, pRsp->bodyLen); + tlen += taosEncodeBinary(buf, pRsp->body, pRsp->bodyLen); + return tlen; +} +void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) { + buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); + buf = taosDecodeFixedI32(buf, &pRsp->status); + buf = taosDecodeFixedI32(buf, &pRsp->bodyLen); + buf = taosDecodeBinary(buf, &pRsp->body, pRsp->bodyLen); + return buf; +} + +int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pBatchReq) { + int tlen = 0; + tlen += taosEncodeFixedI64(buf, pBatchReq->reqId); + int32_t reqNum = taosArrayGetSize(pBatchReq->reqs); + tlen += taosEncodeFixedI32(buf, reqNum); + for (int i = 0; i < reqNum; i++) { + SClientHbReq* pReq = taosArrayGet(pBatchReq->reqs, i); + tlen += tSerializeSClientHbReq(buf, pReq); + } return tlen; } -void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq) { +void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) { + buf = taosDecodeFixedI64(buf, &pBatchReq->reqId); + if (pBatchReq->reqs == NULL) { + pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); + } + int32_t reqNum; + buf = taosDecodeFixedI32(buf, &reqNum); + for (int i = 0; i < reqNum; i++) { + SClientHbReq req = {0}; + buf = tDeserializeSClientHbReq(buf, &req); + taosArrayPush(pBatchReq->reqs, &req); + } + return buf; +} + +int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp) { + int tlen = 0; + int32_t sz = taosArrayGetSize(pBatchRsp->rsps); + tlen += taosEncodeFixedI32(buf, sz); + for (int i = 0; i < sz; i++) { + SClientHbRsp* pRsp = taosArrayGet(pBatchRsp->rsps, i); + tlen += tSerializeSClientHbRsp(buf, pRsp); + } + return tlen; +} + +void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp) { + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp)); + for (int i = 0; i < sz; i++) { + SClientHbRsp rsp = {0}; + buf = tDeserializeSClientHbRsp(buf, &rsp); + taosArrayPush(pBatchRsp->rsps, &rsp); + } return buf; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 71b61730c3..79e5d9eae5 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -258,24 +258,39 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { } static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { -#if 0 SMnode *pMnode = pReq->pMnode; char *batchReqStr = pReq->rpcMsg.pCont; SClientHbBatchReq batchReq = {0}; - tDeserializeClientHbBatchReq(batchReqStr, &batchReq); + tDeserializeSClientHbBatchReq(batchReqStr, &batchReq); SArray *pArray = batchReq.reqs; int sz = taosArrayGetSize(pArray); + + SClientHbBatchRsp batchRsp = {0}; + batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); + for (int i = 0; i < sz; i++) { - SClientHbReq* pReq = taosArrayGet(pArray, i); - if (pReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { - - } else if (pReq->connKey.hbType == HEARTBEAT_TYPE_MQ) { + SClientHbReq* pHbReq = taosArrayGet(pArray, i); + if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { + } else if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_MQ) { + SClientHbRsp rsp = { + .status = 0, + .connKey = pHbReq->connKey, + .bodyLen = 0, + .body = NULL + }; + taosArrayPush(batchRsp.rsps, &rsp); } } + int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); + void* buf = rpcMallocCont(tlen); + void* bufCopy = buf; + tSerializeSClientHbBatchRsp(&bufCopy, &batchRsp); + pReq->contLen = tlen; + pReq->pCont = buf; return 0; -#else +#if 0 SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index bf047517d3..a74b1c01f5 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -96,6 +96,38 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) { } TEST_F(MndTestProfile, 04_HeartBeatMsg) { + + SClientHbBatchReq batchReq; + batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq)); + SClientHbReq req = {0}; + req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ}; + req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + SKv kv; + kv.key = (void*)"abc"; + kv.keyLen = 4; + kv.value = (void*)"bcd"; + kv.valueLen = 4; + taosHashPut(req.info, kv.key, kv.keyLen, kv.value, kv.valueLen); + taosArrayPush(batchReq.reqs, &req); + + int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq); + + void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); + void* bufCopy = buf; + tSerializeSClientHbBatchReq(&bufCopy, &batchReq); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + char* pRspChar = (char*)pMsg->pCont; + SClientHbBatchRsp rsp = {0}; + tDeserializeSClientHbBatchRsp(pRspChar, &rsp); + int sz = taosArrayGetSize(rsp.rsps); + ASSERT_EQ(sz, 1); + SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); + EXPECT_EQ(pRsp->connKey.connId, 123); + EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); + EXPECT_EQ(pRsp->status, 0); +#if 0 int32_t contLen = sizeof(SHeartBeatReq); SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen); @@ -129,9 +161,12 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) { EXPECT_EQ(pRsp->epSet.numOfEps, 1); EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +#endif } TEST_F(MndTestProfile, 05_KillConnMsg) { + // temporary remove since kill will use new heartbeat msg +#if 0 { int32_t contLen = sizeof(SKillConnReq); @@ -190,6 +225,7 @@ TEST_F(MndTestProfile, 05_KillConnMsg) { connId = pRsp->connId; } +#endif } TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) { @@ -204,6 +240,8 @@ TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) { } TEST_F(MndTestProfile, 07_KillQueryMsg) { + // temporary remove since kill will use new heartbeat msg +#if 0 { int32_t contLen = sizeof(SKillQueryReq); @@ -252,6 +290,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } +#endif } TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) { From 2e40293b4d0579e957999ca52bd68a2dc707fea3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 14 Jan 2022 16:45:15 +0800 Subject: [PATCH 52/63] add test for new heartbeat --- source/client/src/clientHb.c | 25 ++++++++++++- source/client/src/clientImpl.c | 2 +- source/client/src/clientMsgHandler.c | 2 +- source/client/test/clientTests.cpp | 36 +++++++++---------- .../dnode/mnode/impl/test/profile/profile.cpp | 2 +- 5 files changed, 45 insertions(+), 22 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index bd94bc05ec..7bdde457d5 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -51,10 +51,14 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + if (pAppHbMgr->activeInfo == NULL) { + return NULL; + } + void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { - taosArrayPush(pBatchReq->reqs, pIter); SClientHbReq* pOneReq = pIter; + taosArrayPush(pBatchReq->reqs, pOneReq); taosHashClear(pOneReq->info); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); @@ -84,7 +88,14 @@ static void* hbThreadFunc(void* param) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for(int i = 0; i < sz; i++) { SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i); + int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); + if (connCnt == 0) { + continue; + } SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr); + if (pReq == NULL) { + continue; + } int tlen = tSerializeSClientHbBatchReq(NULL, pReq); void *buf = malloc(tlen); if (buf == NULL) { @@ -146,10 +157,22 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { // init hash info pAppHbMgr->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + + if (pAppHbMgr->activeInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(pAppHbMgr); + return NULL; + } pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq; // init getInfoFunc pAppHbMgr->getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + if (pAppHbMgr->getInfoFuncs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(pAppHbMgr); + return NULL; + } + taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); return pAppHbMgr; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b53c455cbd..1a25cdc20c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -116,7 +116,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - /*p->pAppHbMgr = appHbMgrInit(p);*/ + p->pAppHbMgr = appHbMgrInit(p); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index ba29b81132..497ef1ac95 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; - /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/ + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 108f126e15..9d4e7c8152 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -147,29 +147,29 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} // -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +//TEST(testCase, create_db_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } + //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //ASSERT_TRUE(pFields == NULL); - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); + //int32_t numOfFields = taos_num_fields(pRes); + //ASSERT_EQ(numOfFields, 0); - taos_free_result(pRes); + //taos_free_result(pRes); - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} + //pRes = taos_query(pConn, "create database abc1 vgroups 4"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_close(pConn); +//} // //TEST(testCase, create_dnode_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index a74b1c01f5..4b329886eb 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -96,7 +96,6 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) { } TEST_F(MndTestProfile, 04_HeartBeatMsg) { - SClientHbBatchReq batchReq; batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq)); SClientHbReq req = {0}; @@ -127,6 +126,7 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) { EXPECT_EQ(pRsp->connKey.connId, 123); EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); EXPECT_EQ(pRsp->status, 0); + #if 0 int32_t contLen = sizeof(SHeartBeatReq); From c5ece5715cfe9c57615e58c23d9024f41a0d3ecf Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 14 Jan 2022 17:30:57 +0800 Subject: [PATCH 53/63] set refObjId to 0 --- include/common/tmsg.h | 2 +- source/client/src/clientHb.c | 38 +++++++++++++++++------------- source/client/test/clientTests.cpp | 1 + 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e98446fbdf..9cf3068314 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -197,7 +197,7 @@ void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; - taosArrayDestroyEx(req->reqs, tFreeClientHbReq); + //taosArrayDestroyEx(req->reqs, tFreeClientHbReq); free(pReq); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7bdde457d5..6d7fc9f81a 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -51,10 +51,6 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); - if (pAppHbMgr->activeInfo == NULL) { - return NULL; - } - void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { SClientHbReq* pOneReq = pIter; @@ -71,7 +67,7 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { taosHashCopyKey(pIter, &connKey); getConnInfoFp(connKey, NULL); - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, pIter); } return pBatchReq; @@ -87,7 +83,8 @@ static void* hbThreadFunc(void* param) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for(int i = 0; i < sz; i++) { - SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i); + SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); if (connCnt == 0) { continue; @@ -102,20 +99,27 @@ static void* hbThreadFunc(void* param) { //TODO: error handling break; } - tSerializeSClientHbBatchReq(buf, pReq); - SMsgSendInfo info; - info.fp = hbMqAsyncCallBack; - info.msgInfo.pData = buf; - info.msgInfo.len = tlen; - info.msgType = TDMT_MND_HEARTBEAT; - info.param = NULL; - info.requestId = generateRequestId(); - info.requestObjRefId = -1; + void *bufCopy = buf; + tSerializeSClientHbBatchReq(&bufCopy, pReq); + SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo)); + if (pInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tFreeClientHbBatchReq(pReq); + free(buf); + break; + } + pInfo->fp = hbMqAsyncCallBack; + pInfo->msgInfo.pData = buf; + pInfo->msgInfo.len = tlen; + pInfo->msgType = TDMT_MND_HEARTBEAT; + pInfo->param = NULL; + pInfo->requestId = generateRequestId(); + pInfo->requestObjRefId = 0; SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo; int64_t transporterId = 0; SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); - asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, &info); + asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); @@ -182,7 +186,7 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { - SAppHbMgr* pTarget = taosArrayGet(clientHbMgr.appHbMgrs, i); + SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); if (pAppHbMgr == pTarget) { taosHashCleanup(pTarget->activeInfo); taosHashCleanup(pTarget->getInfoFuncs); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 9d4e7c8152..d5b149284e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,6 +53,7 @@ TEST(testCase, connect_Test) { if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } + sleep(3); taos_close(pConn); } From 552c151f494468827c03548342c46f6a934392d5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 14 Jan 2022 05:05:54 -0500 Subject: [PATCH 54/63] TD-12678 dsGetDataBlock allowed to call again after the query is completed to return statistics --- source/libs/executor/src/dataDispatcher.c | 6 ++++++ source/libs/planner/src/planner.c | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a69084f3db..a6af9ff388 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -182,6 +182,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; + if (NULL == pDispatcher->nextOutput.pData) { + assert(pDispatcher->queryEnd); + pOutput->useconds = pDispatcher->useconds; + pOutput->precision = pDispatcher->schema.precision; + return TSDB_CODE_SUCCESS; + } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 21f57d95d5..bf815b26b2 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From 2f3a58f8d6abcdbd73783372d491c8c6c1ff503a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jan 2022 18:35:57 +0800 Subject: [PATCH 55/63] [td-11818]support taos_create_topic --- include/client/taos.h | 3 +- include/common/tmsg.h | 5 +- include/common/tname.h | 2 - include/util/tcoding.h | 1 + include/util/tdef.h | 2 +- source/client/src/clientImpl.c | 97 ++++++++---- source/client/test/clientTests.cpp | 172 ++++++++++----------- source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/src/mndTopic.c | 49 +++--- source/libs/catalog/src/catalog.c | 28 ++-- source/libs/planner/src/physicalPlanJson.c | 12 +- source/libs/scheduler/src/scheduler.c | 4 +- 12 files changed, 208 insertions(+), 171 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 4669ca51f7..84f6255710 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -193,8 +193,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); - -DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); +DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2aaa2168cc..3fb4579451 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1053,6 +1053,7 @@ typedef struct { typedef struct { int8_t igExists; char* name; + char* sql; char* physicalPlan; char* logicalPlan; } SCMCreateTopicReq; @@ -1061,6 +1062,7 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT int tlen = 0; tlen += taosEncodeFixedI8(buf, pReq->igExists); tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->physicalPlan); tlen += taosEncodeString(buf, pReq->logicalPlan); return tlen; @@ -1069,6 +1071,7 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) { buf = taosDecodeFixedI8(buf, &(pReq->igExists)); buf = taosDecodeString(buf, &(pReq->name)); + buf = taosDecodeString(buf, &(pReq->sql)); buf = taosDecodeString(buf, &(pReq->physicalPlan)); buf = taosDecodeString(buf, &(pReq->logicalPlan)); return buf; @@ -1191,7 +1194,7 @@ typedef struct { } SMVSubscribeRsp; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; + char name[TSDB_TOPIC_NAME_LEN]; int8_t igExists; int32_t execLen; void* executor; diff --git a/include/common/tname.h b/include/common/tname.h index 11d97dac06..12a0d34cb4 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -25,14 +25,12 @@ #define T_NAME_ACCT 0x1u #define T_NAME_DB 0x2u #define T_NAME_TABLE 0x4u -#define T_NAME_TOPIC 0x8u typedef struct SName { uint8_t type; //db_name_t, table_name_t int32_t acctId; char dbname[TSDB_DB_NAME_LEN]; char tname[TSDB_TABLE_NAME_LEN]; - char topicName[TSDB_TOPIC_NAME_LEN]; } SName; int32_t tNameExtractFullName(const SName* name, char* dst); diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 226856901f..8198787048 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -351,6 +351,7 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { buf = taosDecodeVariantU64(buf, &size); *value = (char *)malloc((size_t)size + 1); + if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)size); diff --git a/include/util/tdef.h b/include/util/tdef.h index 9f16b58e0d..428de5d171 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -181,7 +181,7 @@ do { \ #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_SQL_SHOW_LEN 512 +#define TSDB_MAX_SQL_SHOW_LEN 1024 #define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb #define TSDB_APP_NAME_LEN TSDB_UNI_LEN diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3090955f69..df7e10af7d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -13,12 +13,12 @@ #include "tpagedfile.h" #include "tref.h" -#define CHECK_CODE_GOTO(expr, lable) \ +#define CHECK_CODE_GOTO(expr, label) \ do { \ int32_t code = expr; \ if (TSDB_CODE_SUCCESS != code) { \ terrno = code; \ - goto lable; \ + goto label; \ } \ } while (0) @@ -258,36 +258,62 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); } -TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { - STscObj* pTscObj = (STscObj*)taos; - SRequestObj* pRequest = NULL; - SQueryNode* pQuery = NULL; - SQueryDag* pDag = NULL; - char *dagStr = NULL; +TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { + STscObj *pTscObj = (STscObj*)taos; + SRequestObj *pRequest = NULL; + SQueryNode *pQueryNode = NULL; + char *pStr = NULL; terrno = TSDB_CODE_SUCCESS; + if (taos == NULL || topicName == NULL || sql == NULL) { + tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { + tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (sqlLen > tsMaxSQLStringLen) { + tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + goto _return; + } + + tscDebug("start to create topic, %s", topicName); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); -//temporary disabled until planner ready -#if 0 - CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); - //TODO: check sql valid + // todo check for invalid sql statement and return with error code - CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return); + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return); - dagStr = qDagToString(pDag); - if(dagStr == NULL) { - //TODO + pStr = qDagToString(pRequest->body.pDag); + if(pStr == NULL) { + goto _return; } -#endif + + // The topic should be related to a database that the queried table is belonged to. + SName name = {0}; + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName); + + tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; + tNameExtractFullName(&name, topicFname); SCMCreateTopicReq req = { - .name = (char*)name, - .igExists = 0, - /*.physicalPlan = dagStr,*/ - .physicalPlan = (char*)sql, - .logicalPlan = "", + .name = (char*) topicFname, + .igExists = 0, + .physicalPlan = (char*) pStr, + .sql = (char*) sql, + .logicalPlan = "no logic plan", }; int tlen = tSerializeSCMCreateTopicReq(NULL, &req); @@ -295,27 +321,32 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq if(buf == NULL) { goto _return; } + void* abuf = buf; tSerializeSCMCreateTopicReq(&abuf, &req); /*printf("formatted: %s\n", dagStr);*/ pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + pRequest->type = TDMT_MND_CREATE_TOPIC; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); _return: - qDestroyQuery(pQuery); - qDestroyQueryDag(pDag); - destroySendMsgInfo(body); + qDestroyQuery(pQueryNode); + if (body != NULL) { + destroySendMsgInfo(body); + } + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { pRequest->code = terrno; } + return pRequest; } @@ -330,22 +361,22 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { nPrintTsc("%s", sql) SRequestObj *pRequest = NULL; - SQueryNode *pQuery = NULL; + SQueryNode *pQueryNode = NULL; terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); - if (qIsDdlQuery(pQuery)) { - CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); + if (qIsDdlQuery(pQueryNode)) { + CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag), _return); + CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return); pRequest->code = terrno; } _return: - qDestroyQuery(pQuery); + qDestroyQuery(pQueryNode); if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 7086da0d55..429657b617 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -48,14 +48,14 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); } -//TEST(testCase, connect_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// if (pConn == NULL) { -// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); -// } -// taos_close(pConn); -//} -// +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); + } + taos_close(pConn); +} + //TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -514,39 +514,29 @@ TEST(testCase, driverInit_Test) { taos_init(); } // taosHashCleanup(phash); //} // -//// TEST(testCase, create_topic_Test) { -//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -//// assert(pConn != NULL); -//// -//// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); -//// if (taos_errno(pRes) != 0) { -//// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "use abc1"); -//// if (taos_errno(pRes) != 0) { -//// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); -//// if (taos_errno(pRes) != 0) { -//// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); -//// } -//// -//// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -//// ASSERT_TRUE(pFields == NULL); -//// -//// int32_t numOfFields = taos_num_fields(pRes); -//// ASSERT_EQ(numOfFields, 0); -//// -//// taos_free_result(pRes); -//// -//// char* sql = "select * from st1"; -//// tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); -//// taos_close(pConn); -////} +TEST(testCase, create_topic_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from tu"; + pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} // //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -565,7 +555,7 @@ TEST(testCase, driverInit_Test) { taos_init(); } // taos_free_result(pRes); // taos_close(pConn); //} -// + //TEST(testCase, projection_query_tables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -573,28 +563,28 @@ TEST(testCase, driverInit_Test) { taos_init(); } // TAOS_RES* pRes = taos_query(pConn, "use abc1"); // taos_free_result(pRes); // -// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// for(int32_t i = 0; i < 100; ++i) { -// char sql[512] = {0}; -// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); -// TAOS_RES* p = taos_query(pConn, sql); -// if (taos_errno(p) != 0) { -// printf("failed to insert data, reason:%s\n", taos_errstr(p)); -// } -// -// taos_free_result(p); -// } +//// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// for(int32_t i = 0; i < 100; ++i) { +//// char sql[512] = {0}; +//// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); +//// TAOS_RES* p = taos_query(pConn, sql); +//// if (taos_errno(p) != 0) { +//// printf("failed to insert data, reason:%s\n", taos_errstr(p)); +//// } +//// +//// taos_free_result(p); +//// } // // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { @@ -617,30 +607,30 @@ TEST(testCase, driverInit_Test) { taos_init(); } // taos_close(pConn); //} -TEST(testCase, projection_query_stables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "select ts,k from m1"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, projection_query_stables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select ts,k from m1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a55e0dd2b2..0c2524f48c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -350,7 +350,7 @@ typedef struct SMqTopicObj { // TODO: add cache and change name to id typedef struct SMqConsumerTopic { - char name[TSDB_TOPIC_FNAME_LEN]; + char name[TSDB_TOPIC_NAME_LEN]; SList *vgroups; // SList } SMqConsumerTopic; @@ -409,7 +409,7 @@ typedef struct SMqVGroupHbObj { #if 0 typedef struct SCGroupObj { - char name[TSDB_TOPIC_FNAME_LEN]; + char name[TSDB_TOPIC_NAME_LEN]; int64_t createTime; int64_t updateTime; uint64_t uid; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index acdc718f20..67aeae0d4c 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -118,11 +118,13 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); + + pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); - SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); - SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); +// SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); +// SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); +// SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); +// SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER) @@ -178,7 +180,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { SName name = {0}; - tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TOPIC); + tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); char db[TSDB_TABLE_FNAME_LEN] = {0}; tNameGetFullDbName(&name, db); @@ -203,20 +205,24 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq return pDrop; } -static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *pCreate) { +static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) { // deserialize and other stuff return 0; } static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { SMqTopicObj topicObj = {0}; - tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); topicObj.createTime = taosGetTimestampMs(); topicObj.updateTime = topicObj.createTime; topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); topicObj.dbUid = pDb->uid; topicObj.version = 1; + topicObj.sql = strdup(pCreate->sql); + topicObj.physicalPlan = strdup(pCreate->physicalPlan); + topicObj.logicalPlan = strdup(pCreate->logicalPlan); + topicObj.sqlLen = strlen(pCreate->sql); SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); if (pTopicRaw == NULL) return -1; @@ -228,46 +234,47 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; char *msgStr = pMsg->rpcMsg.pCont; - SCMCreateTopicReq *pCreate; - tDeserializeSCMCreateTopicReq(msgStr, pCreate); - mDebug("topic:%s, start to create", pCreate->name); + SCMCreateTopicReq createTopicReq = {0}; + tDeserializeSCMCreateTopicReq(msgStr, &createTopicReq); - if (mndCheckCreateTopicMsg(pCreate) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql); + + if (mndCheckCreateTopicMsg(&createTopicReq) != 0) { + mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); return -1; } - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, createTopicReq.name); if (pTopic != NULL) { sdbRelease(pMnode->pSdb, pTopic); - if (pCreate->igExists) { - mDebug("topic:%s, already exist, ignore exist is set", pCreate->name); + if (createTopicReq.igExists) { + mDebug("topic:%s, already exist, ignore exist is set", createTopicReq.name); return 0; } else { terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; - mError("db:%s, failed to create since %s", pCreate->name, terrstr()); + mError("db:%s, failed to create since %s", createTopicReq.name, terrstr()); return -1; } } - SDbObj *pDb = mndAcquireDbByTopic(pMnode, pCreate->name); + SDbObj *pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); return -1; } - int32_t code = mndCreateTopic(pMnode, pMsg, pCreate, pDb); + int32_t code = mndCreateTopic(pMnode, pMsg, &createTopicReq, pDb); mndReleaseDb(pMnode, pDb); if (code != 0) { terrno = code; - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); return -1; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_SUCCESS; } static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 94f34b8e17..f6b752bc6c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -285,16 +285,16 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, char dbFullName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFullName); - ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, pTableName->tname); + ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)pTableName->tname}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, pTableName->tname); + ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } @@ -313,21 +313,21 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NONE(output->metaType); - ctgDebug("tablemeta not exist in vnode, tbName:%s", pTableName->tname); + ctgDebug("tablemeta not exist in vnode, tbName:%s", tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for table meta from vnode, code:%x, tbName:%s", rpcRsp.code, pTableName->tname); + ctgError("error rsp for table meta from vnode, code:%x, tbName:%s", rpcRsp.code, tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%x, tbName:%s", code, pTableName->tname); + ctgError("Process vnode tablemeta rsp failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } - ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, pTableName->tname); + ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } @@ -776,7 +776,7 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con STableMetaOutput *output = &voutput; if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew table meta, supposed to be stable, tbName:%s", pTableName->tname); + ctgDebug("will renew table meta, supposed to be stable, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); @@ -787,13 +787,13 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con output = &moutput; } } else { - ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", pTableName->tname, isSTable); + ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", pTableName->tname, voutput.metaType); + ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); @@ -820,7 +820,7 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con } if (CTG_IS_META_NONE(output->metaType)) { - ctgError("no tablemeta got, tbNmae:%s", pTableName->tname); + ctgError("no tablemeta got, tbNmae:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -860,7 +860,7 @@ int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMg CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); if (0 == exist) { - ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", pTableName->tname); + ctgError("renew tablemeta succeed but get from cache failed, may be deleted, tbName:%s", tNameGetTableName(pTableName)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1241,7 +1241,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S } else { int32_t vgId = tbMeta->vgId; if (NULL == taosHashGetClone(dbVgroup->vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { - ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, pTableName->tname); + ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1252,7 +1252,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S } if (NULL == taosArrayPush(vgList, &vgroupInfo)) { - ctgError("taosArrayPush vgroupInfo to array failed, vgId:%d, tbName:%s", vgId, pTableName->tname); + ctgError("taosArrayPush vgroupInfo to array failed, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 2e553fe4e6..97eb63ac31 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -1057,14 +1057,18 @@ cJSON* qDagToJson(const SQueryDag* pDag) { if(pRoot == NULL) { return NULL; } - cJSON_AddNumberToObject(pRoot, "numOfSubplans", pDag->numOfSubplans); - cJSON_AddNumberToObject(pRoot, "queryId", pDag->queryId); + + cJSON_AddNumberToObject(pRoot, "Number", pDag->numOfSubplans); + cJSON_AddNumberToObject(pRoot, "QueryId", pDag->queryId); + cJSON *pLevels = cJSON_CreateArray(); if(pLevels == NULL) { cJSON_Delete(pRoot); return NULL; } - cJSON_AddItemToObject(pRoot, "pSubplans", pLevels); + + cJSON_AddItemToObject(pRoot, "Subplans", pLevels); + size_t level = taosArrayGetSize(pDag->pSubplans); for(size_t i = 0; i < level; i++) { const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i); @@ -1074,6 +1078,7 @@ cJSON* qDagToJson(const SQueryDag* pDag) { cJSON_Delete(pRoot); return NULL; } + cJSON_AddItemToArray(pLevels, plansOneLevel); for(size_t j = 0; j < num; j++) { cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j)); @@ -1081,6 +1086,7 @@ cJSON* qDagToJson(const SQueryDag* pDag) { cJSON_Delete(pRoot); return NULL; } + cJSON_AddItemToArray(plansOneLevel, pSubplan); } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 09e5291c85..9457476217 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1029,6 +1029,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = pTask->msg; break; } + case TDMT_VND_QUERY: { msgSize = sizeof(SSubQueryMsg) + pTask->msgLen; msg = calloc(1, msgSize); @@ -1047,7 +1048,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, pMsg->contentLen = htonl(pTask->msgLen); memcpy(pMsg->msg, pTask->msg, pTask->msgLen); break; - } + } + case TDMT_VND_RES_READY: { msgSize = sizeof(SResReadyReq); msg = calloc(1, msgSize); From 8ac88a625ea0885819e75dbf616748de7aefb1bc Mon Sep 17 00:00:00 2001 From: lihui Date: Fri, 14 Jan 2022 18:40:37 +0800 Subject: [PATCH 56/63] [modify bug] --- tests/test/c/create_table.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index aae4dc7074..0376ab70ad 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -187,13 +187,19 @@ void *threadFunc(void *param) { int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); pInfo->startMs = beginMs; - for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { - int64_t batch = (pInfo->tableEndIndex - t); - batch = MIN(batch, batchNum); + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex;) { + //int64_t batch = (pInfo->tableEndIndex - t); + //batch = MIN(batch, batchNum); int32_t len = sprintf(qstr, "create table"); - for (int32_t i = 0; i < batch; ++i) { - len += sprintf(qstr + len, " t%" PRId64 " using %s tags(%" PRId64 ")", t + i, stbName, t + i); + for (int32_t i = 0; i < batchNum;) { + len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); + t++; + i++; + if (t > pInfo->tableEndIndex) { + break; + } } int64_t startTs = taosGetTimestampUs(); @@ -212,11 +218,11 @@ void *threadFunc(void *param) { curMs = taosGetTimestampMs(); if (curMs - beginMs > 10000) { beginMs = curMs; + //printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); printCreateProgress(pInfo, t); } - t += (batch - 1); } - printCreateProgress(pInfo, pInfo->tableEndIndex); + printCreateProgress(pInfo, t); } if (insertData) { From 41f600698c955dedcbc9dcd26acaf7b2ca044b1d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 14 Jan 2022 22:49:05 +0800 Subject: [PATCH 57/63] add libuv --- source/libs/transport/inc/rpcHead.h | 21 ++++ source/libs/transport/src/rpcMain.c | 182 +++++++++++++++++++++++----- 2 files changed, 175 insertions(+), 28 deletions(-) diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 7317d84af1..66821db133 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -22,6 +22,27 @@ extern "C" { #endif #ifdef USE_UV +typedef struct { + char version : 4; // RPC version + char comp : 4; // compression algorithm, 0:no compression 1:lz4 + char resflag : 2; // reserved bits + char spi : 3; // security parameter index + char encrypt : 3; // encrypt algorithm, 0: no encryption + uint16_t tranId; // transcation ID + uint32_t linkUid; // for unique connection ID assigned by client + uint64_t ahandle; // ahandle assigned by client + uint32_t sourceId; // source ID, an index for connection list + uint32_t destId; // destination ID, an index for connection list + uint32_t destIp; // destination IP address, for NAT scenario + char user[TSDB_UNI_LEN]; // user ID + uint16_t port; // for UDP only, port may be changed + char empty[1]; // reserved + uint16_t msgType; // message type + int32_t msgLen; // message length including the header iteslf + uint32_t msgVer; + int32_t code; // code in response message + uint8_t content[0]; // message body starts from here +} SRpcHead; #else diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 818d129032..a1c0c05fc3 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#ifdef USE_UV #include +#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" @@ -68,6 +70,8 @@ typedef struct { #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +static const char* notify = "a"; + typedef struct SThreadObj { pthread_t thread; uv_pipe_t* pipe; @@ -90,23 +94,39 @@ typedef struct SServerObj { uint32_t port; } SServerObj; +typedef struct SContent { + char* buf; + int len; + int cap; + int toRead; +} SContent; + typedef struct SConnCtx { uv_tcp_t* pTcp; + uv_write_t* pWriter; uv_timer_t* pTimer; + uv_async_t* pWorkerAsync; queue queue; int ref; int persist; // persist connection or not + SContent pCont; + int count; } SConnCtx; -static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void onTimeout(uv_timer_t* handle); +static void allocReadBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void allocConnBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void onTimeout(uv_timer_t* handle); static void onWrite(uv_write_t* req, int status); static void onAccept(uv_stream_t* stream, int status); static void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void workerAsyncCB(uv_async_t* handle); +static SConnCtx* connCtxCreate(); +static void connCtxDestroy(SConnCtx* ctx); +static void uvConnCtxDestroy(uv_handle_t* handle); + static void* workerThread(void* arg); static void* acceptThread(void* arg); @@ -131,12 +151,11 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThread; i++) { SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); - + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { return NULL; } - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write @@ -147,7 +166,7 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, tDebug("sucess to create worker-thread %d", i); // printf("thread %d create\n", i); } else { - // clear all resource later + // TODO: clear all other resource later tError("failed to create worker-thread %d", i); } srv->pThreadObj[i] = thrd; @@ -171,7 +190,6 @@ void* rpcOpen(const SRpcInit* pInit) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; } @@ -190,26 +208,106 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - buf->base = malloc(suggested_size); - buf->len = suggested_size; +void allocReadBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + static const int CAPACITY = 1024; + tDebug("pre alloc buffer for read "); + SConnCtx* ctx = handle->data; + SContent* pCont = &ctx->pCont; + if (pCont->cap == 0) { + pCont->buf = (char*)calloc(CAPACITY, sizeof(char)); + pCont->len = 0; + pCont->cap = CAPACITY; + pCont->toRead = -1; + + buf->base = pCont->buf; + buf->len = CAPACITY; + } else { + if (pCont->len >= pCont->cap) { + if (pCont->toRead == -1) { + pCont->cap *= 2; + pCont->buf = realloc(pCont->buf, pCont->cap); + } else if (pCont->len + pCont->toRead > pCont->cap) { + pCont->cap = pCont->len + pCont->toRead; + pCont->buf = realloc(pCont->buf, pCont->len + pCont->toRead); + } + } + buf->base = pCont->buf + pCont->len; + buf->len = pCont->cap - pCont->len; + } + + // if (ctx->pCont.cap == 0) { + // ctx->pCont.buf = (char*)calloc(64, sizeof(char)); + // ctx->pCont.len = 0; + // ctx->pCont.cap = 64; + // // + // buf->base = ctx->pCont.buf; + // buf->len = sz; + //} else { + // if (ctx->pCont.len + sz > ctx->pCont.cap) { + // ctx->pCont.cap *= 2; + // ctx->pCont.buf = realloc(ctx->pCont.buf, ctx->pCont.cap); + // } + // buf->base = ctx->pCont.buf + ctx->pCont.len; + // buf->len = sz; + //} +} +// change later +static bool handleUserData(SContent* data) { + SRpcHead rpcHead; + + bool finish = false; + int32_t msgLen, leftLen, retLen; + int32_t headLen = sizeof(rpcHead); + if (data->len >= headLen) { + memcpy((char*)&rpcHead, data->buf, headLen); + msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + if (msgLen + headLen <= data->len) { + return true; + } else { + return false; + } + } else { + return false; + } +} + +void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + SConnCtx* ctx = cli->data; + SContent* pCont = &ctx->pCont; + if (nread > 0) { + pCont->len += nread; + bool finish = handleUserData(pCont); + if (finish == false) { + tDebug("continue read"); + } else { + tDebug("read completely"); + } + return; + } + + if (nread != UV_EOF) { + tDebug("Read error %s\n", uv_err_name(nread)); + } + uv_close((uv_handle_t*)cli, uvConnCtxDestroy); +} +void allocConnBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(sizeof(char)); + buf->len = 2; } void onTimeout(uv_timer_t* handle) { // opt tDebug("time out"); } -void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { - // opt - tDebug("data already was read on a stream"); -} void onWrite(uv_write_t* req, int status) { + SConnCtx* ctx = req->data; if (status == 0) { tDebug("data already was written on stream"); + } else { + connCtxDestroy(ctx); } - free(req); - // opt } @@ -243,7 +341,7 @@ void onAccept(uv_stream_t* stream, int status) { if (uv_accept(stream, (uv_stream_t*)cli) == 0) { uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); - uv_buf_t buf = uv_buf_init("a", 1); + uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); @@ -253,6 +351,7 @@ void onAccept(uv_stream_t* stream, int status) { } } void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + tDebug("connection coming"); if (nread < 0) { if (nread != UV_EOF) { tError("read error %s", uv_err_name(nread)); @@ -261,6 +360,11 @@ void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_close((uv_handle_t*)q, NULL); return; } + // free memory allocated by + assert(nread == strlen(notify)); + assert(buf->base[0] == notify[0]); + free(buf->base); + SThreadObj* pObj = (SThreadObj*)container_of(q, struct SThreadObj, pipe); uv_pipe_t* pipe = (uv_pipe_t*)q; @@ -268,30 +372,33 @@ void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tError("No pending count"); return; } + uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConnCtx* pConn = malloc(sizeof(SConnCtx)); + SConnCtx* pConn = connCtxCreate(); /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); uv_timer_init(pObj->loop, pConn->pTimer); - pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); pConn->pWorkerAsync = pObj->workerAsync; // thread safty + + // init client handle + pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, pConn->pTcp); + pConn->pTcp->data = pConn; + + // init write request, just + pConn->pWriter = calloc(1, sizeof(uv_write_t)); + pConn->pWriter->data = pConn; if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tDebug("new connection created: %d", fd); - uv_timer_start(pConn->pTimer, onTimeout, 10, 0); - uv_read_start((uv_stream_t*)(pConn->pTcp), allocBuffer, onRead); + uv_read_start((uv_stream_t*)(pConn->pTcp), allocReadBuffer, onRead); } else { - uv_timer_stop(pConn->pTimer); - free(pConn->pTimer); - uv_close((uv_handle_t*)pConn->pTcp, NULL); - free(pConn->pTcp); - free(pConn); + connCtxDestroy(pConn); } } @@ -325,11 +432,30 @@ void* workerThread(void* arg) { pObj->workerAsync = malloc(sizeof(uv_async_t)); uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); - // pObj->workerAsync->data = (void*)pObj; - - uv_read_start((uv_stream_t*)pObj->pipe, allocBuffer, onConnection); + uv_read_start((uv_stream_t*)pObj->pipe, allocConnBuffer, onConnection); uv_run(pObj->loop, UV_RUN_DEFAULT); } +static SConnCtx* connCtxCreate() { + SConnCtx* pConn = (SConnCtx*)calloc(1, sizeof(SConnCtx)); + return pConn; +} +static void connCtxDestroy(SConnCtx* ctx) { + if (ctx == NULL) { + return; + } + uv_timer_stop(ctx->pTimer); + free(ctx->pTimer); + uv_close((uv_handle_t*)ctx->pTcp, NULL); + free(ctx->pTcp); + free(ctx->pWriter); + free(ctx); + // handle +} +static void uvConnCtxDestroy(uv_handle_t* handle) { + SConnCtx* ctx = handle->data; + connCtxDestroy(ctx); +} + #else #define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) From fdb79077c4c2d1c945a234a66d123a93794329df Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 15 Jan 2022 18:20:53 +0800 Subject: [PATCH 58/63] add libuv --- source/libs/transport/src/rpcMain.c | 208 +++++++++++++--------------- 1 file changed, 98 insertions(+), 110 deletions(-) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index a1c0c05fc3..37ef10ba5b 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -66,10 +66,31 @@ typedef struct { struct SRpcConn* connList; // connection list } SRpcInfo; +typedef struct { + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + SEpSet* pSet; // for synchronous API + char msg[0]; // RpcHead starts from here +} SRpcReqContext; + #ifdef USE_UV #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) static const char* notify = "a"; typedef struct SThreadObj { @@ -94,12 +115,12 @@ typedef struct SServerObj { uint32_t port; } SServerObj; -typedef struct SContent { +typedef struct SConnBuffer { char* buf; int len; int cap; - int toRead; -} SContent; + int left; +} SConnBuffer; typedef struct SConnCtx { uv_tcp_t* pTcp; @@ -110,18 +131,18 @@ typedef struct SConnCtx { queue queue; int ref; int persist; // persist connection or not - SContent pCont; + SConnBuffer connBuf; int count; } SConnCtx; -static void allocReadBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -static void allocConnBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void onTimeout(uv_timer_t* handle); -static void onWrite(uv_write_t* req, int status); -static void onAccept(uv_stream_t* stream, int status); -static void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); -static void workerAsyncCB(uv_async_t* handle); +static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void uvOnTimeoutCb(uv_timer_t* handle); +static void uvOnWriteCb(uv_write_t* req, int status); +static void uvOnAcceptCb(uv_stream_t* stream, int status); +static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void uvWorkerAsyncCb(uv_async_t* handle); static SConnCtx* connCtxCreate(); static void connCtxDestroy(SConnCtx* ctx); @@ -193,95 +214,68 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; } -void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } -void rpcFreeCont(void* cont) { return; } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } - -void rpcSendResponse(const SRpcMsg* pMsg) {} - -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } -int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } -void rpcCancelRequest(int64_t rid) { return; } - -void allocReadBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { static const int CAPACITY = 1024; - tDebug("pre alloc buffer for read "); - SConnCtx* ctx = handle->data; - SContent* pCont = &ctx->pCont; - if (pCont->cap == 0) { - pCont->buf = (char*)calloc(CAPACITY, sizeof(char)); - pCont->len = 0; - pCont->cap = CAPACITY; - pCont->toRead = -1; + /* + * formate of data buffer: + * |<-------SRpcReqContext------->|<------------data read from socket----------->| + */ - buf->base = pCont->buf; + SConnCtx* ctx = handle->data; + SConnBuffer* pBuf = &ctx->connBuf; + if (pBuf->cap == 0) { + pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); + pBuf->len = 0; + pBuf->cap = CAPACITY; + pBuf->left = -1; + + buf->base = pBuf->buf + RPC_RESERVE_SIZE; buf->len = CAPACITY; } else { - if (pCont->len >= pCont->cap) { - if (pCont->toRead == -1) { - pCont->cap *= 2; - pCont->buf = realloc(pCont->buf, pCont->cap); - } else if (pCont->len + pCont->toRead > pCont->cap) { - pCont->cap = pCont->len + pCont->toRead; - pCont->buf = realloc(pCont->buf, pCont->len + pCont->toRead); + if (pBuf->len >= pBuf->cap) { + if (pBuf->left == -1) { + pBuf->cap *= 2; + pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); + } else if (pBuf->len + pBuf->left > pBuf->cap) { + pBuf->cap = pBuf->len + pBuf->left; + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); } } - buf->base = pCont->buf + pCont->len; - buf->len = pCont->cap - pCont->len; + buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; + buf->len = pBuf->cap - pBuf->len; } - - // if (ctx->pCont.cap == 0) { - // ctx->pCont.buf = (char*)calloc(64, sizeof(char)); - // ctx->pCont.len = 0; - // ctx->pCont.cap = 64; - // // - // buf->base = ctx->pCont.buf; - // buf->len = sz; - //} else { - // if (ctx->pCont.len + sz > ctx->pCont.cap) { - // ctx->pCont.cap *= 2; - // ctx->pCont.buf = realloc(ctx->pCont.buf, ctx->pCont.cap); - // } - // buf->base = ctx->pCont.buf + ctx->pCont.len; - // buf->len = sz; - //} } -// change later -static bool handleUserData(SContent* data) { +// check data read from socket completely or not +// +static bool isReadAll(SConnBuffer* data) { + // TODO(yihao): handle pipeline later SRpcHead rpcHead; - - bool finish = false; - int32_t msgLen, leftLen, retLen; - int32_t headLen = sizeof(rpcHead); + int32_t headLen = sizeof(rpcHead); if (data->len >= headLen) { memcpy((char*)&rpcHead, data->buf, headLen); - msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - if (msgLen + headLen <= data->len) { - return true; - } else { + int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + if (msgLen > data->len) { + data->left = msgLen - data->len; return false; + } else { + return true; } } else { return false; } } -void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { +void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // opt - SConnCtx* ctx = cli->data; - SContent* pCont = &ctx->pCont; + SConnCtx* ctx = cli->data; + SConnBuffer* pBuf = &ctx->connBuf; if (nread > 0) { - pCont->len += nread; - bool finish = handleUserData(pCont); - if (finish == false) { - tDebug("continue read"); + pBuf->len += nread; + if (isReadAll(pBuf)) { + tDebug("alread read complete packet"); } else { - tDebug("read completely"); + tDebug("read half packet, continue to read"); } return; } @@ -291,17 +285,17 @@ void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } uv_close((uv_handle_t*)cli, uvConnCtxDestroy); } -void allocConnBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->base = malloc(sizeof(char)); buf->len = 2; } -void onTimeout(uv_timer_t* handle) { +void uvOnTimeoutCb(uv_timer_t* handle) { // opt tDebug("time out"); } -void onWrite(uv_write_t* req, int status) { +void uvOnWriteCb(uv_write_t* req, int status) { SConnCtx* ctx = req->data; if (status == 0) { tDebug("data already was written on stream"); @@ -311,7 +305,7 @@ void onWrite(uv_write_t* req, int status) { // opt } -void workerAsyncCB(uv_async_t* handle) { +void uvWorkerAsyncCb(uv_async_t* handle) { SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); SConnCtx* conn = NULL; @@ -329,7 +323,7 @@ void workerAsyncCB(uv_async_t* handle) { } } -void onAccept(uv_stream_t* stream, int status) { +void uvOnAcceptCb(uv_stream_t* stream, int status) { if (status == -1) { return; } @@ -345,12 +339,12 @@ void onAccept(uv_stream_t* stream, int status) { pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); - uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); } } -void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { +void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tDebug("connection coming"); if (nread < 0) { if (nread != UV_EOF) { @@ -396,7 +390,7 @@ void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); tDebug("new connection created: %d", fd); - uv_read_start((uv_stream_t*)(pConn->pTcp), allocReadBuffer, onRead); + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { connCtxDestroy(pConn); } @@ -412,7 +406,7 @@ void* acceptThread(void* arg) { uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { tError("Listen error %s\n", uv_err_name(err)); return NULL; } @@ -430,9 +424,9 @@ void* workerThread(void* arg) { QUEUE_INIT(&pObj->conn); pObj->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); + uv_async_init(pObj->loop, pObj->workerAsync, uvWorkerAsyncCb); - uv_read_start((uv_stream_t*)pObj->pipe, allocConnBuffer, onConnection); + uv_read_start((uv_stream_t*)pObj->pipe, uvAllocConnBufferCb, uvOnConnectionCb); uv_run(pObj->loop, UV_RUN_DEFAULT); } static SConnCtx* connCtxCreate() { @@ -455,6 +449,20 @@ static void uvConnCtxDestroy(uv_handle_t* handle) { SConnCtx* ctx = handle->data; connCtxDestroy(ctx); } +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } #else @@ -465,26 +473,6 @@ static void uvConnCtxDestroy(uv_handle_t* handle) { #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) -typedef struct { - SRpcInfo * pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void * ahandle; // handle provided by app - struct SRpcConn *pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t * pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg * pRsp; // for synchronous API - tsem_t * pSem; // for synchronous API - SEpSet * pSet; // for synchronous API - char msg[0]; // RpcHead starts from here -} SRpcReqContext; - typedef struct SRpcConn { char info[48]; // debug info: label + pConn + ahandle int sid; // session ID From 60b48e3ee0b06a7e6591ff5e8d88c584c6ae68d4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 16 Jan 2022 11:58:45 +0800 Subject: [PATCH 59/63] refactor code --- source/libs/transport/inc/rpcHead.h | 26 - source/libs/transport/inc/rpcTcp.h | 4 - source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/rpcCache.c | 4 - source/libs/transport/src/rpcMain.c | 439 +------------- source/libs/transport/src/rpcTcp.c | 8 - source/libs/transport/src/rpcUdp.c | 4 - source/libs/transport/src/transport.c | 697 +++++++++++++++++++++++ 8 files changed, 727 insertions(+), 456 deletions(-) diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 66821db133..5ddf1a83c9 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -21,31 +21,6 @@ extern "C" { #endif -#ifdef USE_UV -typedef struct { - char version : 4; // RPC version - char comp : 4; // compression algorithm, 0:no compression 1:lz4 - char resflag : 2; // reserved bits - char spi : 3; // security parameter index - char encrypt : 3; // encrypt algorithm, 0: no encryption - uint16_t tranId; // transcation ID - uint32_t linkUid; // for unique connection ID assigned by client - uint64_t ahandle; // ahandle assigned by client - uint32_t sourceId; // source ID, an index for connection list - uint32_t destId; // destination ID, an index for connection list - uint32_t destIp; // destination IP address, for NAT scenario - char user[TSDB_UNI_LEN]; // user ID - uint16_t port; // for UDP only, port may be changed - char empty[1]; // reserved - uint16_t msgType; // message type - int32_t msgLen; // message length including the header iteslf - uint32_t msgVer; - int32_t code; // code in response message - uint8_t content[0]; // message body starts from here -} SRpcHead; - -#else - #define RPC_CONN_TCP 2 extern int tsRpcOverhead; @@ -96,7 +71,6 @@ typedef struct { } SRpcDigest; #pragma pack(pop) -#endif #ifdef __cplusplus } diff --git a/source/libs/transport/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h index 5e5c43a1db..ad42307516 100644 --- a/source/libs/transport/inc/rpcTcp.h +++ b/source/libs/transport/inc/rpcTcp.h @@ -21,8 +21,6 @@ extern "C" { #endif -#ifdef USE_UV -#else void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); void taosStopTcpServer(void *param); void taosCleanUpTcpServer(void *param); @@ -35,8 +33,6 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin void taosCloseTcpConnection(void *chandle); int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); -#endif - #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 067b371b84..f93753cfe9 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,6 +16,7 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#include "rpcHead.h" #ifdef __cplusplus extern "C" { #endif diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 40767d2ba5..1db2808126 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -22,9 +22,6 @@ #include "ttimer.h" #include "tutil.h" -#ifdef USE_UV - -#else typedef struct SConnHash { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -295,4 +292,3 @@ static void rpcUnlockCache(int64_t *lockedBy) { assert(false); } } -#endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 37ef10ba5b..f381768a34 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#ifdef USE_UV -#include -#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" @@ -36,6 +33,17 @@ #include "ttimer.h" #include "tutil.h" +static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; + +int tsRpcMaxUdpSize = 15000; // bytes +int tsProgressTimer = 100; +// not configurable +int tsRpcMaxRetry; +int tsRpcHeadSize; +int tsRpcOverhead; + +#ifndef USE_UV + typedef struct { int sessions; // number of sessions allowed int numOfThreads; // number of threads to process incoming messages @@ -51,28 +59,28 @@ typedef struct { char secret[TSDB_PASSWORD_LEN]; // secret for the link char ckey[TSDB_PASSWORD_LEN]; // ciphering key - void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; - void* parent; - void* idPool; // handle to ID pool - void* tmrCtrl; // handle to timer - SHashObj* hash; // handle returned by hash utility - void* tcphandle; // returned handle from TCP initialization - void* udphandle; // returned handle from UDP initialization - void* pCache; // connection cache + void * parent; + void * idPool; // handle to ID pool + void * tmrCtrl; // handle to timer + SHashObj * hash; // handle returned by hash utility + void * tcphandle; // returned handle from TCP initialization + void * udphandle; // returned handle from UDP initialization + void * pCache; // connection cache pthread_mutex_t mutex; - struct SRpcConn* connList; // connection list + struct SRpcConn *connList; // connection list } SRpcInfo; typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo + SRpcInfo * pRpc; // associated SRpcInfo SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - struct SRpcConn* pConn; // pConn allocated + void * ahandle; // handle provided by app + struct SRpcConn *pConn; // pConn allocated tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app + uint8_t * pCont; // content provided by app int32_t contLen; // content length int32_t code; // error code int16_t numOfTry; // number of try for different servers @@ -80,394 +88,14 @@ typedef struct { int8_t redirect; // flag to indicate redirect int8_t connType; // connection type int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - SEpSet* pSet; // for synchronous API + SRpcMsg * pRsp; // for synchronous API + tsem_t * pSem; // for synchronous API + SEpSet * pSet; // for synchronous API char msg[0]; // RpcHead starts from here } SRpcReqContext; -#ifdef USE_UV - -#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) - -#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) -static const char* notify = "a"; - -typedef struct SThreadObj { - pthread_t thread; - uv_pipe_t* pipe; - uv_loop_t* loop; - uv_async_t* workerAsync; // - int fd; - queue conn; - pthread_mutex_t connMtx; -} SThreadObj; - -typedef struct SServerObj { - pthread_t thread; - uv_tcp_t server; - uv_loop_t* loop; - int workerIdx; - int numOfThread; - SThreadObj** pThreadObj; - uv_pipe_t** pipe; - uint32_t ip; - uint32_t port; -} SServerObj; - -typedef struct SConnBuffer { - char* buf; - int len; - int cap; - int left; -} SConnBuffer; - -typedef struct SConnCtx { - uv_tcp_t* pTcp; - uv_write_t* pWriter; - uv_timer_t* pTimer; - - uv_async_t* pWorkerAsync; - queue queue; - int ref; - int persist; // persist connection or not - SConnBuffer connBuf; - int count; -} SConnCtx; - -static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -static void uvOnTimeoutCb(uv_timer_t* handle); -static void uvOnWriteCb(uv_write_t* req, int status); -static void uvOnAcceptCb(uv_stream_t* stream, int status); -static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); -static void uvWorkerAsyncCb(uv_async_t* handle); - -static SConnCtx* connCtxCreate(); -static void connCtxDestroy(SConnCtx* ctx); -static void uvConnCtxDestroy(uv_handle_t* handle); - -static void* workerThread(void* arg); -static void* acceptThread(void* arg); - -void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); - -int32_t rpcInit() { return -1; } -void rpcCleanup() { return; }; - -void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - // opte -} -void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SServerObj* srv = calloc(1, sizeof(SServerObj)); - srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - srv->numOfThread = numOfThreads; - srv->workerIdx = 0; - srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); - srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); - srv->ip = ip; - srv->port = port; - uv_loop_init(srv->loop); - - for (int i = 0; i < srv->numOfThread; i++) { - SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); - int fds[2]; - if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { - return NULL; - } - uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); - uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - - thrd->fd = fds[0]; - thrd->pipe = &(srv->pipe[i][1]); // init read - int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); - if (err == 0) { - tDebug("sucess to create worker-thread %d", i); - // printf("thread %d create\n", i); - } else { - // TODO: clear all other resource later - tError("failed to create worker-thread %d", i); - } - srv->pThreadObj[i] = thrd; - } - - int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); - if (err == 0) { - tDebug("success to create accept-thread"); - } else { - // clear all resource later - } - - return srv; -} -void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); - if (pRpc == NULL) { - return NULL; - } - if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); - } - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); - return pRpc; -} - -void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - static const int CAPACITY = 1024; - /* - * formate of data buffer: - * |<-------SRpcReqContext------->|<------------data read from socket----------->| - */ - - SConnCtx* ctx = handle->data; - SConnBuffer* pBuf = &ctx->connBuf; - if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); - pBuf->len = 0; - pBuf->cap = CAPACITY; - pBuf->left = -1; - - buf->base = pBuf->buf + RPC_RESERVE_SIZE; - buf->len = CAPACITY; - } else { - if (pBuf->len >= pBuf->cap) { - if (pBuf->left == -1) { - pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); - } else if (pBuf->len + pBuf->left > pBuf->cap) { - pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); - } - } - buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; - buf->len = pBuf->cap - pBuf->len; - } -} -// check data read from socket completely or not -// -static bool isReadAll(SConnBuffer* data) { - // TODO(yihao): handle pipeline later - SRpcHead rpcHead; - int32_t headLen = sizeof(rpcHead); - if (data->len >= headLen) { - memcpy((char*)&rpcHead, data->buf, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - if (msgLen > data->len) { - data->left = msgLen - data->len; - return false; - } else { - return true; - } - } else { - return false; - } -} - -void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { - // opt - SConnCtx* ctx = cli->data; - SConnBuffer* pBuf = &ctx->connBuf; - if (nread > 0) { - pBuf->len += nread; - if (isReadAll(pBuf)) { - tDebug("alread read complete packet"); - } else { - tDebug("read half packet, continue to read"); - } - return; - } - - if (nread != UV_EOF) { - tDebug("Read error %s\n", uv_err_name(nread)); - } - uv_close((uv_handle_t*)cli, uvConnCtxDestroy); -} -void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - buf->base = malloc(sizeof(char)); - buf->len = 2; -} - -void uvOnTimeoutCb(uv_timer_t* handle) { - // opt - tDebug("time out"); -} - -void uvOnWriteCb(uv_write_t* req, int status) { - SConnCtx* ctx = req->data; - if (status == 0) { - tDebug("data already was written on stream"); - } else { - connCtxDestroy(ctx); - } - // opt -} - -void uvWorkerAsyncCb(uv_async_t* handle) { - SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); - SConnCtx* conn = NULL; - - // opt later - pthread_mutex_lock(&pObj->connMtx); - if (!QUEUE_IS_EMPTY(&pObj->conn)) { - queue* head = QUEUE_HEAD(&pObj->conn); - conn = QUEUE_DATA(head, SConnCtx, queue); - QUEUE_REMOVE(&conn->queue); - } - pthread_mutex_unlock(&pObj->connMtx); - if (conn == NULL) { - tError("except occurred, do nothing"); - return; - } -} - -void uvOnAcceptCb(uv_stream_t* stream, int status) { - if (status == -1) { - return; - } - SServerObj* pObj = container_of(stream, SServerObj, server); - - uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pObj->loop, cli); - - if (uv_accept(stream, (uv_stream_t*)cli) == 0) { - uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); - - uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); - - pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; - tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); - uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); - } else { - uv_close((uv_handle_t*)cli, NULL); - } -} -void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { - tDebug("connection coming"); - if (nread < 0) { - if (nread != UV_EOF) { - tError("read error %s", uv_err_name(nread)); - } - // TODO(log other failure reason) - uv_close((uv_handle_t*)q, NULL); - return; - } - // free memory allocated by - assert(nread == strlen(notify)); - assert(buf->base[0] == notify[0]); - free(buf->base); - - SThreadObj* pObj = (SThreadObj*)container_of(q, struct SThreadObj, pipe); - - uv_pipe_t* pipe = (uv_pipe_t*)q; - if (!uv_pipe_pending_count(pipe)) { - tError("No pending count"); - return; - } - - uv_handle_type pending = uv_pipe_pending_type(pipe); - assert(pending == UV_TCP); - - SConnCtx* pConn = connCtxCreate(); - /* init conn timer*/ - pConn->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pObj->loop, pConn->pTimer); - - pConn->pWorkerAsync = pObj->workerAsync; // thread safty - - // init client handle - pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pObj->loop, pConn->pTcp); - pConn->pTcp->data = pConn; - - // init write request, just - pConn->pWriter = calloc(1, sizeof(uv_write_t)); - pConn->pWriter->data = pConn; - - if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { - uv_os_fd_t fd; - uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); - tDebug("new connection created: %d", fd); - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); - } else { - connCtxDestroy(pConn); - } -} - -void* acceptThread(void* arg) { - // opt - SServerObj* srv = (SServerObj*)arg; - uv_tcp_init(srv->loop, &srv->server); - - struct sockaddr_in bind_addr; - - uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); - uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); - int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { - tError("Listen error %s\n", uv_err_name(err)); - return NULL; - } - uv_run(srv->loop, UV_RUN_DEFAULT); -} -void* workerThread(void* arg) { - SThreadObj* pObj = (SThreadObj*)arg; - - pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pObj->loop); - - uv_pipe_init(pObj->loop, pObj->pipe, 1); - uv_pipe_open(pObj->pipe, pObj->fd); - - QUEUE_INIT(&pObj->conn); - - pObj->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pObj->loop, pObj->workerAsync, uvWorkerAsyncCb); - - uv_read_start((uv_stream_t*)pObj->pipe, uvAllocConnBufferCb, uvOnConnectionCb); - uv_run(pObj->loop, UV_RUN_DEFAULT); -} -static SConnCtx* connCtxCreate() { - SConnCtx* pConn = (SConnCtx*)calloc(1, sizeof(SConnCtx)); - return pConn; -} -static void connCtxDestroy(SConnCtx* ctx) { - if (ctx == NULL) { - return; - } - uv_timer_stop(ctx->pTimer); - free(ctx->pTimer); - uv_close((uv_handle_t*)ctx->pTcp, NULL); - free(ctx->pTcp); - free(ctx->pWriter); - free(ctx); - // handle -} -static void uvConnCtxDestroy(uv_handle_t* handle) { - SConnCtx* ctx = handle->data; - connCtxDestroy(ctx); -} -void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } -void rpcFreeCont(void* cont) { return; } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } - -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } - -void rpcSendResponse(const SRpcMsg* pMsg) {} - -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } -int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } -void rpcCancelRequest(int64_t rid) { return; } - -#else - #define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) +#define rpcHeadFromCont(cont) ((SRpcHead *)((char *)cont - sizeof(SRpcHead))) #define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) @@ -510,15 +138,6 @@ typedef struct SRpcConn { SRpcReqContext *pContext; // request context } SRpcConn; -static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; - -int tsRpcMaxUdpSize = 15000; // bytes -int tsProgressTimer = 100; -// not configurable -int tsRpcMaxRetry; -int tsRpcHeadSize; -int tsRpcOverhead; - static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 9fa51a6fdc..81c464a661 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -14,9 +14,6 @@ */ #include "rpcTcp.h" -#ifdef USE_UV -#include -#endif #include "os.h" #include "rpcHead.h" #include "rpcLog.h" @@ -24,9 +21,6 @@ #include "taoserror.h" #include "tutil.h" -#ifdef USE_UV - -#else typedef struct SFdObj { void * signature; SOCKET fd; // TCP socket FD @@ -662,5 +656,3 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tfree(pFdObj); } - -#endif diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 79956cc98d..b57cf57c55 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -22,9 +22,6 @@ #include "ttimer.h" #include "tutil.h" -#ifdef USE_UV -// no support upd currently -#else #define RPC_MAX_UDP_CONNS 256 #define RPC_MAX_UDP_PKTS 1000 #define RPC_UDP_BUF_TIME 5 // mseconds @@ -260,4 +257,3 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c return ret; } -#endif diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index f2f48bbc8a..a6c9cee0b7 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -12,3 +12,700 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#ifdef USE_UV + +#include +#include "lz4.h" +#include "os.h" +#include "rpcCache.h" +#include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tidpool.h" +#include "tmd5.h" +#include "tmempool.h" +#include "tmsg.h" +#include "transportInt.h" +#include "tref.h" +#include "trpc.h" +#include "ttimer.h" +#include "tutil.h" + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) +static const char* notify = "a"; + +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + int index; // for UDP server only, round robin for multiple threads + char label[TSDB_LABEL_LEN]; + + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization + void* udphandle; // returned handle from UDP initialization + void* pCache; // connection cache + pthread_mutex_t mutex; + struct SRpcConn* connList; // connection list +} SRpcInfo; + +typedef struct { + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + SEpSet* pSet; // for synchronous API + char msg[0]; // RpcHead starts from here +} SRpcReqContext; + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + uv_async_t* workerAsync; // + queue conn; + pthread_mutex_t connMtx; + void* shandle; +} SThreadObj; + +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) +#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) +#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) +#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) +#define rpcIsReq(type) (type & 1U) + +typedef struct SServerObj { + pthread_t thread; + uv_tcp_t server; + uv_loop_t* loop; + int workerIdx; + int numOfThread; + SThreadObj** pThreadObj; + uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; +} SServerObj; + +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; +} SConnBuffer; + +typedef struct SRpcConn { + uv_tcp_t* pTcp; + uv_write_t* pWriter; + uv_timer_t* pTimer; + + uv_async_t* pWorkerAsync; + queue queue; + int ref; + int persist; // persist connection or not + SConnBuffer connBuf; + int count; + void* shandle; // rpc init + void* ahandle; + + // del later + char secured; + int spi; + char info[64]; + char user[TSDB_UNI_LEN]; // user ID for the link + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; // ciphering key +} SRpcConn; + +// auth function +static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen); +// compress data +static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); + +static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void uvOnTimeoutCb(uv_timer_t* handle); +static void uvOnWriteCb(uv_write_t* req, int status); +static void uvOnAcceptCb(uv_stream_t* stream, int status); +static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void uvWorkerAsyncCb(uv_async_t* handle); + +static SRpcConn* connCreate(); +static void connDestroy(SRpcConn* conn); +static void uvConnDestroy(uv_handle_t* handle); + +static void* workerThread(void* arg); +static void* acceptThread(void* arg); + +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); + +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + SServerObj* srv = calloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + srv->numOfThread = numOfThreads; + srv->workerIdx = 0; + srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); + srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); + srv->ip = ip; + srv->port = port; + uv_loop_init(srv->loop); + + for (int i = 0; i < srv->numOfThread; i++) { + SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + return NULL; + } + uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); + uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write + + thrd->shandle = shandle; + thrd->fd = fds[0]; + thrd->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); + if (err == 0) { + tDebug("sucess to create worker-thread %d", i); + // printf("thread %d create\n", i); + } else { + // TODO: clear all other resource later + tError("failed to create worker-thread %d", i); + } + srv->pThreadObj[i] = thrd; + } + + int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + if (err == 0) { + tDebug("success to create accept-thread"); + } else { + // clear all resource later + } + + return srv; +} +void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + /* + * formate of data buffer: + * |<-------SRpcReqContext------->|<------------data read from socket----------->| + */ + static const int CAPACITY = 1024; + + SRpcConn* ctx = handle->data; + SConnBuffer* pBuf = &ctx->connBuf; + if (pBuf->cap == 0) { + pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); + pBuf->len = 0; + pBuf->cap = CAPACITY; + pBuf->left = -1; + + buf->base = pBuf->buf + RPC_RESERVE_SIZE; + buf->len = CAPACITY; + } else { + if (pBuf->len >= pBuf->cap) { + if (pBuf->left == -1) { + pBuf->cap *= 2; + pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); + } else if (pBuf->len + pBuf->left > pBuf->cap) { + pBuf->cap = pBuf->len + pBuf->left; + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); + } + } + buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; + buf->len = pBuf->cap - pBuf->len; + } +} +// check data read from socket completely or not +// +static bool isReadAll(SConnBuffer* data) { + // TODO(yihao): handle pipeline later + SRpcHead rpcHead; + int32_t headLen = sizeof(rpcHead); + if (data->len >= headLen) { + memcpy((char*)&rpcHead, data->buf, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + if (msgLen > data->len) { + data->left = msgLen - data->len; + return false; + } else { + return true; + } + } else { + return false; + } +} +static void uvDoProcess(SRecvInfo* pRecv) { + SRpcHead* pHead = (SRpcHead*)pRecv->msg; + SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; + SRpcConn* pConn = pRecv->thandle; + + tDump(pRecv->msg, pRecv->msgLen); + + terrno = 0; + SRpcReqContext* pContest; + + // do auth and check +} +static int uvAuthData(SRpcConn* pConn, char* msg, int len) { + SRpcHead* pHead = (SRpcHead*)msg; + int code = 0; + + if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { + // secured link, or no authentication + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, secured link, no auth is required", pConn->info); + return 0; + } + + if (!rpcIsReq(pHead->msgType)) { + // for response, if code is auth failure, it shall bypass the auth process + code = htonl(pHead->code); + if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || + code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + return 0; + } + } + + code = 0; + if (pHead->spi == pConn->spi) { + // authentication + SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); + + int32_t delta; + delta = (int32_t)htonl(pDigest->timeStamp); + delta -= (int32_t)taosGetTimestampSec(); + if (abs(delta) > 900) { + tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); + code = TSDB_CODE_RPC_INVALID_TIME_STAMP; + } else { + if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + // tDebug("%s, authentication failed, msg discarded", pConn->info); + code = TSDB_CODE_RPC_AUTH_FAILURE; + } else { + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); + if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client + // tTrace("%s, message is authenticated", pConn->info); + } + } + } else { + tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); + code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; + } + + return code; +} +static void uvProcessData(SRpcConn* ctx) { + SRecvInfo info; + SRecvInfo* p = &info; + SConnBuffer* pBuf = &ctx->connBuf; + p->msg = pBuf->buf + RPC_RESERVE_SIZE; + p->msgLen = pBuf->len; + p->ip = 0; + p->port = 0; + p->shandle = ctx->shandle; // + p->thandle = ctx; + p->chandle = NULL; + + // + SRpcHead* pHead = (SRpcHead*)p->msg; + assert(rpcIsReq(pHead->msgType)); + + SRpcInfo* pRpc = (SRpcInfo*)p->shandle; + SRpcConn* pConn = (SRpcConn*)p->thandle; + + pConn->ahandle = (void*)pHead->ahandle; + pHead->code = htonl(pHead->code); + + SRpcMsg rpcMsg; + + pHead = rpcDecompressRpcMsg(pHead); + rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); + rpcMsg.pCont = pHead->content; + rpcMsg.msgType = pHead->msgType; + rpcMsg.code = pHead->code; + rpcMsg.ahandle = pConn->ahandle; + rpcMsg.handle = pConn; + (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); + // auth + // validate msg type +} +void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + SRpcConn* ctx = cli->data; + SConnBuffer* pBuf = &ctx->connBuf; + if (nread > 0) { + pBuf->len += nread; + if (isReadAll(pBuf)) { + tDebug("alread read complete packet"); + uvProcessData(ctx); + } else { + tDebug("read half packet, continue to read"); + } + return; + } + + if (nread != UV_EOF) { + tDebug("Read error %s\n", uv_err_name(nread)); + } + uv_close((uv_handle_t*)cli, uvConnDestroy); +} +void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(sizeof(char)); + buf->len = 2; +} + +void uvOnTimeoutCb(uv_timer_t* handle) { + // opt + tDebug("time out"); +} + +void uvOnWriteCb(uv_write_t* req, int status) { + SRpcConn* conn = req->data; + if (status == 0) { + tDebug("data already was written on stream"); + } else { + connDestroy(conn); + } + // opt +} + +void uvWorkerAsyncCb(uv_async_t* handle) { + SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); + SRpcConn* conn = NULL; + + // opt later + pthread_mutex_lock(&pObj->connMtx); + if (!QUEUE_IS_EMPTY(&pObj->conn)) { + queue* head = QUEUE_HEAD(&pObj->conn); + conn = QUEUE_DATA(head, SRpcConn, queue); + QUEUE_REMOVE(&conn->queue); + } + pthread_mutex_unlock(&pObj->connMtx); + if (conn == NULL) { + tError("except occurred, do nothing"); + return; + } +} + +void uvOnAcceptCb(uv_stream_t* stream, int status) { + if (status == -1) { + return; + } + SServerObj* pObj = container_of(stream, SServerObj, server); + + uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, cli); + + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + + uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); + + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); + } else { + uv_close((uv_handle_t*)cli, NULL); + } +} +void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + tDebug("connection coming"); + if (nread < 0) { + if (nread != UV_EOF) { + tError("read error %s", uv_err_name(nread)); + } + // TODO(log other failure reason) + uv_close((uv_handle_t*)q, NULL); + return; + } + // free memory allocated by + assert(nread == strlen(notify)); + assert(buf->base[0] == notify[0]); + free(buf->base); + + SThreadObj* pObj = q->data; + + uv_pipe_t* pipe = (uv_pipe_t*)q; + if (!uv_pipe_pending_count(pipe)) { + tError("No pending count"); + return; + } + + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SRpcConn* pConn = connCreate(); + pConn->shandle = pObj->shandle; + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pObj->loop, pConn->pTimer); + + pConn->pWorkerAsync = pObj->workerAsync; // thread safty + + // init client handle + pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, pConn->pTcp); + pConn->pTcp->data = pConn; + + // init write request, just + pConn->pWriter = calloc(1, sizeof(uv_write_t)); + pConn->pWriter->data = pConn; + + if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); + tDebug("new connection created: %d", fd); + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } else { + connDestroy(pConn); + } +} + +void* acceptThread(void* arg) { + // opt + SServerObj* srv = (SServerObj*)arg; + uv_tcp_init(srv->loop, &srv->server); + + struct sockaddr_in bind_addr; + + uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); +} +void* workerThread(void* arg) { + SThreadObj* pObj = (SThreadObj*)arg; + + pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pObj->loop); + + uv_pipe_init(pObj->loop, pObj->pipe, 1); + uv_pipe_open(pObj->pipe, pObj->fd); + + pObj->pipe->data = pObj; + + QUEUE_INIT(&pObj->conn); + + pObj->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pObj->loop, pObj->workerAsync, uvWorkerAsyncCb); + + uv_read_start((uv_stream_t*)pObj->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + uv_run(pObj->loop, UV_RUN_DEFAULT); +} +static SRpcConn* connCreate() { + SRpcConn* pConn = (SRpcConn*)calloc(1, sizeof(SRpcConn)); + return pConn; +} +static void connDestroy(SRpcConn* conn) { + if (conn == NULL) { + return; + } + uv_timer_stop(conn->pTimer); + free(conn->pTimer); + uv_close((uv_handle_t*)conn->pTcp, NULL); + free(conn->pTcp); + free(conn->pWriter); + free(conn); + // handle +} +static void uvConnDestroy(uv_handle_t* handle) { + SRpcConn* conn = handle->data; + connDestroy(conn); +} +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + return NULL; + } + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + return pRpc; +} +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + +static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + int ret = -1; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; + + return ret; +} +static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + memcpy(pAuth, context.digest, sizeof(context.digest)); +} + +static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen) { + SRpcHead* pHead = (SRpcHead*)msg; + + if (pConn->spi && pConn->secured == 0) { + // add auth part + pHead->spi = pConn->spi; + SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen); + pDigest->timeStamp = htonl(taosGetTimestampSec()); + msgLen += sizeof(SRpcDigest); + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + } else { + pHead->spi = 0; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + } + + return msgLen; +} + +static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { + SRpcHead* pHead = rpcHeadFromCont(pCont); + int32_t finalLen = 0; + int overhead = sizeof(SRpcComp); + + if (!NEEDTO_COMPRESSS_MSG(contLen)) { + return contLen; + } + + char* buf = malloc(contLen + overhead + 8); // 8 extra bytes + if (buf == NULL) { + tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); + return contLen; + } + + int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); + tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); + + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (compLen > 0 && compLen < contLen - overhead) { + SRpcComp* pComp = (SRpcComp*)pCont; + pComp->reserved = 0; + pComp->contLen = htonl(contLen); + memcpy(pCont + overhead, buf, compLen); + + pHead->comp = 1; + tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); + finalLen = compLen + overhead; + } else { + finalLen = contLen; + } + + free(buf); + return finalLen; +} + +static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { + int overhead = sizeof(SRpcComp); + SRpcHead* pNewHead = NULL; + uint8_t* pCont = pHead->content; + SRpcComp* pComp = (SRpcComp*)pHead->content; + + if (pHead->comp) { + // decompress the content + assert(pComp->reserved == 0); + int contLen = htonl(pComp->contLen); + + // prepare the temporary buffer to decompress message + char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD); + pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext + + if (pNewHead) { + int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; + int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen); + assert(origLen == contLen); + + memcpy(pNewHead, pHead, sizeof(SRpcHead)); + pNewHead->msgLen = rpcMsgLenFromCont(origLen); + /// rpcFreeMsg(pHead); // free the compressed message buffer + pHead = pNewHead; + tTrace("decomp malloc mem:%p", temp); + } else { + tError("failed to allocate memory to decompress msg, contLen:%d", contLen); + } + } + + return pHead; +} +int32_t rpcInit(void) { + // impl later + return -1; +} + +void rpcCleanup(void) { + // impl later + return; +} +#endif From 1e06149929613e777c332710c72242355750769c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 16 Jan 2022 15:56:48 +0800 Subject: [PATCH 60/63] [td-11818] support prefix in show, and add if not exists check. --- source/client/src/clientImpl.c | 8 +-- source/libs/executor/inc/executorimpl.h | 13 ++-- source/libs/executor/src/executorimpl.c | 83 +++++++++++++++++++++++-- source/libs/parser/inc/astToMsg.h | 2 +- source/libs/parser/src/astToMsg.c | 22 ++++++- source/libs/parser/src/dCDAstProcess.c | 10 ++- 6 files changed, 119 insertions(+), 19 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index df7e10af7d..582d6ffdb8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -71,15 +71,15 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, return NULL; } - char tmp[TSDB_DB_NAME_LEN] = {0}; + char localDb[TSDB_DB_NAME_LEN] = {0}; if (db != NULL) { if(!validateDbName(db)) { terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; return NULL; } - tstrncpy(tmp, db, sizeof(tmp)); - strdequote(tmp); + tstrncpy(localDb, db, sizeof(localDb)); + strdequote(localDb); } char secretEncrypt[32] = {0}; @@ -122,7 +122,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } tfree(key); - return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst); + return taosConnectImpl(ip, user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst); } int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 42c0f2a6e4..34d89a75b3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -374,6 +374,12 @@ typedef struct STaskParam { struct SUdfInfo *pUdfInfo; } STaskParam; +typedef struct SExchangeInfo { + int32_t numOfSources; + SEpSet *pEpset; + int32_t bytes; // total load bytes from remote +} SExchangeInfo; + typedef struct STableScanInfo { void *pTsdbReadHandle; int32_t numOfBlocks; @@ -393,12 +399,9 @@ typedef struct STableScanInfo { SSDataBlock block; int32_t numOfOutput; int64_t elapsedTime; - int32_t tableIndex; - int32_t prevGroupId; // previous table group id int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - STimeWindow window; } STableScanInfo; typedef struct STagScanInfo { @@ -542,8 +545,10 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); +SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t numOfSources, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperator(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 031900d2bd..93f792e6e3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3564,8 +3564,6 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt // } // reverse order time range - SWAP(pTableScanInfo->window.skey, pTableScanInfo->window.ekey, TSKEY); - SET_REVERSE_SCAN_FLAG(pTableScanInfo); // setTaskStatus(pTableScanInfo, QUERY_NOT_COMPLETED); @@ -4913,10 +4911,85 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { pOperator->status = OP_EXEC_DONE; return pBlock; #endif +} + +int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) { } -SOperatorInfo* createTableScanOperator(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { +static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + + *newgroup = false; + + SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { // todo handle malloc error + } + + SEpSet epSet; + + int64_t sId = -1, queryId = 0, taskId = 1, vgId = 1; + pMsg->header.vgId = htonl(vgId); + + pMsg->sId = htobe64(sId); + pMsg->taskId = htobe64(taskId); + pMsg->queryId = htobe64(queryId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("QID:%"PRIx64 ",TID:%"PRIx64 " calloc %d failed", queryId, taskId, (int32_t)sizeof(SMsgSendInfo)); + } + + pMsgSendInfo->param = NULL; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = TDMT_VND_FETCH; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + void* pTransporter = NULL; + int32_t code = asyncSendMsgToServer(pTransporter, &epSet, &transporterId, pMsgSendInfo); + + printf("abc\n"); + getchar(); + + // add it into the sink node + +} + +SOperatorInfo* createExchangeOperatorInfo(const SVgroupInfo* pVgroups, int32_t numOfSources, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + assert(numOfSources > 0); + + SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + if (pInfo == NULL || pOperator == NULL) { + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pInfo->numOfSources = numOfSources; + + pOperator->name = "ExchangeOperator"; + pOperator->operatorType = OP_Exchange; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = NULL; + pOperator->exec = doLoadRemoteData; + pOperator->pTaskInfo = pTaskInfo; + + return pOperator; +} + +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -4995,7 +5068,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableSeqScanOperator"; -// pOperator->operatorType = OP_TableSeqScan; + pOperator->operatorType = OP_TableSeqScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -7284,7 +7357,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask if (pPhyNode->info.type == OP_TableScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - return createTableScanOperator(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); + return createTableScanOperatorInfo(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 8acbc6bc11..8f2c2ad4b3 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -8,7 +8,7 @@ SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, char* msgBuf, int32_t msgLen); +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 5b841594e0..a5f617a78c 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -85,8 +85,12 @@ SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, cha return pMsg; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, char* msgBuf, int32_t msgLen) { +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) { SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); + if (pShowMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return pShowMsg; + } pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { @@ -105,7 +109,18 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, char* msgBuf, if (pShowInfo->showType == TSDB_MGMT_TABLE_STB || pShowInfo->showType == TSDB_MGMT_TABLE_VGROUP) { SName n = {0}; - tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + + if (pShowInfo->prefix.n > 0) { + if (pShowInfo->prefix.n >= TSDB_DB_FNAME_LEN) { + terrno = buildInvalidOperationMsg(pMsgBuf, "prefix name is too long"); + tfree(pShowMsg); + return NULL; + } + tNameSetDbName(&n, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); + } else { + tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + } + tNameGetFullDbName(&n, pShowMsg->db); } @@ -240,6 +255,9 @@ SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len } SMCreateStbReq* pCreateStbMsg = (SMCreateStbReq*)calloc(1, sizeof(SMCreateStbReq) + (numOfCols + numOfTags) * sizeof(SSchema)); + if (pCreateStbMsg == NULL) { + return NULL; + } char* pMsg = NULL; #if 0 diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 4d4d68e962..38f6284398 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -109,7 +109,11 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out } *pEpSet = pCtx->mgmtEpSet; - *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len); + *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf); + if (*output == NULL) { + return terrno; + } + *outputLen = sizeof(SShowReq) /* + htons(pShowMsg->payloadLen)*/; } @@ -312,9 +316,9 @@ int32_t doCheckForCreateTable(SCreateTableSql* pCreateTable, SMsgBuf* pMsgBuf) { assert(pFieldList != NULL); // if sql specifies db, use it, otherwise use default db - SToken* pzTableName = &(pCreateTable->name); + SToken* pNameToken = &(pCreateTable->name); - if (parserValidateNameToken(pzTableName) != TSDB_CODE_SUCCESS) { + if (parserValidateIdToken(pNameToken) != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } From 6600ae49f6edfaa5ac0593b80d34f4798342e076 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 16 Jan 2022 20:03:13 +0800 Subject: [PATCH 61/63] refactor code --- source/libs/transport/src/transport.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index a6c9cee0b7..f33d54c4f3 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -151,6 +151,7 @@ typedef struct SRpcConn { } SRpcConn; // auth function +static int uvAuthMsg(SRpcConn* pConn, char* msg, int msgLen); static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen); @@ -259,7 +260,7 @@ static bool isReadAll(SConnBuffer* data) { SRpcHead rpcHead; int32_t headLen = sizeof(rpcHead); if (data->len >= headLen) { - memcpy((char*)&rpcHead, data->buf, headLen); + memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen); int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); if (msgLen > data->len) { data->left = msgLen - data->len; @@ -283,7 +284,7 @@ static void uvDoProcess(SRecvInfo* pRecv) { // do auth and check } -static int uvAuthData(SRpcConn* pConn, char* msg, int len) { +static int uvAuthMsg(SRpcConn* pConn, char* msg, int len) { SRpcHead* pHead = (SRpcHead*)msg; int code = 0; @@ -334,16 +335,16 @@ static int uvAuthData(SRpcConn* pConn, char* msg, int len) { return code; } -static void uvProcessData(SRpcConn* ctx) { +static void uvProcessData(SRpcConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; - SConnBuffer* pBuf = &ctx->connBuf; + SConnBuffer* pBuf = &pConn->connBuf; p->msg = pBuf->buf + RPC_RESERVE_SIZE; p->msgLen = pBuf->len; p->ip = 0; p->port = 0; - p->shandle = ctx->shandle; // - p->thandle = ctx; + p->shandle = pConn->shandle; // + p->thandle = pConn; p->chandle = NULL; // @@ -351,9 +352,14 @@ static void uvProcessData(SRpcConn* ctx) { assert(rpcIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; - SRpcConn* pConn = (SRpcConn*)p->thandle; - pConn->ahandle = (void*)pHead->ahandle; + // auth here + + int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); + if (code != 0) { + terrno = code; + } + // rpcCheckAuthentication(pConn, (char*)pHead, pBuf->len); pHead->code = htonl(pHead->code); SRpcMsg rpcMsg; @@ -383,6 +389,9 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } + if (terrno != 0) { + // handle err code + } if (nread != UV_EOF) { tDebug("Read error %s\n", uv_err_name(nread)); @@ -547,6 +556,7 @@ static void connDestroy(SRpcConn* conn) { uv_timer_stop(conn->pTimer); free(conn->pTimer); uv_close((uv_handle_t*)conn->pTcp, NULL); + free(conn->connBuf.buf); free(conn->pTcp); free(conn->pWriter); free(conn); From 41c3160daaef575c209967ec40517361aa649764 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 16 Jan 2022 21:57:42 +0800 Subject: [PATCH 62/63] refactor code --- source/libs/transport/src/transport.c | 81 ++++++++++++++++++--------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index f33d54c4f3..93bbaf2820 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -135,12 +135,13 @@ typedef struct SRpcConn { uv_async_t* pWorkerAsync; queue queue; int ref; - int persist; // persist connection or not - SConnBuffer connBuf; + int persist; // persist connection or not + SConnBuffer connBuf; // read buf, + SConnBuffer writeBuf; // write buf int count; void* shandle; // rpc init - void* ahandle; - + void* ahandle; // + void* hostThread; // del later char secured; int spi; @@ -335,6 +336,11 @@ static int uvAuthMsg(SRpcConn* pConn, char* msg, int len) { return code; } +// refers specifically to query or insert timeout +static void uvHandleActivityTimeout(uv_timer_t* handle) { + // impl later + SRpcConn* conn = handle->data; +} static void uvProcessData(SRpcConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; @@ -358,8 +364,8 @@ static void uvProcessData(SRpcConn* pConn) { int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); if (code != 0) { terrno = code; + return; } - // rpcCheckAuthentication(pConn, (char*)pHead, pBuf->len); pHead->code = htonl(pHead->code); SRpcMsg rpcMsg; @@ -371,7 +377,9 @@ static void uvProcessData(SRpcConn* pConn) { rpcMsg.code = pHead->code; rpcMsg.ahandle = pConn->ahandle; rpcMsg.handle = pConn; + (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); + uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0); // auth // validate msg type } @@ -419,21 +427,23 @@ void uvOnWriteCb(uv_write_t* req, int status) { } void uvWorkerAsyncCb(uv_async_t* handle) { - SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); + SThreadObj* pThrd = container_of(handle, SThreadObj, workerAsync); SRpcConn* conn = NULL; // opt later - pthread_mutex_lock(&pObj->connMtx); - if (!QUEUE_IS_EMPTY(&pObj->conn)) { - queue* head = QUEUE_HEAD(&pObj->conn); + pthread_mutex_lock(&pThrd->connMtx); + if (!QUEUE_IS_EMPTY(&pThrd->conn)) { + queue* head = QUEUE_HEAD(&pThrd->conn); conn = QUEUE_DATA(head, SRpcConn, queue); QUEUE_REMOVE(&conn->queue); } - pthread_mutex_unlock(&pObj->connMtx); + pthread_mutex_unlock(&pThrd->connMtx); if (conn == NULL) { tError("except occurred, do nothing"); return; } + uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); + uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } void uvOnAcceptCb(uv_stream_t* stream, int status) { @@ -472,7 +482,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(buf->base[0] == notify[0]); free(buf->base); - SThreadObj* pObj = q->data; + SThreadObj* pThrd = q->data; uv_pipe_t* pipe = (uv_pipe_t*)q; if (!uv_pipe_pending_count(pipe)) { @@ -484,16 +494,18 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(pending == UV_TCP); SRpcConn* pConn = connCreate(); - pConn->shandle = pObj->shandle; + pConn->shandle = pThrd->shandle; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pObj->loop, pConn->pTimer); + uv_timer_init(pThrd->loop, pConn->pTimer); + pConn->pTimer->data = pConn; - pConn->pWorkerAsync = pObj->workerAsync; // thread safty + pConn->hostThread = pThrd; + pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pObj->loop, pConn->pTcp); + uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; // init write request, just @@ -527,23 +539,23 @@ void* acceptThread(void* arg) { uv_run(srv->loop, UV_RUN_DEFAULT); } void* workerThread(void* arg) { - SThreadObj* pObj = (SThreadObj*)arg; + SThreadObj* pThrd = (SThreadObj*)arg; - pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pObj->loop); + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); - uv_pipe_init(pObj->loop, pObj->pipe, 1); - uv_pipe_open(pObj->pipe, pObj->fd); + uv_pipe_init(pThrd->loop, pThrd->pipe, 1); + uv_pipe_open(pThrd->pipe, pThrd->fd); - pObj->pipe->data = pObj; + pThrd->pipe->data = pThrd; - QUEUE_INIT(&pObj->conn); + QUEUE_INIT(&pThrd->conn); - pObj->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pObj->loop, pObj->workerAsync, uvWorkerAsyncCb); + pThrd->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); - uv_read_start((uv_stream_t*)pObj->pipe, uvAllocConnBufferCb, uvOnConnectionCb); - uv_run(pObj->loop, UV_RUN_DEFAULT); + uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + uv_run(pThrd->loop, UV_RUN_DEFAULT); } static SRpcConn* connCreate() { SRpcConn* pConn = (SRpcConn*)calloc(1, sizeof(SRpcConn)); @@ -583,9 +595,22 @@ void* rpcMallocCont(int contLen) { return NULL; } void rpcFreeCont(void* cont) { return; } void* rpcReallocCont(void* ptr, int contLen) { return NULL; } -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { + // impl later + return; +} -void rpcSendResponse(const SRpcMsg* pMsg) {} +void rpcSendResponse(const SRpcMsg* pMsg) { + SRpcConn* pConn = pMsg->handle; + SThreadObj* pThrd = pConn->hostThread; + + // opt later + pthread_mutex_lock(&pThrd->connMtx); + QUEUE_PUSH(&pThrd->conn, &pConn->queue); + pthread_mutex_unlock(&pThrd->connMtx); + + uv_async_send(pConn->pWorkerAsync); +} void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } From 4a1809a15df4081cf6a8ca3ff6a1d926ae9ade4c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 16 Jan 2022 23:01:16 +0800 Subject: [PATCH 63/63] [td-11818] Add more checks for invalid input, refactor. --- source/client/src/clientImpl.c | 20 ++-- source/client/test/clientTests.cpp | 160 +++++++++++++------------ source/libs/parser/src/astGenerator.c | 5 +- source/libs/parser/src/astToMsg.c | 4 + source/libs/parser/src/dCDAstProcess.c | 1 + source/libs/parser/src/parserUtil.c | 35 ++++-- 6 files changed, 135 insertions(+), 90 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 582d6ffdb8..26d54c1387 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -58,7 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i return strdup(key); } -static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); +static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { @@ -82,7 +82,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, strdequote(localDb); } - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; if (auth == NULL) { if (!validatePassword(pass)) { terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; @@ -111,6 +111,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, char* key = getClusterKey(user, secretEncrypt, ip, port); + // TODO: race condition here. SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); @@ -122,7 +123,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } tfree(key); - return taosConnectImpl(ip, user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst); + return taosConnectImpl(user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst); } int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) { @@ -420,7 +421,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return 0; } -STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { +STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { STscObj *pTscObj = createTscObj(user, auth, db, pAppInfo); if (NULL == pTscObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -479,7 +480,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) { STscObj *pObj = pRequest->pTscObj; char* db = getConnectionDB(pObj); - tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + if (db != NULL) { + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + } tfree(db); pConnect->pid = htonl(appInfo.pid); @@ -679,9 +682,12 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t char* getConnectionDB(STscObj* pObj) { char *p = NULL; pthread_mutex_lock(&pObj->mutex); - p = strndup(pObj->db, tListLen(pObj->db)); - pthread_mutex_unlock(&pObj->mutex); + size_t len = strlen(pObj->db); + if (len > 0) { + p = strndup(pObj->db, tListLen(pObj->db)); + } + pthread_mutex_unlock(&pObj->mutex); return p; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 429657b617..db2881ccc1 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -48,13 +48,13 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); } -TEST(testCase, connect_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); - } - taos_close(pConn); -} +//TEST(testCase, connect_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// if (pConn == NULL) { +// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); +// } +// taos_close(pConn); +//} //TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -262,7 +262,7 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// + //TEST(testCase, create_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -273,13 +273,7 @@ TEST(testCase, connect_Test) { // } // taos_free_result(pRes); // -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); // if (taos_errno(pRes) != 0) { // printf("error in create stable, reason:%s\n", taos_errstr(pRes)); // } @@ -291,22 +285,40 @@ TEST(testCase, connect_Test) { // ASSERT_EQ(numOfFields, 0); // // taos_free_result(pRes); -// taos_close(pConn); -//} // -//TEST(testCase, create_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); +// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); +// } // -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); // taos_free_result(pRes); +// pRes = taos_query(pConn, "drop stable `123_$^)`"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); +// } // // taos_close(pConn); //} -// + +TEST(testCase, create_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + ASSERT_EQ(taos_errno(pRes), 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + ASSERT_NE(taos_errno(pRes), 0); + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, create_ctable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -325,37 +337,37 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// -//TEST(testCase, show_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// + +TEST(testCase, show_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); + // TAOS_RES* pRes = taos_query(pConn, "use abc1"); // if (taos_errno(pRes) != 0) { // printf("failed to use db, reason:%s\n", taos_errstr(pRes)); // } // taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show stables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + + TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); + if (taos_errno(pRes) != 0) { + printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} // //TEST(testCase, show_vgroup_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -514,29 +526,29 @@ TEST(testCase, connect_Test) { // taosHashCleanup(phash); //} // -TEST(testCase, create_topic_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_topic_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == nullptr); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// char* sql = "select * from tu"; +// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +// taos_free_result(pRes); +// taos_close(pConn); +//} // //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 34ed8bd355..8e8da92bf5 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -972,14 +972,15 @@ void tSetDbName(SToken *pCpxName, SToken *pDb) { void tSetColumnInfo(SField *pField, SToken *pName, SField *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - // column name is too long, set the it to be invalid. + // The column name is too long, set it to be invalid. if ((int32_t) pName->n >= maxLen) { - pName->n = -1; + pField->name[0] = 0; } else { strncpy(pField->name, pName->z, pName->n); pField->name[pName->n] = 0; } + // denote an invalid data type in the column definition. pField->type = pType->type; if(!isValidDataType(pField->type)){ pField->bytes = 0; diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index a5f617a78c..697fd0c4cb 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -117,6 +117,10 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, SMsgBuf* pMsgB return NULL; } tNameSetDbName(&n, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); + } else if (pCtx->db == NULL || strlen(pCtx->db) == 0) { + terrno = buildInvalidOperationMsg(pMsgBuf, "database is not specified"); + tfree(pShowMsg); + return NULL; } else { tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db)); } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 38f6284398..955507ae1b 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -977,6 +977,7 @@ SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseCon int32_t msgLen = 0; int32_t code = doCheckAndBuildCreateTableReq(pCreateTable, pCtx, pMsgBuf, (char**) &pModifSqlStmt, &msgLen); if (code != TSDB_CODE_SUCCESS) { + terrno = code; tfree(pModifSqlStmt); return NULL; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7bb0f8ee83..4d506b84a0 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -124,12 +124,13 @@ int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) { } int32_t parserValidateNameToken(SToken* pToken) { - if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) { + if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID || pToken->n == 0) { return TSDB_CODE_TSC_INVALID_OPERATION; } // it is a token quoted with escape char '`' if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) { + pToken->n = strdequote(pToken->z); return TSDB_CODE_SUCCESS; } @@ -1945,17 +1946,30 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) { int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; + const char* msg2 = "invalid database name"; int32_t code = TSDB_CODE_SUCCESS; - char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, false); + char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, true); if (p != NULL) { // db has been specified in sql string so we ignore current db path - tNameSetAcctId(pName, pParseCtx->acctId); + assert(*p == TS_PATH_DELIMITER[0]); - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); + int32_t dbLen = p - pTableName->z; + char name[TSDB_DB_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, dbLen); + dbLen = strdequote(name); - code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE); + code = tNameSetDbName(pName, pParseCtx->acctId, name, dbLen); + if (code != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + int32_t tbLen = pTableName->n - dbLen - 1; + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, p + 1, tbLen); + /*tbLen = */strdequote(tbname); + + code = tNameFromString(pName, tbname, T_NAME_TABLE); if (code != 0) { return buildInvalidOperationMsg(pMsgBuf, msg1); } @@ -1964,10 +1978,17 @@ int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, return buildInvalidOperationMsg(pMsgBuf, msg1); } - tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); + assert(pTableName->n < TSDB_TABLE_FNAME_LEN); char name[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(name, pTableName->z, pTableName->n); + strdequote(name); + + code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); + if (code != TSDB_CODE_SUCCESS) { + code = buildInvalidOperationMsg(pMsgBuf, msg2); + return code; + } code = tNameFromString(pName, name, T_NAME_TABLE); if (code != 0) {