From ffe87632873aaebd106845064411b3f402697a3b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 12 Jan 2022 22:17:40 +0800 Subject: [PATCH 001/103] rename files --- source/libs/tfs/inc/{tfsint.h => tfsint1.h} | 3 + source/libs/tfs/src/{tdisk.c => tfsDisk.c} | 19 ++-- source/libs/tfs/src/{ttier.c => tfsTier.c} | 0 src/inc/tfs.h | 103 -------------------- 4 files changed, 13 insertions(+), 112 deletions(-) rename source/libs/tfs/inc/{tfsint.h => tfsint1.h} (98%) rename source/libs/tfs/src/{tdisk.c => tfsDisk.c} (79%) rename source/libs/tfs/src/{ttier.c => tfsTier.c} (100%) delete mode 100644 src/inc/tfs.h diff --git a/source/libs/tfs/inc/tfsint.h b/source/libs/tfs/inc/tfsint1.h similarity index 98% rename from source/libs/tfs/inc/tfsint.h rename to source/libs/tfs/inc/tfsint1.h index 3c5dccc63b..caa52b5cd6 100644 --- a/source/libs/tfs/inc/tfsint.h +++ b/source/libs/tfs/inc/tfsint1.h @@ -16,6 +16,9 @@ #ifndef TD_TFSINT_H #define TD_TFSINT_H +#include "os.h" + +#include "taoserror.h" #include "tlog.h" #include "tglobal.h" #include "tfs.h" diff --git a/source/libs/tfs/src/tdisk.c b/source/libs/tfs/src/tfsDisk.c similarity index 79% rename from source/libs/tfs/src/tdisk.c rename to source/libs/tfs/src/tfsDisk.c index 22601e48c3..f32069837d 100644 --- a/source/libs/tfs/src/tdisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -12,14 +12,12 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "taoserror.h" +#define _DEFAULT_SOURCE #include "tfsint.h" -// PROTECTED ==================================== -SDisk *tfsNewDisk(int level, int id, const char *dir) { - SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk)); +SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir) { + SDisk *pDisk = calloc(1, sizeof(SDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_FS_OUT_OF_MEMORY; return NULL; @@ -33,18 +31,21 @@ SDisk *tfsNewDisk(int level, int id, const char *dir) { } SDisk *tfsFreeDisk(SDisk *pDisk) { - if (pDisk) { + if (pDisk != NULL) { free(pDisk); } return NULL; } -int tfsUpdateDiskInfo(SDisk *pDisk) { - ASSERT(pDisk != NULL); +int32_t tfsUpdateDiskInfo(SDisk *pDisk) { + if (pDisk == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } SysDiskSize diskSize = {0}; - int code = taosGetDiskSize(pDisk->dir, &diskSize); + int32_t code = taosGetDiskSize(pDisk->dir, &diskSize); if (code != 0) { fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir, strerror(errno)); diff --git a/source/libs/tfs/src/ttier.c b/source/libs/tfs/src/tfsTier.c similarity index 100% rename from source/libs/tfs/src/ttier.c rename to source/libs/tfs/src/tfsTier.c diff --git a/src/inc/tfs.h b/src/inc/tfs.h deleted file mode 100644 index 9ad7a8f66e..0000000000 --- a/src/inc/tfs.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TD_TFS_H -#define TD_TFS_H - -#include "tglobal.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int level; - int id; -} SDiskID; - -#define TFS_UNDECIDED_LEVEL -1 -#define TFS_UNDECIDED_ID -1 -#define TFS_PRIMARY_LEVEL 0 -#define TFS_PRIMARY_ID 0 -#define TFS_MIN_LEVEL 0 -#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1) - -// FS APIs ==================================== -typedef struct { - int64_t tsize; - int64_t used; - int64_t avail; -} SFSMeta; - -typedef struct { - int64_t size; - int64_t used; - int64_t free; - int16_t nAvailDisks; // # of Available disks -} STierMeta; - -int tfsInit(SDiskCfg *pDiskCfg, int ndisk); -void tfsCleanup(); -void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); -void tfsGetMeta(SFSMeta *pMeta); -void tfsAllocDisk(int expLevel, int *level, int *id); - -const char *TFS_PRIMARY_PATH(); -const char *TFS_DISK_PATH(int level, int id); - -// TFILE APIs ==================================== -typedef struct { - int level; - int id; - char rname[TSDB_FILENAME_LEN]; // REL name - char aname[TSDB_FILENAME_LEN]; // ABS name -} TFILE; - -#define TFILE_LEVEL(pf) ((pf)->level) -#define TFILE_ID(pf) ((pf)->id) -#define TFILE_NAME(pf) ((pf)->aname) -#define TFILE_REL_NAME(pf) ((pf)->rname) - -#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags) -#define tfsclose(fd) close(fd) -#define tfsremove(pf) remove(TFILE_NAME(pf)) -#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df)) -#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) - -void tfsInitFile(TFILE *pf, int level, int id, const char *bname); -bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); -int tfsEncodeFile(void **buf, TFILE *pf); -void *tfsDecodeFile(void *buf, TFILE *pf); -void tfsbasename(const TFILE *pf, char *dest); -void tfsdirname(const TFILE *pf, char *dest); - -// DIR APIs ==================================== -int tfsMkdirAt(const char *rname, int level, int id); -int tfsMkdirRecurAt(const char *rname, int level, int id); -int tfsMkdir(const char *rname); -int tfsRmdir(const char *rname); -int tfsRename(char *orname, char *nrname); - -typedef struct TDIR TDIR; - -TDIR * tfsOpendir(const char *rname); -const TFILE *tfsReaddir(TDIR *tdir); -void tfsClosedir(TDIR *tdir); - -#ifdef __cplusplus -} -#endif - -#endif From ba42f8cd93fdacfe900c1c44bce2ac9011ac797e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 12 Jan 2022 22:20:29 +0800 Subject: [PATCH 002/103] rename files --- source/libs/tfs/inc/{tfsint1.h => tfsInt.h} | 12 +++++------- source/libs/tfs/src/tfs.c | 9 ++------- source/libs/tfs/src/tfsDisk.c | 2 +- source/libs/tfs/src/tfsTier.c | 6 ++---- 4 files changed, 10 insertions(+), 19 deletions(-) rename source/libs/tfs/inc/{tfsint1.h => tfsInt.h} (98%) diff --git a/source/libs/tfs/inc/tfsint1.h b/source/libs/tfs/inc/tfsInt.h similarity index 98% rename from source/libs/tfs/inc/tfsint1.h rename to source/libs/tfs/inc/tfsInt.h index caa52b5cd6..40eb61ba07 100644 --- a/source/libs/tfs/inc/tfsint1.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -18,15 +18,13 @@ #include "os.h" +#include "taosdef.h" #include "taoserror.h" -#include "tlog.h" -#include "tglobal.h" -#include "tfs.h" #include "tcoding.h" - -#ifdef __cplusplus -extern "C" { -#endif +#include "tfs.h" +#include "tglobal.h" +#include "thash.h" +#include "tlog.h" extern int fsDebugFlag; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 88d6d587a7..a054981961 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -13,13 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" - -#include "taosdef.h" -#include "taoserror.h" -#include "tfs.h" -#include "tfsint.h" -#include "thash.h" +#define _DEFAULT_SOURCE +#include "tfsInt.h" #define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index f32069837d..98656c84e6 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "tfsint.h" +#include "tfsInt.h" SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir) { SDisk *pDisk = calloc(1, sizeof(SDisk)); diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 3b19797acf..90057e61d5 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -12,11 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tfsint.h" +#define _DEFAULT_SOURCE +#include "tfsInt.h" #define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock)) #define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock)) From 9e4141fa6d517fe32b6398015799d05f2e932f62 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 12 Jan 2022 22:57:38 +0800 Subject: [PATCH 003/103] add error codes --- include/util/taoserror.h | 2 +- source/libs/tfs/inc/tfsInt.h | 27 ++++++------- source/libs/tfs/src/tfsDisk.c | 13 +++--- source/libs/tfs/src/tfsTier.c | 75 ++++++++++++++++++++--------------- source/util/src/terror.c | 2 +- 5 files changed, 64 insertions(+), 55 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 854d16f67d..4f424111a8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -406,7 +406,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) //"WAL out of memory") // tfs -#define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") +#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config") #define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) //"tfs too many mount") #define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) //"tfs duplicate primary mount") diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index 40eb61ba07..4e53fcdb8e 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -26,7 +26,7 @@ #include "thash.h" #include "tlog.h" -extern int fsDebugFlag; +extern int32_t fsDebugFlag; // For debug purpose #define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }} @@ -47,8 +47,8 @@ typedef struct { } SDiskMeta; typedef struct SDisk { - int level; - int id; + int32_t level; + int32_t id; char dir[TSDB_FILENAME_LEN]; SDiskMeta dmeta; } SDisk; @@ -61,19 +61,19 @@ typedef struct SDisk { #define DISK_USED_SIZE(pd) ((pd)->dmeta.used) #define DISK_FREE_SIZE(pd) ((pd)->dmeta.free) -SDisk *tfsNewDisk(int level, int id, const char *dir); -SDisk *tfsFreeDisk(SDisk *pDisk); -int tfsUpdateDiskInfo(SDisk *pDisk); +SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); +SDisk *tfsFreeDisk(SDisk *pDisk); +int32_t tfsUpdateDiskInfo(SDisk *pDisk); // ttier.c ====================================================== typedef struct STier { pthread_spinlock_t lock; - int level; + int32_t level; int16_t ndisk; // # of disks mounted to this tier int16_t nextid; // next disk id to allocate STierMeta tmeta; - SDisk * disks[TSDB_MAX_DISKS_PER_TIER]; + SDisk *disks[TSDB_MAX_DISKS_PER_TIER]; } STier; #define TIER_LEVEL(pt) ((pt)->level) @@ -83,12 +83,11 @@ typedef struct STier { #define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks) #define DISK_AT_TIER(pt, id) ((pt)->disks[id]) -int tfsInitTier(STier *pTier, int level); -void tfsDestroyTier(STier *pTier); -SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); -void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta); -int tfsAllocDiskOnTier(STier *pTier); -void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta); +int32_t tfsInitTier(STier *pTier, int32_t level); +void tfsDestroyTier(STier *pTier); +SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); +void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta); +int32_t tfsAllocDiskOnTier(STier *pTier); void tfsPosNextId(STier *pTier); #ifdef __cplusplus diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 98656c84e6..dd4f349c42 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -19,7 +19,7 @@ SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir) { SDisk *pDisk = calloc(1, sizeof(SDisk)); if (pDisk == NULL) { - terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -34,6 +34,7 @@ SDisk *tfsFreeDisk(SDisk *pDisk) { if (pDisk != NULL) { free(pDisk); } + return NULL; } @@ -44,17 +45,15 @@ int32_t tfsUpdateDiskInfo(SDisk *pDisk) { } SysDiskSize diskSize = {0}; - - int32_t code = taosGetDiskSize(pDisk->dir, &diskSize); - if (code != 0) { + if (taosGetDiskSize(pDisk->dir, &diskSize) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + return -1 } pDisk->dmeta.size = diskSize.tsize; pDisk->dmeta.used = diskSize.used; pDisk->dmeta.free = diskSize.avail; - - return code; + return 0; } diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 90057e61d5..8b057d8755 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -16,45 +16,50 @@ #define _DEFAULT_SOURCE #include "tfsInt.h" -#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock)) -#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock)) +#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) +#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) -// PROTECTED ========================================== -int tfsInitTier(STier *pTier, int level) { - memset((void *)pTier, 0, sizeof(*pTier)); +int32_t tfsInitTier(STier *pTier, int32_t level) { + if (pTier == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } - int code = pthread_spin_init(&(pTier->lock), 0); - if (code) { + memset(pTier, 0, sizeof(STier)); + + int32_t code = pthread_spin_init(&pTier->lock, 0); + if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; } pTier->level = level; - return 0; } void tfsDestroyTier(STier *pTier) { - for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { + if (pTier == NULL) return; + + for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id)); } pTier->ndisk = 0; - pthread_spin_destroy(&(pTier->lock)); } SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { - ASSERT(pTier->level == pCfg->level); - - int id = 0; - SDisk *pDisk; + if (pTier == NULL || pCfg == NULL || pTier->level != pCfg->level) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) { terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; return NULL; } + int32_t id = 0; if (pTier->level == 0) { if (DISK_AT_TIER(pTier, 0) != NULL) { id = pTier->ndisk; @@ -73,30 +78,31 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { id = pTier->ndisk; } - pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); + SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); if (pDisk == NULL) return NULL; + DISK_AT_TIER(pTier, id) = pDisk; pTier->ndisk++; fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); - return DISK_AT_TIER(pTier, id); } void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) { - STierMeta tmeta; + STierMeta tmeta = {0}; if (pTierMeta == NULL) { pTierMeta = &tmeta; } - memset(pTierMeta, 0, sizeof(*pTierMeta)); + memset(pTierMeta, 0, sizeof(STierMeta)); tfsLockTier(pTier); - for (int id = 0; id < pTier->ndisk; id++) { - if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) { + for (int32_t id = 0; id < pTier->ndisk; id++) { + if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) != 0) { continue; } + pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id)); pTierMeta->used += DISK_USED_SIZE(DISK_AT_TIER(pTier, id)); pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id)); @@ -109,22 +115,26 @@ void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) { } // Round-Robin to allocate disk on a tier -int tfsAllocDiskOnTier(STier *pTier) { - ASSERT(pTier->ndisk > 0); - int id = TFS_UNDECIDED_ID; - SDisk *pDisk; +int32_t tfsAllocDiskOnTier(STier *pTier) { + if (pTier == NULL || pTier->ndisk <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } tfsLockTier(pTier); if (TIER_AVAIL_DISKS(pTier) <= 0) { tfsUnLockTier(pTier); - return id; + return TFS_UNDECIDED_ID; } - id = pTier->nextid; + int32_t id = pTier->nextid; while (true) { - pDisk = DISK_AT_TIER(pTier, id); - ASSERT(pDisk != NULL); + SDisk *pDisk = DISK_AT_TIER(pTier, id); + if (pDisk == NULL) { + tfsUnLockTier(pTier); + return TFS_UNDECIDED_ID; + } if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) { id = (id + 1) % pTier->ndisk; @@ -145,7 +155,7 @@ int tfsAllocDiskOnTier(STier *pTier) { } void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) { - ASSERT(pTierMeta != NULL); + if (pTierMeta == NULL || pTierMeta == NULL) return; tfsLockTier(pTier); *pTierMeta = pTier->tmeta; @@ -153,10 +163,11 @@ void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) { } void tfsPosNextId(STier *pTier) { - ASSERT(pTier->ndisk > 0); - int nextid = 0; + if (pTier == NULL || pTier->ndisk <= 0) return; - for (int id = 1; id < pTier->ndisk; id++) { + int32_t nextid = 0; + + for (int32_t id = 1; id < pTier->ndisk; id++) { SDisk *pLDisk = DISK_AT_TIER(pTier, nextid); SDisk *pDisk = DISK_AT_TIER(pTier, id); if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8294bca959..6d0547c7ae 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -387,7 +387,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") // tfs -TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, "tfs out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, "tfs too many mount") TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, "tfs duplicate primary mount") From 7087e4427b90e5d51c42a42f18bff378b0517187 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 12 Jan 2022 23:15:12 +0800 Subject: [PATCH 004/103] minor changes --- include/libs/tfs/tfs.h | 54 ++++++++++---------- source/libs/tfs/inc/tfsInt.h | 8 +-- source/libs/tfs/src/tfs.c | 98 ++++++++++++++++++------------------ 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 793c861363..973231c264 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TD_TFS_H -#define TD_TFS_H +#ifndef _TD_TFS_H_ +#define _TD_TFS_H_ #include "tglobal.h" @@ -23,8 +23,8 @@ extern "C" { #endif typedef struct { - int level; - int id; + int32_t level; + int32_t id; } SDiskID; #define TFS_UNDECIDED_LEVEL -1 @@ -48,21 +48,21 @@ typedef struct { int16_t nAvailDisks; // # of Available disks } STierMeta; -int tfsInit(SDiskCfg *pDiskCfg, int ndisk); -void tfsCleanup(); -void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); -void tfsGetMeta(SFSMeta *pMeta); -void tfsAllocDisk(int expLevel, int *level, int *id); +int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk); +void tfsCleanup(); +void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); +void tfsGetMeta(SFSMeta *pMeta); +void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id); const char *TFS_PRIMARY_PATH(); -const char *TFS_DISK_PATH(int level, int id); +const char *TFS_DISK_PATH(int32_t level, int32_t id); // TFILE APIs ==================================== typedef struct { - int level; - int id; - char rname[TSDB_FILENAME_LEN]; // REL name - char aname[TSDB_FILENAME_LEN]; // ABS name + int32_t level; + int32_t id; + char rname[TSDB_FILENAME_LEN]; // REL name + char aname[TSDB_FILENAME_LEN]; // ABS name } TFILE; #define TFILE_LEVEL(pf) ((pf)->level) @@ -76,23 +76,23 @@ typedef struct { #define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df)) #define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) -void tfsInitFile(TFILE *pf, int level, int id, const char *bname); -bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); -int tfsEncodeFile(void **buf, TFILE *pf); -void *tfsDecodeFile(void *buf, TFILE *pf); -void tfsbasename(const TFILE *pf, char *dest); -void tfsdirname(const TFILE *pf, char *dest); +void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname); +bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); +int32_t tfsEncodeFile(void **buf, TFILE *pf); +void *tfsDecodeFile(void *buf, TFILE *pf); +void tfsbasename(const TFILE *pf, char *dest); +void tfsdirname(const TFILE *pf, char *dest); // DIR APIs ==================================== -int tfsMkdirAt(const char *rname, int level, int id); -int tfsMkdirRecurAt(const char *rname, int level, int id); -int tfsMkdir(const char *rname); -int tfsRmdir(const char *rname); -int tfsRename(char *orname, char *nrname); +int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id); +int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id); +int32_t tfsMkdir(const char *rname); +int32_t tfsRmdir(const char *rname); +int32_t tfsRename(char *orname, char *nrname); typedef struct TDIR TDIR; -TDIR * tfsOpendir(const char *rname); +TDIR *tfsOpendir(const char *rname); const TFILE *tfsReaddir(TDIR *tdir); void tfsClosedir(TDIR *tdir); @@ -100,4 +100,4 @@ void tfsClosedir(TDIR *tdir); } #endif -#endif +#endif /*_TD_TFS_H_*/ diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index 4e53fcdb8e..bfa4c4380d 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TD_TFSINT_H -#define TD_TFSINT_H +#ifndef _TD_TFSINT_H_ +#define _TD_TFSINT_H_ #include "os.h" @@ -88,10 +88,10 @@ void tfsDestroyTier(STier *pTier); SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta); int32_t tfsAllocDiskOnTier(STier *pTier); -void tfsPosNextId(STier *pTier); +void tfsPosNextId(STier *pTier); #ifdef __cplusplus } #endif -#endif +#endif /*_TD_TFSINT_H_*/ diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index a054981961..bbeb66fe05 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -21,9 +21,9 @@ typedef struct { pthread_spinlock_t lock; SFSMeta meta; - int nlevel; + int32_t nlevel; STier tiers[TSDB_MAX_TIERS]; - SHashObj * map; // name to did map + SHashObj *map; // name to did map } SFS; typedef struct { @@ -47,21 +47,21 @@ static SFS tfs = {0}; static SFS *pfs = &tfs; // STATIC DECLARATION -static int tfsMount(SDiskCfg *pCfg); -static int tfsCheck(); -static int tfsCheckAndFormatCfg(SDiskCfg *pCfg); -static int tfsFormatDir(char *idir, char *odir); -static SDisk *tfsGetDiskByID(SDiskID did); -static SDisk *tfsGetDiskByName(const char *dir); -static int tfsOpendirImpl(TDIR *tdir); -static void tfsInitDiskIter(SDiskIter *pIter); -static SDisk *tfsNextDisk(SDiskIter *pIter); +static int32_t tfsMount(SDiskCfg *pCfg); +static int32_t tfsCheck(); +static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg); +static int32_t tfsFormatDir(char *idir, char *odir); +static SDisk *tfsGetDiskByID(SDiskID did); +static SDisk *tfsGetDiskByName(const char *dir); +static int32_t tfsOpendirImpl(TDIR *tdir); +static void tfsInitDiskIter(SDiskIter *pIter); +static SDisk *tfsNextDisk(SDiskIter *pIter); // FS APIs ==================================== -int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { +int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) { ASSERT(ndisk > 0); - for (int level = 0; level < TSDB_MAX_TIERS; level++) { + for (int32_t level = 0; level < TSDB_MAX_TIERS; level++) { if (tfsInitTier(TFS_TIER_AT(level), level) < 0) { while (true) { level--; @@ -84,7 +84,7 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { return -1; } - for (int idisk = 0; idisk < ndisk; idisk++) { + for (int32_t idisk = 0; idisk < ndisk; idisk++) { if (tfsMount(pDiskCfg + idisk) < 0) { tfsCleanup(); return -1; @@ -97,7 +97,7 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { } tfsUpdateInfo(NULL, NULL, 0); - for (int level = 0; level < TFS_NLEVEL(); level++) { + for (int32_t level = 0; level < TFS_NLEVEL(); level++) { tfsPosNextId(TFS_TIER_AT(level)); } @@ -109,7 +109,7 @@ void tfsCleanup() { pfs->map = NULL; pthread_spin_destroy(&(pfs->lock)); - for (int level = 0; level < TFS_NLEVEL(); level++) { + for (int32_t level = 0; level < TFS_NLEVEL(); level++) { tfsDestroyTier(TFS_TIER_AT(level)); } } @@ -124,7 +124,7 @@ void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) { memset(pFSMeta, 0, sizeof(*pFSMeta)); - for (int level = 0; level < TFS_NLEVEL(); level++) { + for (int32_t level = 0; level < TFS_NLEVEL(); level++) { STierMeta *pTierMeta = &tierMeta; if (tierMetas && level < numTiers) { pTierMeta = tierMetas + level; @@ -152,7 +152,7 @@ void tfsGetMeta(SFSMeta *pMeta) { /* Allocate an existing available tier level */ -void tfsAllocDisk(int expLevel, int *level, int *id) { +void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id) { ASSERT(expLevel >= 0); *level = expLevel; @@ -177,10 +177,10 @@ void tfsAllocDisk(int expLevel, int *level, int *id) { } const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); } -const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); } +const char *TFS_DISK_PATH(int32_t level, int32_t id) { return DISK_DIR(TFS_DISK_AT(level, id)); } // TFILE APIs ==================================== -void tfsInitFile(TFILE *pf, int level, int id, const char *bname) { +void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname) { ASSERT(TFS_IS_VALID_DISK(level, id)); SDisk *pDisk = TFS_DISK_AT(level, id); @@ -203,8 +203,8 @@ bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) { return true; } -int tfsEncodeFile(void **buf, TFILE *pf) { - int tlen = 0; +int32_t tfsEncodeFile(void **buf, TFILE *pf) { + int32_t tlen = 0; tlen += taosEncodeVariantI32(buf, pf->level); tlen += taosEncodeVariantI32(buf, pf->id); @@ -215,7 +215,7 @@ int tfsEncodeFile(void **buf, TFILE *pf) { void *tfsDecodeFile(void *buf, TFILE *pf) { int32_t level, id; - char * rname; + char *rname; buf = taosDecodeVariantI32(buf, &(level)); buf = taosDecodeVariantI32(buf, &(id)); @@ -242,7 +242,7 @@ void tfsdirname(const TFILE *pf, char *dest) { } // DIR APIs ==================================== -int tfsMkdirAt(const char *rname, int level, int id) { +int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) { SDisk *pDisk = TFS_DISK_AT(level, id); char aname[TMPNAME_LEN]; @@ -255,7 +255,7 @@ int tfsMkdirAt(const char *rname, int level, int id) { return 0; } -int tfsMkdirRecurAt(const char *rname, int level, int id) { +int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { if (tfsMkdirAt(rname, level, id) < 0) { if (errno == ENOENT) { // Try to create upper @@ -288,10 +288,10 @@ int tfsMkdirRecurAt(const char *rname, int level, int id) { return 0; } -int tfsMkdir(const char *rname) { - for (int level = 0; level < TFS_NLEVEL(); level++) { +int32_t tfsMkdir(const char *rname) { + for (int32_t level = 0; level < TFS_NLEVEL(); level++) { STier *pTier = TFS_TIER_AT(level); - for (int id = 0; id < TIER_NDISKS(pTier); id++) { + for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { if (tfsMkdirAt(rname, level, id) < 0) { return -1; } @@ -301,12 +301,12 @@ int tfsMkdir(const char *rname) { return 0; } -int tfsRmdir(const char *rname) { +int32_t tfsRmdir(const char *rname) { char aname[TMPNAME_LEN] = "\0"; - for (int level = 0; level < TFS_NLEVEL(); level++) { + for (int32_t level = 0; level < TFS_NLEVEL(); level++) { STier *pTier = TFS_TIER_AT(level); - for (int id = 0; id < TIER_NDISKS(pTier); id++) { + for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { SDisk *pDisk = DISK_AT_TIER(pTier, id); snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); @@ -318,13 +318,13 @@ int tfsRmdir(const char *rname) { return 0; } -int tfsRename(char *orname, char *nrname) { +int32_t tfsRename(char *orname, char *nrname) { char oaname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0"; - for (int level = 0; level < pfs->nlevel; level++) { + for (int32_t level = 0; level < pfs->nlevel; level++) { STier *pTier = TFS_TIER_AT(level); - for (int id = 0; id < TIER_NDISKS(pTier); id++) { + for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { SDisk *pDisk = DISK_AT_TIER(pTier, id); snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname); @@ -339,11 +339,11 @@ int tfsRename(char *orname, char *nrname) { struct TDIR { SDiskIter iter; - int level; - int id; + int32_t level; + int32_t id; char dirname[TSDB_FILENAME_LEN]; TFILE tfile; - DIR * dir; + DIR *dir; }; TDIR *tfsOpendir(const char *rname) { @@ -402,9 +402,9 @@ void tfsClosedir(TDIR *tdir) { } // private -static int tfsMount(SDiskCfg *pCfg) { +static int32_t tfsMount(SDiskCfg *pCfg) { SDiskID did; - SDisk * pDisk = NULL; + SDisk *pDisk = NULL; if (tfsCheckAndFormatCfg(pCfg) < 0) return -1; @@ -422,7 +422,7 @@ static int tfsMount(SDiskCfg *pCfg) { return 0; } -static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) { +static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { char dirName[TSDB_FILENAME_LEN] = "\0"; struct stat pstat; @@ -481,10 +481,10 @@ static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) { return 0; } -static int tfsFormatDir(char *idir, char *odir) { +static int32_t tfsFormatDir(char *idir, char *odir) { wordexp_t wep = {0}; - int code = wordexp(idir, &wep, 0); + int32_t code = wordexp(idir, &wep, 0); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); return -1; @@ -502,14 +502,14 @@ static int tfsFormatDir(char *idir, char *odir) { return 0; } -static int tfsCheck() { +static int32_t tfsCheck() { if (TFS_PRIMARY_DISK() == NULL) { fError("no primary disk is set"); terrno = TSDB_CODE_FS_NO_PRIMARY_DISK; return -1; } - for (int level = 0; level < TFS_NLEVEL(); level++) { + for (int32_t level = 0; level < TFS_NLEVEL(); level++) { if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) { fError("no disk at level %d", level); terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; @@ -523,8 +523,8 @@ static int tfsCheck() { static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); } static SDisk *tfsGetDiskByName(const char *dir) { SDiskID did; - SDisk * pDisk = NULL; - void * pr = NULL; + SDisk *pDisk = NULL; + void *pr = NULL; pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN)); if (pr == NULL) return NULL; @@ -536,7 +536,7 @@ static SDisk *tfsGetDiskByName(const char *dir) { return pDisk; } -static int tfsOpendirImpl(TDIR *tdir) { +static int32_t tfsOpendirImpl(TDIR *tdir) { SDisk *pDisk = NULL; char adir[TMPNAME_LEN * 2] = "\0"; @@ -567,8 +567,8 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) { if (pDisk == NULL) return NULL; - int level = DISK_LEVEL(pDisk); - int id = DISK_ID(pDisk); + int32_t level = DISK_LEVEL(pDisk); + int32_t id = DISK_ID(pDisk); id++; if (id < TIER_NDISKS(TFS_TIER_AT(level))) { From 31ce27054bde733da1b3e4c244637545b02191ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 17 Jan 2022 05:30:05 +0000 Subject: [PATCH 005/103] more --- .devcontainer/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fc9a51af65..01172ae9c9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,3 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} # [Optional] Uncomment this section to install additional packages. # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends +RUN apt-get update && apt-get -y install tree vim From 0c5a85d463d011295bc1486e62d2537f618c2893 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Jan 2022 13:58:43 +0800 Subject: [PATCH 006/103] minor changes --- source/libs/tfs/src/tfs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index bbeb66fe05..87adba6bea 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -79,7 +79,7 @@ int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) { pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (pfs->map == NULL) { - terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tfsCleanup(); return -1; } @@ -318,6 +318,7 @@ int32_t tfsRmdir(const char *rname) { return 0; } +#if 0 int32_t tfsRename(char *orname, char *nrname) { char oaname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0"; @@ -336,6 +337,7 @@ int32_t tfsRename(char *orname, char *nrname) { return 0; } +#endif struct TDIR { SDiskIter iter; From d8a815e1e784fd0f123f0521aa3d952c1991ed3e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Jan 2022 00:54:16 -0800 Subject: [PATCH 007/103] add stb test --- source/dnode/mgmt/impl/test/CMakeLists.txt | 1 - .../dnode/mgmt/impl/test/stb/CMakeLists.txt | 11 - source/dnode/mgmt/impl/test/stb/dstb.cpp | 139 --------- source/dnode/mgmt/impl/test/vnode/vnode.cpp | 279 ++++++++++++------ 4 files changed, 190 insertions(+), 240 deletions(-) delete mode 100644 source/dnode/mgmt/impl/test/stb/CMakeLists.txt delete mode 100644 source/dnode/mgmt/impl/test/stb/dstb.cpp diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 4d6ae91def..ce93a14d3f 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -5,5 +5,4 @@ add_subdirectory(bnode) add_subdirectory(snode) add_subdirectory(mnode) add_subdirectory(vnode) -add_subdirectory(stb) add_subdirectory(sut) diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt deleted file mode 100644 index 912f892c29..0000000000 --- a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. DSTB_SRC) -add_executable(dnode_test_stb ${DSTB_SRC}) -target_link_libraries( - dnode_test_stb - PUBLIC sut -) - -add_test( - NAME dnode_test_stb - COMMAND dnode_test_stb -) diff --git a/source/dnode/mgmt/impl/test/stb/dstb.cpp b/source/dnode/mgmt/impl/test/stb/dstb.cpp deleted file mode 100644 index 82f5761ffe..0000000000 --- a/source/dnode/mgmt/impl/test/stb/dstb.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/** - * @file db.cpp - * @author slguan (slguan@taosdata.com) - * @brief DNODE module vnode tests - * @version 0.1 - * @date 2021-12-20 - * - * @copyright Copyright (c) 2021 - * - */ - -#include "sut.h" - -class DndTestVnode : public ::testing::Test { - protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_stb", 9116); } - static void TearDownTestSuite() { test.Cleanup(); } - - static Testbase test; - - public: - void SetUp() override {} - void TearDown() override {} -}; - -Testbase DndTestVnode::test; - -TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) { - { - for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateVnodeReq); - - SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(1); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); - } - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); - } - } - } - - { - for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SAlterVnodeReq); - - SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); - } - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - } - - { - for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SDropVnodeReq); - - SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropVnodeReq); - rpcMsg.msgType = TDMT_DND_DROP_VNODE; - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_NOT_DEPLOYED); - } - } - } -} diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 6b042c3ce6..1799bf66b0 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -25,115 +25,216 @@ class DndTestVnode : public ::testing::Test { Testbase DndTestVnode::test; -TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) { - { - for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateVnodeReq); +TEST_F(DndTestVnode, 01_Create_Vnode) { + for (int i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SCreateVnodeReq); - SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(1); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); - } + SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(1); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9527); + } - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); - } + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (i == 0) { + ASSERT_EQ(pRsp->code, 0); + test.Restart(); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); } } +} +TEST_F(DndTestVnode, 02_ALTER_Vnode) { + for (int i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SAlterVnodeReq); + + SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9527); + } + + SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } +} + +TEST_F(DndTestVnode, 03_Create_Stb) { + for (int i = 0; i < 1; ++i) { + SVCreateTbReq req = {0}; + req.ver = 0; + req.name = (char*)"stb1"; + req.ttl = 0; + req.keep = 0; + req.type = TD_SUPER_TABLE; + + SSchema schemas[5] = {0}; + { + SSchema* pSchema = &schemas[0]; + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema->name, "ts"); + } + + { + SSchema* pSchema = &schemas[1]; + pSchema->bytes = htonl(4); + pSchema->type = TSDB_DATA_TYPE_INT; + strcpy(pSchema->name, "col1"); + } + + { + SSchema* pSchema = &schemas[2]; + pSchema->bytes = htonl(2); + pSchema->type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema->name, "tag1"); + } + + { + SSchema* pSchema = &schemas[3]; + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema->name, "tag2"); + } + + { + SSchema* pSchema = &schemas[4]; + pSchema->bytes = htonl(16); + pSchema->type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema->name, "tag3"); + } + + req.stbCfg.suid = 9527; + req.stbCfg.nCols = 2; + req.stbCfg.pSchema = &schemas[0]; + req.stbCfg.nTagCols = 3; + req.stbCfg.pTagSchema = &schemas[2]; + + int32_t bsize = tSerializeSVCreateTbReq(NULL, &req); + void* buf = rpcMallocCont(sizeof(SMsgHead) + bsize); + SMsgHead* pMsgHead = (SMsgHead*)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); + pMsgHead->vgId = htonl(2); + + void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tSerializeSVCreateTbReq(&pBuf, &req); + + int32_t contLen = sizeof(SMsgHead) + bsize; + + SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, buf, contLen); + ASSERT_NE(pRsp, nullptr); + if (i == 0) { + ASSERT_EQ(pRsp->code, 0); + test.Restart(); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_TABLE_ALREADY_EXIST); + } + } +} + +TEST_F(DndTestVnode, 04_ALTER_Stb) { +#if 0 { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SAlterVnodeReq); - - SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); - } - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_ALTER_STB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } } +#endif +} +TEST_F(DndTestVnode, 05_DROP_Stb) { +#if 0 { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SDropVnodeReq); - - SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropVnodeReq); - rpcMsg.msgType = TDMT_DND_DROP_VNODE; - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, pReq, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { ASSERT_EQ(pRsp->code, 0); test.Restart(); } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); } } } +#endif } + +TEST_F(DndTestVnode, 06_DROP_Vnode) { + for (int i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SDropVnodeReq); + + SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropVnodeReq); + rpcMsg.msgType = TDMT_DND_DROP_VNODE; + + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (i == 0) { + ASSERT_EQ(pRsp->code, 0); + test.Restart(); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_NOT_DEPLOYED); + } + } +} \ No newline at end of file From feef8c395dca6cbb53d396304121d7ff2e6f7018 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 02:37:20 +0000 Subject: [PATCH 008/103] more --- source/libs/tdb/inc/tdb.h | 2 +- source/libs/tdb/src/db/tdbDB.c | 25 ++++++++++++++++++++++--- source/libs/tdb/src/inc/tdbDB.h | 11 +++++++++++ source/libs/tdb/test/tdbTest.cpp | 6 +++--- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 905b08ee0b..eee8f8ed33 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -44,7 +44,7 @@ typedef struct { // TDB Operations TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type); -TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags); +TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); #ifdef __cplusplus diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c index 2af40d8642..eaf85ea4a1 100644 --- a/source/libs/tdb/src/db/tdbDB.c +++ b/source/libs/tdb/src/db/tdbDB.c @@ -56,9 +56,28 @@ _err: return 0; } -TDB_EXTERN int tdbOpenDB(TDB* dbp, uint32_t flags) { - // TODO - return 0; +TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) { + int ret = 0; + + if ((dbp->fname = strdup(fname)) == NULL) { + ret = -1; + return ret; + } + + // Create the backup file if the file not exists + + // Open the file as a sub-db or a master-db + if (dbname) { + if ((dbp->dbname = strdup(dbname)) == NULL) { + ret = -1; + return ret; + } + // TODO: Open the DB as a SUB-DB in this file + } else { + // TODO: Open the DB as a MASTER-DB in this file + } + + return ret; } TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index fca197dc39..d0ef9e22d0 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -25,6 +25,14 @@ extern "C" { #endif +typedef struct { + // TODO +} TDB_MPOOL; + +typedef struct { + int fd; +} TDB_FH; + struct TDB { pgsize_t pageSize; tdb_db_t type; @@ -35,6 +43,9 @@ struct TDB { TDB_HASH * hash; TDB_HEAP * heap; } dbam; // db access method + + TDB_FH * fhp; // The backup file handle + TDB_MPOOL *mph; // The memory pool handle }; #ifdef __cplusplus diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 38c3b0b917..f27e17f1ca 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,9 +6,9 @@ TEST(tdb_api_test, tdb_create_open_close_db_test) { int ret; TDB *dbp; - tdbCreateDB(&dbp, TDB_BTREE_T); + // tdbCreateDB(&dbp, TDB_BTREE_T); - tdbOpenDB(dbp, 0); + // tdbOpenDB(dbp, 0); - tdbCloseDB(dbp, 0); + // tdbCloseDB(dbp, 0); } \ No newline at end of file From 7ee4403d845f8a527a95df5c1ee3d74cd7874c4f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Jan 2022 11:21:13 +0800 Subject: [PATCH 009/103] [td-11818] Return result to query worker --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 319 ++++++----- source/libs/executor/inc/executorimpl.h | 4 +- source/libs/executor/src/executorMain.c | 45 +- source/libs/executor/src/executorimpl.c | 676 ++++++++++++------------ source/libs/qworker/src/qworker.c | 2 +- 6 files changed, 511 insertions(+), 537 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 71b014d025..e4f8d291eb 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -44,7 +44,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskI * @param handle * @return */ -int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle); +struct SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle); /** * Retrieve the produced results information, if current query is not paused or completed, diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bf80490416..d07a5ffc77 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3454,6 +3454,7 @@ void filterPrepare(void* expr, void* param) { } } + static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = ((STableKeyInfo*) p1)->pTable; @@ -3537,8 +3538,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); -// assert((*p)->type == TSDB_CHILD_TABLE); - if (ret == 0) { STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; taosArrayPush(g, &info1); @@ -3554,7 +3553,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } -#if 0 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3587,145 +3585,138 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC sup.pTagSchema = pTagSchema; sup.pCols = pCols; - taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); +// taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); +// createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; } -static bool tableFilterFp(const void* pNode, void* param) { - tQueryInfo* pInfo = (tQueryInfo*) param; +//static bool tableFilterFp(const void* pNode, void* param) { +// tQueryInfo* pInfo = (tQueryInfo*) param; +// +// STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); +// +// char* val = NULL; +// if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { +// val = (char*) TABLE_NAME(pTable); +// } else { +// val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); +// } +// +// if (pInfo->optr == TSDB_RELATION_ISNULL || pInfo->optr == TSDB_RELATION_NOTNULL) { +// if (pInfo->optr == TSDB_RELATION_ISNULL) { +// return (val == NULL) || isNull(val, pInfo->sch.type); +// } else if (pInfo->optr == TSDB_RELATION_NOTNULL) { +// return (val != NULL) && (!isNull(val, pInfo->sch.type)); +// } +// } else if (pInfo->optr == TSDB_RELATION_IN) { +// int type = pInfo->sch.type; +// if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_TIMESTAMP) { +// int64_t v; +// GET_TYPED_DATA(v, int64_t, pInfo->sch.type, val); +// return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); +// } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { +// uint64_t v; +// GET_TYPED_DATA(v, uint64_t, pInfo->sch.type, val); +// return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); +// } +// else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { +// double v; +// GET_TYPED_DATA(v, double, pInfo->sch.type, val); +// return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); +// } else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR){ +// return NULL != taosHashGet((SHashObj *)pInfo->q, varDataVal(val), varDataLen(val)); +// } +// +// } +// +// int32_t ret = 0; +// if (val == NULL) { //the val is possible to be null, so check it out carefully +// ret = -1; // val is missing in table tags value pairs +// } else { +// ret = pInfo->compare(val, pInfo->q); +// } +// +// switch (pInfo->optr) { +// case TSDB_RELATION_EQUAL: { +// return ret == 0; +// } +// case TSDB_RELATION_NOT_EQUAL: { +// return ret != 0; +// } +// case TSDB_RELATION_GREATER_EQUAL: { +// return ret >= 0; +// } +// case TSDB_RELATION_GREATER: { +// return ret > 0; +// } +// case TSDB_RELATION_LESS_EQUAL: { +// return ret <= 0; +// } +// case TSDB_RELATION_LESS: { +// return ret < 0; +// } +// case TSDB_RELATION_LIKE: { +// return ret == 0; +// } +// case TSDB_RELATION_MATCH: { +// return ret == 0; +// } +// case TSDB_RELATION_NMATCH: { +// return ret == 0; +// } +// case TSDB_RELATION_IN: { +// return ret == 1; +// } +// +// default: +// assert(false); +// } +// +// return true; +//} - STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); +//static void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); - char* val = NULL; - if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { - val = (char*) TABLE_NAME(pTable); - } else { - val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); - } - - if (pInfo->optr == TSDB_RELATION_ISNULL || pInfo->optr == TSDB_RELATION_NOTNULL) { - if (pInfo->optr == TSDB_RELATION_ISNULL) { - return (val == NULL) || isNull(val, pInfo->sch.type); - } else if (pInfo->optr == TSDB_RELATION_NOTNULL) { - return (val != NULL) && (!isNull(val, pInfo->sch.type)); - } - } else if (pInfo->optr == TSDB_RELATION_IN) { - int type = pInfo->sch.type; - if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t v; - GET_TYPED_DATA(v, int64_t, pInfo->sch.type, val); - return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - uint64_t v; - GET_TYPED_DATA(v, uint64_t, pInfo->sch.type, val); - return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); - } - else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { - double v; - GET_TYPED_DATA(v, double, pInfo->sch.type, val); - return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); - } else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR){ - return NULL != taosHashGet((SHashObj *)pInfo->q, varDataVal(val), varDataLen(val)); - } - - } - - int32_t ret = 0; - if (val == NULL) { //the val is possible to be null, so check it out carefully - ret = -1; // val is missing in table tags value pairs - } else { - ret = pInfo->compare(val, pInfo->q); - } - - switch (pInfo->optr) { - case TSDB_RELATION_EQUAL: { - return ret == 0; - } - case TSDB_RELATION_NOT_EQUAL: { - return ret != 0; - } - case TSDB_RELATION_GREATER_EQUAL: { - return ret >= 0; - } - case TSDB_RELATION_GREATER: { - return ret > 0; - } - case TSDB_RELATION_LESS_EQUAL: { - return ret <= 0; - } - case TSDB_RELATION_LESS: { - return ret < 0; - } - case TSDB_RELATION_LIKE: { - return ret == 0; - } - case TSDB_RELATION_MATCH: { - return ret == 0; - } - case TSDB_RELATION_NMATCH: { - return ret == 0; - } - case TSDB_RELATION_IN: { - return ret == 1; - } - - default: - assert(false); - } - - return true; -} - -static void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); - -static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { - // query according to the expression tree - SExprTraverseSupp supp = { - .nodeFilterFn = (__result_filter_fn_t) tableFilterFp, - .setupInfoFn = filterPrepare, - .pExtInfo = pSTable->tagSchema, - }; - - getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); - tExprTreeDestroy(pExpr, destroyHelper); - return TSDB_CODE_SUCCESS; -} +//static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { +// // query according to the expression tree +// SExprTraverseSupp supp = { +// .nodeFilterFn = (__result_filter_fn_t) tableFilterFp, +// .setupInfoFn = filterPrepare, +// .pExtInfo = pSTable->tagSchema, +// }; +// +// getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); +// tExprTreeDestroy(pExpr, destroyHelper); +// return TSDB_CODE_SUCCESS; +//} int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols) { - if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; - - STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - if (pTable == NULL) { - tsdbError("%p failed to get stable, uid:%" PRIu64, tsdb, uid); + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId) { + STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); + if (pTbCfg == NULL) { + tsdbError("%p failed to get stable, uid:%"PRIu64", reqId:0x%"PRIx64, tsdb, uid, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - tsdbUnlockRepoMeta(tsdb); - goto _error; } - if (pTable->type != TSDB_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId, - pTable->name->data); - terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client - - tsdbUnlockRepoMeta(tsdb); + if (pTbCfg->type != META_SUPER_TABLE) { + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", reId:0x%"PRIx64, tsdb, uid, reqId); + terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client goto _error; } //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); + STSchema* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - int32_t ret = getAllTableList(pTable, res); + assert(false); + int32_t ret = 0;//getAllTableList(pTable, res); if (ret != TSDB_CODE_SUCCESS) { - tsdbUnlockRepoMeta(tsdb); goto _error; } @@ -3736,60 +3727,60 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); - if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; return ret; } int32_t ret = TSDB_CODE_SUCCESS; - tExprNode* expr = NULL; - - TRY(TSDB_MAX_TAG_CONDITIONS) { - expr = exprTreeFromTableName(tbnameCond); - if (expr == NULL) { - expr = exprTreeFromBinary(pTagCond, len); - } else { - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL); - tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); - if (tagExpr != NULL) { - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); - tExprNode* tbnameExpr = expr; - expr = calloc(1, sizeof(tExprNode)); - if (expr == NULL) { - THROW( TSDB_CODE_TDB_OUT_OF_MEMORY ); - } - expr->nodeType = TSQL_NODE_EXPR; - expr->_node.optr = (uint8_t)tagNameRelType; - expr->_node.pLeft = tagExpr; - expr->_node.pRight = tbnameExpr; - } - } - CLEANUP_EXECUTE(); - - } CATCH( code ) { - CLEANUP_EXECUTE(); - terrno = code; - tsdbUnlockRepoMeta(tsdb); // unlock tsdb in any cases - - goto _error; - // TODO: more error handling - } END_TRY - - doQueryTableList(pTable, res, expr); - pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId, - pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); - - taosArrayDestroy(res); - - if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; - return ret; +// tExprNode* expr = NULL; +// +// TRY(TSDB_MAX_TAG_CONDITIONS) { +// expr = exprTreeFromTableName(tbnameCond); +// if (expr == NULL) { +// expr = exprTreeFromBinary(pTagCond, len); +// } else { +// CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL); +// tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); +// if (tagExpr != NULL) { +// CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); +// tExprNode* tbnameExpr = expr; +// expr = calloc(1, sizeof(tExprNode)); +// if (expr == NULL) { +// THROW( TSDB_CODE_TDB_OUT_OF_MEMORY ); +// } +// expr->nodeType = TSQL_NODE_EXPR; +// expr->_node.optr = (uint8_t)tagNameRelType; +// expr->_node.pLeft = tagExpr; +// expr->_node.pRight = tbnameExpr; +// } +// } +// CLEANUP_EXECUTE(); +// +// } CATCH( code ) { +// CLEANUP_EXECUTE(); +// terrno = code; +// tsdbUnlockRepoMeta(tsdb); // unlock tsdb in any cases +// +// goto _error; +// // TODO: more error handling +// } END_TRY +// +// doQueryTableList(pTable, res, expr); +// pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); +// pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); +// +// tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId, +// pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); +// +// taosArrayDestroy(res); +// +// if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; +// return ret; _error: return terrno; } +#if 0 int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 34d89a75b3..adb305ab09 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -597,7 +597,6 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); void freeParam(STaskParam *param); -int32_t convertQueryMsg(SQueryTableReq *pQueryMsg, STaskParam* param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); @@ -638,7 +637,8 @@ size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); void setQueryKilled(SQInfo *pQInfo); void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); -void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); +void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code); + void calculateOperatorProfResults(SQInfo* pQInfo); void queryCostStatis(SQInfo *pQInfo); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index daeefba253..e451f888bb 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -135,7 +135,7 @@ int waitMoment(SQInfo* pQInfo){ } #endif -int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { +SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); @@ -144,7 +144,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; - return pTaskInfo->code; + return NULL; } if (pTaskInfo->cost.start == 0) { @@ -153,7 +153,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { if (isTaskKilled(pTaskInfo)) { qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo)); - return pTaskInfo->code; + return NULL; } // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; @@ -168,8 +168,9 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; - qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); - return pTaskInfo->code; + qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), + tstrerror(pTaskInfo->code)); + return NULL; } qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo)); @@ -181,36 +182,18 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { if (handle) { *handle = pTaskInfo->dsHandle; } - - while(1) { - st = taosGetTimestampUs(); - SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); - publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + st = taosGetTimestampUs(); + SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - if (pRes == NULL) { // no results generated yet, abort - dsEndPut(pTaskInfo->dsHandle, pTaskInfo->cost.elapsedTime); - return pTaskInfo->code; - } + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); - bool qcontinue = false; - SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; - pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &qcontinue); + qDebug("QInfo:0x%" PRIx64 " query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", + GET_TASKID(pTaskInfo), 0, 0L, 0); - if (isTaskKilled(pTaskInfo)) { - qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo)); - // } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - // qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - // pRuntimeEnv->resultInfo.total); - } - - if (!qcontinue) { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), - 0, 0L, 0); - return pTaskInfo->code; - } - } + atomic_store_64(&pTaskInfo->owner, 0); + return pRes; } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a515804234..b6df0c527e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4186,14 +4186,14 @@ void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType e } } -void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { +void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code) { SQueryProfEvent event; event.eventType = QUERY_PROF_QUERY_ABORT; event.eventTime = taosGetTimestampUs(); event.abortCode = code; - if (pQInfo->summary.queryProfEvents) { - taosArrayPush(pQInfo->summary.queryProfEvents, &event); + if (pTaskInfo->cost.queryProfEvents) { + taosArrayPush(pTaskInfo->cost.queryProfEvents, &event); } } @@ -7423,358 +7423,358 @@ int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* r * @param pExpr * @return */ -int32_t convertQueryMsg(SQueryTableReq *pQueryMsg, STaskParam* param) { - int32_t code = TSDB_CODE_SUCCESS; - -// if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { -// return TSDB_CODE_QRY_INVALID_MSG; +//int32_t convertQueryMsg(SQueryTableReq *pQueryMsg, STaskParam* param) { +// int32_t code = TSDB_CODE_SUCCESS; +// +//// if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { +//// return TSDB_CODE_QRY_INVALID_MSG; +//// } +// +// pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); +// pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); +// pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); +// pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); +// pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); +// pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); +// pQueryMsg->limit = htobe64(pQueryMsg->limit); +// pQueryMsg->offset = htobe64(pQueryMsg->offset); +// pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); +// +// pQueryMsg->order = htons(pQueryMsg->order); +// pQueryMsg->orderColId = htons(pQueryMsg->orderColId); +// pQueryMsg->queryType = htonl(pQueryMsg->queryType); +//// pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); +// +// pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); +// pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); +// pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); +// +// pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); +// pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); +// +// pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); +// pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); +// pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); +// pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); +// +// pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); +//// pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); +// pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); +// pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); +// pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); +//// pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); +//// pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); +// pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); +// pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); +// pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); +// pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); +// pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); +// +// // query msg safety check +// if (!validateQueryMsg(pQueryMsg)) { +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; // } - - pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); - pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); - pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); - pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - pQueryMsg->limit = htobe64(pQueryMsg->limit); - pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); - - pQueryMsg->order = htons(pQueryMsg->order); - pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->queryType = htonl(pQueryMsg->queryType); -// pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); - - pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); - pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); - pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); - - pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); - - pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); -// pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); - pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); - pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); - pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); -// pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); -// pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); - pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); - pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); - pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); - pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); - pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); - - // query msg safety check - if (!validateQueryMsg(pQueryMsg)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - - char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; - for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { - SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; - - pColInfo->colId = htons(pColInfo->colId); - pColInfo->type = htons(pColInfo->type); - pColInfo->bytes = htons(pColInfo->bytes); - pColInfo->flist.numOfFilters = 0; - - if (!isValidDataType(pColInfo->type)) { - //qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - -/* - int32_t numOfFilters = pColInfo->flist.numOfFilters; - if (numOfFilters > 0) { - pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); - if (pColInfo->flist.filterInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - } - - code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); - if (code != TSDB_CODE_SUCCESS) { - goto _cleanup; - } -*/ - } - - if (pQueryMsg->colCondLen > 0) { - param->colCond = calloc(1, pQueryMsg->colCondLen); - if (param->colCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); - pMsg += pQueryMsg->colCondLen; - } - - - param->tableScanOperator = pQueryMsg->tableScanOperator; - param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); - if (param->pExpr == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; - - for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - param->pExpr[i] = pExprMsg; - -// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); -// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); -// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); -// pExprMsg->colBytes = htons(pExprMsg->colBytes); -// pExprMsg->colType = htons(pExprMsg->colType); - -// pExprMsg->resType = htons(pExprMsg->resType); -// pExprMsg->resBytes = htons(pExprMsg->resBytes); - pExprMsg->interBytes = htonl(pExprMsg->interBytes); - -// pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); -// pExprMsg->resColId = htons(pExprMsg->resColId); -// pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); - pMsg += sizeof(SSqlExpr); - - for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); - pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); - - if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { - pExprMsg->param[j].pz = pMsg; - pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. - } else { - pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); - } - } - -// int16_t functionId = pExprMsg->functionId; -// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { -// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. -// code = TSDB_CODE_QRY_INVALID_MSG; +// +// char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; +// for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { +// SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; +// +// pColInfo->colId = htons(pColInfo->colId); +// pColInfo->type = htons(pColInfo->type); +// pColInfo->bytes = htons(pColInfo->bytes); +// pColInfo->flist.numOfFilters = 0; +// +// if (!isValidDataType(pColInfo->type)) { +// //qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; +// } +// +///* +// int32_t numOfFilters = pColInfo->flist.numOfFilters; +// if (numOfFilters > 0) { +// pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); +// if (pColInfo->flist.filterInfo == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto _cleanup; // } // } - -// if (pExprMsg->flist.numOfFilters > 0) { -// pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); -// } // -// deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); - pExprMsg = (SSqlExpr *)pMsg; - } - - if (pQueryMsg->secondStageOutput) { - pExprMsg = (SSqlExpr *)pMsg; - param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); - - for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { - param->pSecExpr[i] = pExprMsg; - -// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); -// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); -// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); -// pExprMsg->resType = htons(pExprMsg->resType); -// pExprMsg->resBytes = htons(pExprMsg->resBytes); -// pExprMsg->colBytes = htons(pExprMsg->colBytes); -// pExprMsg->colType = htons(pExprMsg->colType); - -// pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - - pMsg += sizeof(SSqlExpr); - - for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); - pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); - - if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { - pExprMsg->param[j].pz = pMsg; - pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. - } else { - pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); - } - } - -// int16_t functionId = pExprMsg->functionId; -// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { -// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. -// code = TSDB_CODE_QRY_INVALID_MSG; -// goto _cleanup; -// } -// } - - pExprMsg = (SSqlExpr *)pMsg; - } - } - - pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); - - if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); - if (param->pGroupColIndex == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].colId); - - param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].colIndex); - - param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); - pMsg += sizeof(param->pGroupColIndex[i].flag); - - memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); - pMsg += tListLen(param->pGroupColIndex[i].name); - } - - pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); - pQueryMsg->orderType = htons(pQueryMsg->orderType); - } - - pQueryMsg->fillType = htons(pQueryMsg->fillType); - if (pQueryMsg->fillType != TSDB_FILL_NONE) { - pQueryMsg->fillVal = (uint64_t)(pMsg); - - int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - v[i] = htobe64(v[i]); - } - - pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; - } - - if (pQueryMsg->numOfTags > 0) { - param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); - if (param->pTagColumnInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - - pTagCol->colId = htons(pTagCol->colId); - pTagCol->bytes = htons(pTagCol->bytes); - pTagCol->type = htons(pTagCol->type); -// pTagCol->flist.numOfFilters = 0; - - param->pTagColumnInfo[i] = *pTagCol; - pMsg += sizeof(SColumnInfo); - } - } - - // the tag query condition expression string is located at the end of query msg - if (pQueryMsg->tagCondLen > 0) { - param->tagCond = calloc(1, pQueryMsg->tagCondLen); - if (param->tagCond == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); - pMsg += pQueryMsg->tagCondLen; - } - - if (pQueryMsg->prevResultLen > 0) { - param->prevResult = calloc(1, pQueryMsg->prevResultLen); - if (param->prevResult == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _cleanup; - } - - memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); - pMsg += pQueryMsg->prevResultLen; - } - -// if (pQueryMsg->tbnameCondLen > 0) { -// param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); -// if (param->tbnameCond == NULL) { +// code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); +// if (code != TSDB_CODE_SUCCESS) { +// goto _cleanup; +// } +//*/ +// } +// +// if (pQueryMsg->colCondLen > 0) { +// param->colCond = calloc(1, pQueryMsg->colCondLen); +// if (param->colCond == NULL) { // code = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto _cleanup; // } // -// strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); -// pMsg += pQueryMsg->tbnameCondLen; +// memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); +// pMsg += pQueryMsg->colCondLen; // } - - //skip ts buf - if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { - pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; - } - - param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); - for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { - int32_t op = htonl(*(int32_t*)pMsg); - taosArrayPush(param->pOperator, &op); - - pMsg += sizeof(int32_t); - } - - if (pQueryMsg->udfContentLen > 0) { - // todo extract udf function in tudf.c -// param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); -// param->pUdfInfo->contLen = pQueryMsg->udfContentLen; // -// pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; -// param->pUdfInfo->resType = *(int8_t*) pMsg; -// pMsg += sizeof(int8_t); // -// param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); -// pMsg += sizeof(int16_t); +// param->tableScanOperator = pQueryMsg->tableScanOperator; +// param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); +// if (param->pExpr == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } // -// tstr* name = (tstr*)(pMsg); -// param->pUdfInfo->name = strndup(name->data, name->len); +// SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; +// +// for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { +// param->pExpr[i] = pExprMsg; +// +//// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); +//// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); +//// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +//// pExprMsg->colBytes = htons(pExprMsg->colBytes); +//// pExprMsg->colType = htons(pExprMsg->colType); +// +//// pExprMsg->resType = htons(pExprMsg->resType); +//// pExprMsg->resBytes = htons(pExprMsg->resBytes); +// pExprMsg->interBytes = htonl(pExprMsg->interBytes); +// +//// pExprMsg->functionId = htons(pExprMsg->functionId); +// pExprMsg->numOfParams = htons(pExprMsg->numOfParams); +//// pExprMsg->resColId = htons(pExprMsg->resColId); +//// pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); +// pMsg += sizeof(SSqlExpr); +// +// for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { +// pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); +// pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); +// +// if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { +// pExprMsg->param[j].pz = pMsg; +// pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. +// } else { +// pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); +// } +// } +// +//// int16_t functionId = pExprMsg->functionId; +//// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { +//// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. +//// code = TSDB_CODE_QRY_INVALID_MSG; +//// goto _cleanup; +//// } +//// } +// +//// if (pExprMsg->flist.numOfFilters > 0) { +//// pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); +//// } +//// +//// deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); +// pExprMsg = (SSqlExpr *)pMsg; +// } +// +// if (pQueryMsg->secondStageOutput) { +// pExprMsg = (SSqlExpr *)pMsg; +// param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); +// +// for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { +// param->pSecExpr[i] = pExprMsg; +// +//// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); +//// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); +//// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +//// pExprMsg->resType = htons(pExprMsg->resType); +//// pExprMsg->resBytes = htons(pExprMsg->resBytes); +//// pExprMsg->colBytes = htons(pExprMsg->colBytes); +//// pExprMsg->colType = htons(pExprMsg->colType); +// +//// pExprMsg->functionId = htons(pExprMsg->functionId); +// pExprMsg->numOfParams = htons(pExprMsg->numOfParams); +// +// pMsg += sizeof(SSqlExpr); +// +// for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { +// pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); +// pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); +// +// if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { +// pExprMsg->param[j].pz = pMsg; +// pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. +// } else { +// pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); +// } +// } +// +//// int16_t functionId = pExprMsg->functionId; +//// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { +//// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. +//// code = TSDB_CODE_QRY_INVALID_MSG; +//// goto _cleanup; +//// } +//// } +// +// pExprMsg = (SSqlExpr *)pMsg; +// } +// } +// +// pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); +// +// if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns +// param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); +// if (param->pGroupColIndex == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } +// +// for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { +// param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); +// pMsg += sizeof(param->pGroupColIndex[i].colId); +// +// param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); +// pMsg += sizeof(param->pGroupColIndex[i].colIndex); +// +// param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); +// pMsg += sizeof(param->pGroupColIndex[i].flag); +// +// memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); +// pMsg += tListLen(param->pGroupColIndex[i].name); +// } +// +// pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); +// pQueryMsg->orderType = htons(pQueryMsg->orderType); +// } +// +// pQueryMsg->fillType = htons(pQueryMsg->fillType); +// if (pQueryMsg->fillType != TSDB_FILL_NONE) { +// pQueryMsg->fillVal = (uint64_t)(pMsg); +// +// int64_t *v = (int64_t *)pMsg; +// for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { +// v[i] = htobe64(v[i]); +// } +// +// pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; +// } +// +// if (pQueryMsg->numOfTags > 0) { +// param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); +// if (param->pTagColumnInfo == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } +// +// for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { +// SColumnInfo* pTagCol = (SColumnInfo*) pMsg; +// +// pTagCol->colId = htons(pTagCol->colId); +// pTagCol->bytes = htons(pTagCol->bytes); +// pTagCol->type = htons(pTagCol->type); +//// pTagCol->flist.numOfFilters = 0; +// +// param->pTagColumnInfo[i] = *pTagCol; +// pMsg += sizeof(SColumnInfo); +// } +// } +// +// // the tag query condition expression string is located at the end of query msg +// if (pQueryMsg->tagCondLen > 0) { +// param->tagCond = calloc(1, pQueryMsg->tagCondLen); +// if (param->tagCond == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } +// +// memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); +// pMsg += pQueryMsg->tagCondLen; +// } +// +// if (pQueryMsg->prevResultLen > 0) { +// param->prevResult = calloc(1, pQueryMsg->prevResultLen); +// if (param->prevResult == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } +// +// memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); +// pMsg += pQueryMsg->prevResultLen; +// } +// +//// if (pQueryMsg->tbnameCondLen > 0) { +//// param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); +//// if (param->tbnameCond == NULL) { +//// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +//// goto _cleanup; +//// } +//// +//// strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); +//// pMsg += pQueryMsg->tbnameCondLen; +//// } +// +// //skip ts buf +// if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { +// pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; +// } +// +// param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); +// for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { +// int32_t op = htonl(*(int32_t*)pMsg); +// taosArrayPush(param->pOperator, &op); // -// pMsg += varDataTLen(name); -// param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); // pMsg += sizeof(int32_t); +// } // -// param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); -// pMsg += sizeof(int32_t); +// if (pQueryMsg->udfContentLen > 0) { +// // todo extract udf function in tudf.c +//// param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); +//// param->pUdfInfo->contLen = pQueryMsg->udfContentLen; +//// +//// pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; +//// param->pUdfInfo->resType = *(int8_t*) pMsg; +//// pMsg += sizeof(int8_t); +//// +//// param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); +//// pMsg += sizeof(int16_t); +//// +//// tstr* name = (tstr*)(pMsg); +//// param->pUdfInfo->name = strndup(name->data, name->len); +//// +//// pMsg += varDataTLen(name); +//// param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); +//// pMsg += sizeof(int32_t); +//// +//// param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); +//// pMsg += sizeof(int32_t); +//// +//// param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); +//// memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); // -// param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); -// memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); - - pMsg += pQueryMsg->udfContentLen; - } - - param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - - //qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " -// "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, -// pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, -// pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, -// pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); - - //qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); - return TSDB_CODE_SUCCESS; - -_cleanup: - freeParam(param); - return code; -} +// pMsg += pQueryMsg->udfContentLen; +// } +// +// param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); +// +// SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; +// if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; +// } +// +// //qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " +//// "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, +//// pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, +//// pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, +//// pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); +// +// //qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); +// return TSDB_CODE_SUCCESS; +// +//_cleanup: +// freeParam(param); +// return code; +//} int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { if (filterNum <= 0) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 09e9af9788..61bd82a49a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -744,7 +744,7 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t queryRsped = true; DataSinkHandle sinkHandle = NULL; - code = qExecTask(pTaskInfo, &sinkHandle); + SSDataBlock* pRes = qExecTask(pTaskInfo, &sinkHandle); if (code) { QW_TASK_ELOG("qExecTask failed, code:%x", code); QW_ERR_JRET(code); From 913455684f950cf30dd2ebde77c14318d65a6370 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 18 Jan 2022 14:12:25 +0800 Subject: [PATCH 010/103] feature/qnode --- include/libs/executor/dataSinkMgt.h | 2 +- include/libs/executor/executor.h | 6 +- source/libs/executor/inc/dataSinkInt.h | 2 +- source/libs/executor/src/dataDispatcher.c | 4 +- source/libs/executor/src/dataSinkMgt.c | 2 +- source/libs/executor/src/executorMain.c | 26 +++--- source/libs/executor/test/executorTests.cpp | 3 +- source/libs/qworker/src/qworker.c | 48 ++++++++--- source/libs/scheduler/src/scheduler.c | 11 ++- source/libs/scheduler/test/schedulerTests.cpp | 80 ++++++++++++++----- 10 files changed, 132 insertions(+), 52 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 5cef3b2253..371cb12405 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -70,7 +70,7 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH */ int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue); -void dsEndPut(DataSinkHandle handle, int64_t useconds); +void dsEndPut(DataSinkHandle handle, uint64_t useconds); /** * Get the length of the data returned by the next call to dsGetDataBlock. diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e4f8d291eb..0fc7fd679e 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "common.h" + typedef void* qTaskInfo_t; typedef void* DataSinkHandle; struct SSubplan; @@ -34,7 +36,7 @@ struct SSubplan; * @param qId * @return */ -int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); +int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, @@ -44,7 +46,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskI * @param handle * @return */ -struct SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle); +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); /** * Retrieve the produced results information, if current query is not paused or completed, diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 69727626af..7003564365 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -32,7 +32,7 @@ typedef struct SDataSinkManager { } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); -typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, int64_t useconds); +typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds); typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd); typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8280f9d0af..edba4fc97d 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -44,7 +44,7 @@ typedef struct SDataDispatchHandle { SDataDispatchBuf nextOutput; int32_t status; bool queryEnd; - int64_t useconds; + uint64_t useconds; pthread_mutex_t mutex; } SDataDispatchHandle; @@ -158,7 +158,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, return TSDB_CODE_SUCCESS; } -static void endPut(struct SDataSinkHandle* pHandle, int64_t useconds) { +static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; pthread_mutex_lock(&pDispatcher->mutex); pDispatcher->queryEnd = true; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 80d99f96c6..eb1f75f359 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -37,7 +37,7 @@ int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pC return pHandleImpl->fPut(pHandleImpl, pInput, pContinue); } -void dsEndPut(DataSinkHandle handle, int64_t useconds) { +void dsEndPut(DataSinkHandle handle, uint64_t useconds) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; return pHandleImpl->fEndPut(pHandleImpl, useconds); } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index e451f888bb..1f5d0cd059 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -68,7 +68,7 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { +int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { assert(tsdb != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; @@ -85,6 +85,8 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ code = dsCreateDataSinker(pSubplan->pDataSink, &(*pTask)->dsHandle); + *handle = (*pTask)->dsHandle; + _error: // if failed to add ref for all tables in this query, abort current query return code; @@ -135,16 +137,18 @@ int waitMoment(SQInfo* pQInfo){ } #endif -SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); + *pRes = NULL; + int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; - return NULL; + return pTaskInfo->code; } if (pTaskInfo->cost.start == 0) { @@ -153,7 +157,7 @@ SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { if (isTaskKilled(pTaskInfo)) { qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo)); - return NULL; + return TSDB_CODE_SUCCESS; } // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; @@ -170,7 +174,7 @@ SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { pTaskInfo->code = ret; qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); - return NULL; + return pTaskInfo->code; } qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo)); @@ -179,21 +183,21 @@ SSDataBlock* qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = 0; - if (handle) { - *handle = pTaskInfo->dsHandle; - } - st = taosGetTimestampUs(); - SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); + *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (NULL == *pRes) { + *useconds = pTaskInfo->cost.elapsedTime; + } + qDebug("QInfo:0x%" PRIx64 " query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), 0, 0L, 0); atomic_store_64(&pTaskInfo->owner, 0); - return pRes; + return pTaskInfo->code; } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 5f5fddbe28..c528d879a3 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -217,5 +217,6 @@ TEST(testCase, build_executor_tree_Test) { "}"; SExecTaskInfo* pTaskInfo = nullptr; - int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo); + DataSinkHandle sinkHandle = nullptr; + int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo, &sinkHandle); } \ No newline at end of file diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 61bd82a49a..a0beaba61d 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -458,6 +458,37 @@ _return: QW_RET(code); } +int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHandle) { + int32_t code = 0; + bool qcontinue = true; + SSDataBlock* pRes = NULL; + uint64_t useconds = 0; + + while (qcontinue) { + code = qExecTask(taskHandle, &pRes, &useconds); + if (code) { + QW_TASK_ELOG("qExecTask failed, code:%x", code); + QW_ERR_JRET(code); + } + + if (NULL == pRes) { + QW_TASK_DLOG("query done, useconds:%"PRIu64, useconds); + dsEndPut(sinkHandle, useconds); + break; + } + + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; + code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); + if (code) { + QW_TASK_ELOG("dsPutDataBlock failed, code:%x", code); + QW_ERR_JRET(code); + } + } + +_return: + + QW_RET(code); +} int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void **rspMsg, SOutputData *pOutput) { @@ -733,7 +764,9 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo); + DataSinkHandle sinkHandle = NULL; + + code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); @@ -743,12 +776,7 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t queryRsped = true; - DataSinkHandle sinkHandle = NULL; - SSDataBlock* pRes = qExecTask(pTaskInfo, &sinkHandle); - if (code) { - QW_TASK_ELOG("qExecTask failed, code:%x", code); - QW_ERR_JRET(code); - } + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), pTaskInfo, sinkHandle)); _return: @@ -840,11 +868,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; - code = qExecTask(taskHandle, &sinkHandle); - if (code) { - QW_TASK_ELOG("qExecTask failed, code:%x", code); - QW_ERR_JRET(code); - } + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), taskHandle, sinkHandle)); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CQUERY); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 2d5322fc2c..c53926f8c1 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -412,6 +412,8 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + ++addNum; } } @@ -792,6 +794,11 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } + + SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg; + if (rsp) { + pJob->resNumOfRows += rsp->affectedRows; + } #endif SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -1355,9 +1362,9 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, true)); - *pJob = job; + job = *pJob; pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index d72c4920d5..5332c6fcd1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -34,10 +34,12 @@ #include "stub.h" #include "addr_any.h" + namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); + void schtInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; @@ -113,9 +115,9 @@ void schtBuildInsertDag(SQueryDag *dag) { dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = taosArrayInit(1, POINTER_BYTES); - SArray *inserta = taosArrayInit(dag->numOfSubplans, sizeof(SSubplan)); + SArray *inserta = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); - SSubplan insertPlan[2] = {0}; + SSubplan *insertPlan = (SSubplan *)calloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; insertPlan[0].id.templateId = 0x0000000000000003; @@ -131,6 +133,7 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; insertPlan[0].pDataSink = (SDataSink*)calloc(1, sizeof(SDataSink)); + insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; insertPlan[1].id.templateId = 0x0000000000000003; @@ -146,10 +149,11 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; insertPlan[1].pDataSink = (SDataSink*)calloc(1, sizeof(SDataSink)); + insertPlan[1].msgType = TDMT_VND_SUBMIT; - - taosArrayPush(inserta, &insertPlan[0]); - taosArrayPush(inserta, &insertPlan[1]); + taosArrayPush(inserta, &insertPlan); + insertPlan += 1; + taosArrayPush(inserta, &insertPlan); taosArrayPush(dag->pSubplans, &inserta); } @@ -210,6 +214,24 @@ void schtSetRpcSendRequest() { } } +int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { + return 0; +} + + +void schtSetAsyncSendMsgToServer() { + static Stub stub; + stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); + for (const auto& f : result) { + stub.set(f.second, schtAsyncSendMsgToServer); + } + } +} + void *schtSendRsp(void *param) { SSchJob *job = NULL; @@ -230,7 +252,7 @@ void *schtSendRsp(void *param) { SShellSubmitRsp rsp = {0}; rsp.affectedRows = 10; - schHandleResponseMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); + schHandleResponseMsg(job, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); pIter = taosHashIterate(job->execTasks, pIter); } @@ -238,6 +260,23 @@ void *schtSendRsp(void *param) { return NULL; } +void *schtCreateFetchRspThread(void *param) { + struct SSchJob* job = (struct SSchJob*)param; + + sleep(1); + + int32_t code = 0; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + rsp->completed = 1; + rsp->numOfRows = 10; + code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(rsp), 0); + + assert(code == 0); +} + + + + struct SSchJob *pInsertJob = NULL; } @@ -266,6 +305,7 @@ TEST(queryTest, normalCase) { schtSetPlanToString(); schtSetExecNode(); + schtSetAsyncSendMsgToServer(); code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); ASSERT_EQ(code, 0); @@ -276,7 +316,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -287,8 +327,8 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); - + code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + printf("code:%d", code); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); } @@ -298,7 +338,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -309,22 +349,19 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); } - SRetrieveTableRsp rsp = {0}; - rsp.completed = 1; - rsp.numOfRows = 10; - code = schHandleResponseMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0); - - ASSERT_EQ(code, 0); + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_t thread1; + pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job); - void *data = NULL; - + void *data = NULL; code = scheduleFetchRows(job, &data); ASSERT_EQ(code, 0); @@ -340,6 +377,8 @@ TEST(queryTest, normalCase) { scheduleFreeJob(pJob); schtFreeQueryDag(&dag); + + schedulerDestroy(); } @@ -369,6 +408,7 @@ TEST(insertTest, normalCase) { schtBuildInsertDag(&dag); schtSetPlanToString(); + schtSetAsyncSendMsgToServer(); pthread_attr_t thattr; pthread_attr_init(&thattr); @@ -382,6 +422,8 @@ TEST(insertTest, normalCase) { ASSERT_EQ(res.numOfRows, 20); scheduleFreeJob(pInsertJob); + + schedulerDestroy(); } TEST(multiThread, forceFree) { From 9d50c907cf527ce164d861110c8ad329239ab48c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 06:31:44 +0000 Subject: [PATCH 011/103] more --- source/dnode/vnode/inc/meta.h | 7 ++- source/dnode/vnode/src/meta/metaBDBImpl.c | 55 +++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 031ed178f0..75ddc37a03 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -42,7 +42,8 @@ typedef struct { SSchema *pSchema; } SSchemaWrapper; -typedef struct SMTbCursor SMTbCursor; +typedef struct SMTbCursor SMTbCursor; +typedef struct SMCtbCursor SMCtbCursor; typedef SVCreateTbReq STbCfg; @@ -64,6 +65,10 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); char * metaTbCursorNext(SMTbCursor *pTbCur); +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); +char * metaCtbCursorNext(SMCtbCursor *pCtbCur); + // Options void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index ae13d9fddc..490333397f 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -621,4 +621,59 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { tdDestroyTSchemaBuilder(&sb); return pTSchema; +} + +struct SMCtbCursor { + DBC * pCur; + tb_uid_t suid; +}; + +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { + SMCtbCursor *pCtbCur = NULL; + SMetaDB * pDB = pMeta->pDB; + int ret; + + pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); + if (pCtbCur == NULL) { + return NULL; + } + + pCtbCur->suid = uid; + ret = pDB->pCtbIdx->cursor(pDB->pCtbIdx, NULL, &(pCtbCur->pCur), 0); + if (ret != 0) { + free(pCtbCur); + return NULL; + } + + return pCtbCur; +} + +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { + if (pCtbCur) { + if (pCtbCur->pCur) { + pCtbCur->pCur->close(pCtbCur->pCur); + } + + free(pCtbCur); + } +} + +char *metaCtbCursorNext(SMCtbCursor *pCtbCur) { + DBT skey = {0}; + DBT pkey = {0}; + DBT pval = {0}; + void * pBuf; + STbCfg tbCfg; + + // Set key + skey.data = &(pCtbCur->suid); + skey.size = sizeof(pCtbCur->suid); + + if (pCtbCur->pCur->get(pCtbCur->pCur, &skey, &pval, DB_NEXT) == 0) { + pBuf = pval.data; + metaDecodeTbInfo(pBuf, &tbCfg); + return tbCfg.name; + } else { + return NULL; + } } \ No newline at end of file From d6359f3ab11b7001af21b147dbcd622b6f1739b2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Jan 2022 23:20:11 -0800 Subject: [PATCH 012/103] refact tier and disk in tfs module --- include/libs/tfs/tfs.h | 12 +--- include/os/osSysinfo.h | 6 +- source/libs/tfs/inc/tfsInt.h | 47 +++++-------- source/libs/tfs/src/tfs.c | 64 ++++++++---------- source/libs/tfs/src/tfsDisk.c | 31 ++++----- source/libs/tfs/src/tfsTier.c | 122 ++++++++++++---------------------- source/os/src/osSysinfo.c | 9 ++- 7 files changed, 109 insertions(+), 182 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 973231c264..3828a93144 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -36,22 +36,14 @@ typedef struct { // FS APIs ==================================== typedef struct { - int64_t tsize; + int64_t total; int64_t used; int64_t avail; } SFSMeta; -typedef struct { - int64_t size; - int64_t used; - int64_t free; - int16_t nAvailDisks; // # of Available disks -} STierMeta; - int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk); void tfsCleanup(); -void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); -void tfsGetMeta(SFSMeta *pMeta); +void tfsUpdateSize(SFSMeta *pFSMeta); void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id); const char *TFS_PRIMARY_PATH(); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 36ff4194d8..9dcb075489 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -35,12 +35,12 @@ extern char tsLocale[]; extern char tsCharset[]; // default encode string typedef struct { - int64_t tsize; + int64_t total; int64_t used; int64_t avail; -} SysDiskSize; +} SDiskSize; -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize); +int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); bool taosReadProcIO(int64_t *rchars, int64_t *wchars); diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index bfa4c4380d..ce1436eb29 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TFSINT_H_ -#define _TD_TFSINT_H_ +#ifndef _TD_TFS_INT_H_ +#define _TD_TFS_INT_H_ #include "os.h" @@ -39,54 +39,39 @@ extern int32_t fsDebugFlag; // Global Definitions #define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 -// tdisk.c ====================================================== -typedef struct { - int64_t size; - int64_t used; - int64_t free; -} SDiskMeta; - typedef struct SDisk { int32_t level; int32_t id; - char dir[TSDB_FILENAME_LEN]; - SDiskMeta dmeta; + char *path; + SDiskSize size; } SDisk; -#define DISK_LEVEL(pd) ((pd)->level) -#define DISK_ID(pd) ((pd)->id) -#define DISK_DIR(pd) ((pd)->dir) -#define DISK_META(pd) ((pd)->dmeta) -#define DISK_SIZE(pd) ((pd)->dmeta.size) -#define DISK_USED_SIZE(pd) ((pd)->dmeta.used) -#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free) - -SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); -SDisk *tfsFreeDisk(SDisk *pDisk); -int32_t tfsUpdateDiskInfo(SDisk *pDisk); - -// ttier.c ====================================================== - typedef struct STier { pthread_spinlock_t lock; int32_t level; - int16_t ndisk; // # of disks mounted to this tier - int16_t nextid; // next disk id to allocate - STierMeta tmeta; + int16_t nextid; // next disk id to allocate + int16_t ndisk; // # of disks mounted to this tier + int16_t nAvailDisks; // # of Available disks SDisk *disks[TSDB_MAX_DISKS_PER_TIER]; + SDiskSize size; } STier; #define TIER_LEVEL(pt) ((pt)->level) #define TIER_NDISKS(pt) ((pt)->ndisk) #define TIER_SIZE(pt) ((pt)->tmeta.size) #define TIER_FREE_SIZE(pt) ((pt)->tmeta.free) -#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks) + #define DISK_AT_TIER(pt, id) ((pt)->disks[id]) +#define DISK_DIR(pd) ((pd)->path) + +SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); +SDisk *tfsFreeDisk(SDisk *pDisk); +int32_t tfsUpdateDiskSize(SDisk *pDisk); int32_t tfsInitTier(STier *pTier, int32_t level); void tfsDestroyTier(STier *pTier); SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); -void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta); +void tfsUpdateTierSize(STier *pTier); int32_t tfsAllocDiskOnTier(STier *pTier); void tfsPosNextId(STier *pTier); @@ -94,4 +79,4 @@ void tfsPosNextId(STier *pTier); } #endif -#endif /*_TD_TFSINT_H_*/ +#endif /*_TD_TFS_INT_H_*/ diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 87adba6bea..1d11cd6df2 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -59,7 +59,10 @@ static SDisk *tfsNextDisk(SDiskIter *pIter); // FS APIs ==================================== int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) { - ASSERT(ndisk > 0); + if (ndisk < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } for (int32_t level = 0; level < TSDB_MAX_TIERS; level++) { if (tfsInitTier(TFS_TIER_AT(level), level) < 0) { @@ -96,7 +99,7 @@ int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) { return -1; } - tfsUpdateInfo(NULL, NULL, 0); + tfsUpdateSize(NULL); for (int32_t level = 0; level < TFS_NLEVEL(); level++) { tfsPosNextId(TFS_TIER_AT(level)); } @@ -114,27 +117,22 @@ void tfsCleanup() { } } -void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) { - SFSMeta fsMeta; - STierMeta tierMeta; +void tfsUpdateSize(SFSMeta *pFSMeta) { + SFSMeta fsMeta = {0}; + SDiskSize size = {0}; if (pFSMeta == NULL) { pFSMeta = &fsMeta; } - memset(pFSMeta, 0, sizeof(*pFSMeta)); + memset(pFSMeta, 0, sizeof(SFSMeta)); for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - STierMeta *pTierMeta = &tierMeta; - if (tierMetas && level < numTiers) { - pTierMeta = tierMetas + level; - } - STier *pTier = TFS_TIER_AT(level); - tfsUpdateTierInfo(pTier, pTierMeta); - pFSMeta->tsize += pTierMeta->size; - pFSMeta->avail += pTierMeta->free; - pFSMeta->used += pTierMeta->used; + tfsUpdateTierSize(pTier); + pFSMeta->total += pTier->size.total; + pFSMeta->avail += pTier->size.avail; + pFSMeta->used += pTier->size.used; } tfsLock(); @@ -142,14 +140,6 @@ void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numTiers) { tfsUnLock(); } -void tfsGetMeta(SFSMeta *pMeta) { - ASSERT(pMeta); - - tfsLock(); - *pMeta = pfs->meta; - tfsUnLock(); -} - /* Allocate an existing available tier level */ void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id) { @@ -307,9 +297,9 @@ int32_t tfsRmdir(const char *rname) { for (int32_t level = 0; level < TFS_NLEVEL(); level++) { STier *pTier = TFS_TIER_AT(level); for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { - SDisk *pDisk = DISK_AT_TIER(pTier, id); + SDisk *pDisk = pTier->disks[id]; - snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); + snprintf(aname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TS_PATH_DELIMITER, rname); taosRemoveDir(aname); } @@ -351,7 +341,7 @@ struct TDIR { TDIR *tfsOpendir(const char *rname) { TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir)); if (tdir == NULL) { - terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -416,7 +406,7 @@ static int32_t tfsMount(SDiskCfg *pCfg) { fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); return -1; } - did.id = DISK_ID(pDisk); + did.id = pDisk->id; taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1; @@ -551,10 +541,10 @@ static int32_t tfsOpendirImpl(TDIR *tdir) { pDisk = tfsNextDisk(&(tdir->iter)); if (pDisk == NULL) return 0; - tdir->level = DISK_LEVEL(pDisk); - tdir->id = DISK_ID(pDisk); + tdir->level = pDisk->level; + tdir->id = pDisk->id; - snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname); + snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TS_PATH_DELIMITER,tdir->dirname); tdir->dir = opendir(adir); if (tdir->dir != NULL) break; } @@ -569,8 +559,8 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) { if (pDisk == NULL) return NULL; - int32_t level = DISK_LEVEL(pDisk); - int32_t id = DISK_ID(pDisk); + int32_t level = pDisk->level; + int32_t id = pDisk->id; id++; if (id < TIER_NDISKS(TFS_TIER_AT(level))) { @@ -593,21 +583,21 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) { // OTHER FUNCTIONS =================================== void taosGetDisk() { const double unit = 1024 * 1024 * 1024; - SysDiskSize diskSize; + SDiskSize diskSize; SFSMeta fsMeta; - tfsUpdateInfo(&fsMeta, NULL, 0); - tsTotalDataDirGB = (float)(fsMeta.tsize / unit); + tfsUpdateSize(&fsMeta); + tsTotalDataDirGB = (float)(fsMeta.total / unit); tsUsedDataDirGB = (float)(fsMeta.used / unit); tsAvailDataDirGB = (float)(fsMeta.avail / unit); if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.tsize / unit); + tsTotalLogDirGB = (float)(diskSize.total / unit); tsAvailLogDirGB = (float)(diskSize.avail / unit); } if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.tsize / unit); + tsTotalTmpDirGB = (float)(diskSize.total / unit); tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); } } diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index dd4f349c42..a5ef1121ff 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -16,44 +16,41 @@ #define _DEFAULT_SOURCE #include "tfsInt.h" -SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir) { +SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { SDisk *pDisk = calloc(1, sizeof(SDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pDisk->path = strdup(path); + if (pDisk->path == NULL) { + free(pDisk); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pDisk->level = level; pDisk->id = id; - tstrncpy(pDisk->dir, dir, TSDB_FILENAME_LEN); - + taosGetDiskSize(pDisk->path, &pDisk->size); return pDisk; } SDisk *tfsFreeDisk(SDisk *pDisk) { if (pDisk != NULL) { + free(pDisk->path); free(pDisk); } return NULL; } -int32_t tfsUpdateDiskInfo(SDisk *pDisk) { - if (pDisk == NULL) { - terrno = TSDB_CODE_INVALID_PARA; +int32_t tfsUpdateDiskSize(SDisk *pDisk) { + if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr()); return -1; } - SysDiskSize diskSize = {0}; - if (taosGetDiskSize(pDisk->dir, &diskSize) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir, - strerror(errno)); - return -1 - } - - pDisk->dmeta.size = diskSize.tsize; - pDisk->dmeta.used = diskSize.used; - pDisk->dmeta.free = diskSize.avail; return 0; } diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 8b057d8755..2a26c23ead 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -20,16 +20,10 @@ #define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) int32_t tfsInitTier(STier *pTier, int32_t level) { - if (pTier == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - return -1; - } - memset(pTier, 0, sizeof(STier)); - int32_t code = pthread_spin_init(&pTier->lock, 0); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); + if (pthread_spin_init(&pTier->lock, 0) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -38,10 +32,8 @@ int32_t tfsInitTier(STier *pTier, int32_t level) { } void tfsDestroyTier(STier *pTier) { - if (pTier == NULL) return; - for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { - DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id)); + pTier->disks[id] = tfsFreeDisk(pTier->disks[id]); } pTier->ndisk = 0; @@ -49,19 +41,14 @@ void tfsDestroyTier(STier *pTier) { } SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { - if (pTier == NULL || pCfg == NULL || pTier->level != pCfg->level) { - terrno = TSDB_CODE_INVALID_PARA; - return -1; - } - - if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) { + if (pTier->ndisk >= TSDB_MAX_DISKS_PER_TIER) { terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; return NULL; } int32_t id = 0; if (pTier->level == 0) { - if (DISK_AT_TIER(pTier, 0) != NULL) { + if (pTier->disks[0] != NULL) { id = pTier->ndisk; } else { if (pCfg->primary) { @@ -69,108 +56,85 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { } else { id = pTier->ndisk + 1; } - if (id >= TSDB_MAX_DISKS_PER_TIER) { - terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; - return NULL; - } } } else { id = pTier->ndisk; } + if (id >= TSDB_MAX_DISKS_PER_TIER) { + terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; + return NULL; + } + SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); if (pDisk == NULL) return NULL; - DISK_AT_TIER(pTier, id) = pDisk; + pTier->disks[id] = pDisk; pTier->ndisk++; fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); - return DISK_AT_TIER(pTier, id); + return pTier->disks[id]; } -void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) { - STierMeta tmeta = {0}; - - if (pTierMeta == NULL) { - pTierMeta = &tmeta; - } - memset(pTierMeta, 0, sizeof(STierMeta)); +void tfsUpdateTierSize(STier *pTier) { + SDiskSize size = {0}; + int16_t nAvailDisks = 0; tfsLockTier(pTier); for (int32_t id = 0; id < pTier->ndisk; id++) { - if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) != 0) { - continue; - } + SDisk *pDisk = pTier->disks[id]; + if (pDisk == NULL) continue; - pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id)); - pTierMeta->used += DISK_USED_SIZE(DISK_AT_TIER(pTier, id)); - pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id)); - pTierMeta->nAvailDisks++; + size.total += pDisk->size.total; + size.used += pDisk->size.used; + size.avail += pDisk->size.avail; + nAvailDisks++; } - pTier->tmeta = *pTierMeta; + pTier->size = size; + pTier->nAvailDisks = nAvailDisks; tfsUnLockTier(pTier); } // Round-Robin to allocate disk on a tier int32_t tfsAllocDiskOnTier(STier *pTier) { - if (pTier == NULL || pTier->ndisk <= 0) { - terrno = TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_FS_NO_VALID_DISK; + + tfsLockTier(pTier); + + if (pTier->ndisk <= 0 || pTier->nAvailDisks <= 0) { + tfsUnLockTier(pTier); return -1; } - tfsLockTier(pTier); + int32_t retId = -1; + for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; ++id) { + int32_t diskId = (pTier->nextid + id) % pTier->ndisk; + SDisk *pDisk = pTier->disks[diskId]; - if (TIER_AVAIL_DISKS(pTier) <= 0) { - tfsUnLockTier(pTier); - return TFS_UNDECIDED_ID; - } + if (pDisk == NULL) continue; - int32_t id = pTier->nextid; - while (true) { - SDisk *pDisk = DISK_AT_TIER(pTier, id); - if (pDisk == NULL) { - tfsUnLockTier(pTier); - return TFS_UNDECIDED_ID; - } + if (pDisk->size.avail < TFS_MIN_DISK_FREE_SIZE) continue; - if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) { - id = (id + 1) % pTier->ndisk; - if (id == pTier->nextid) { - tfsUnLockTier(pTier); - return TFS_UNDECIDED_ID; - } else { - continue; - } - } else { - pTier->nextid = (id + 1) % pTier->ndisk; - break; - } + retId = diskId; + terrno = 0; + pTier->nextid = (diskId + 1) % pTier->ndisk; + break; } tfsUnLockTier(pTier); - return id; -} - -void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) { - if (pTierMeta == NULL || pTierMeta == NULL) return; - - tfsLockTier(pTier); - *pTierMeta = pTier->tmeta; - tfsUnLockTier(pTier); + return retId; } void tfsPosNextId(STier *pTier) { - if (pTier == NULL || pTier->ndisk <= 0) return; - int32_t nextid = 0; for (int32_t id = 1; id < pTier->ndisk; id++) { - SDisk *pLDisk = DISK_AT_TIER(pTier, nextid); - SDisk *pDisk = DISK_AT_TIER(pTier, id); - if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) { + SDisk *pLDisk = pTier->disks[nextid]; + SDisk *pDisk = pTier->disks[id]; + if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) { nextid = id; } } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 0be17ca2b9..cae1b18b3c 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -121,7 +121,7 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { return true; } -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { +int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { unsigned _int64 i64FreeBytesToCaller; unsigned _int64 i64TotalBytes; unsigned _int64 i64FreeBytes; @@ -438,7 +438,7 @@ int taosSystem(const char *cmd) { void taosSetCoreDump() {} -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { +int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { struct statvfs info; if (statvfs(dataDir, &info)) { //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); @@ -771,13 +771,12 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { return true; } -int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { +int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { struct statvfs info; if (statvfs(dataDir, &info)) { - //printf("failed to get disk size, dataDir:%s errno:%s", dataDir, strerror(errno)); return -1; } else { - diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->total = info.f_blocks * info.f_frsize; diskSize->avail = info.f_bavail * info.f_frsize; diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; return 0; From ddf7dcc94f637ce0a2e3df75e61da629326bbdc4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Jan 2022 16:02:12 +0800 Subject: [PATCH 013/103] fix mem leak --- include/common/tmsg.h | 11 ++++--- source/client/src/clientHb.c | 12 ++++--- source/client/test/clientTests.cpp | 4 ++- source/dnode/mnode/impl/inc/mndDef.h | 18 +++++++++-- source/dnode/mnode/impl/src/mndConsumer.c | 39 ++++++++++++++--------- source/dnode/mnode/impl/src/mndProfile.c | 5 +++ source/dnode/mnode/impl/src/mnode.c | 11 +++++++ 7 files changed, 73 insertions(+), 27 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f32fdcbae7..dfd376f1e9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -188,16 +188,19 @@ void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); static FORCE_INLINE void tFreeClientHbReq(void *pReq) { SClientHbReq* req = (SClientHbReq*)pReq; - taosHashCleanup(req->info); - free(pReq); + if (req->info) taosHashCleanup(req->info); } int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); -static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { +static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; - //taosArrayDestroyEx(req->reqs, tFreeClientHbReq); + if (deep) { + taosArrayDestroyEx(req->reqs, tFreeClientHbReq); + } else { + taosArrayDestroy(req->reqs); + } free(pReq); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6d7fc9f81a..97ef77abcc 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -60,15 +60,17 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } +#if 0 pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, NULL); while (pIter != NULL) { FGetConnInfo getConnInfoFp = (FGetConnInfo)pIter; SClientHbKey connKey; taosHashCopyKey(pIter, &connKey); - getConnInfoFp(connKey, NULL); + SArray* pArray = getConnInfoFp(connKey, NULL); pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, pIter); } +#endif return pBatchReq; } @@ -99,12 +101,12 @@ static void* hbThreadFunc(void* param) { //TODO: error handling break; } - void *bufCopy = buf; - tSerializeSClientHbBatchReq(&bufCopy, pReq); + void *abuf = buf; + tSerializeSClientHbBatchReq(&abuf, pReq); SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tFreeClientHbBatchReq(pReq); + tFreeClientHbBatchReq(pReq, false); free(buf); break; } @@ -120,7 +122,7 @@ static void* hbThreadFunc(void* param) { int64_t transporterId = 0; SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); - tFreeClientHbBatchReq(pReq); + tFreeClientHbBatchReq(pReq, false); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 415d6a57ce..13915fd85d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,7 +53,9 @@ TEST(testCase, connect_Test) { if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } - sleep(3); + while (1) { + sleep(3); + } taos_close(pConn); } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index de101b0f06..7da0f3826e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -325,6 +325,19 @@ typedef struct SMqTopicConsumer { } SMqTopicConsumer; #endif +typedef struct SMqConsumerEp { + int32_t vgId; + SEpSet epset; + int64_t consumerId; +} SMqConsumerEp; + +typedef struct SMqCgroupTopicPair { + char key[TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN]; + SArray* assigned; + SArray* unassignedConsumer; + SArray* unassignedVg; +} SMqCgroupTopicPair; + typedef struct SMqCGroup { char name[TSDB_CONSUMER_GROUP_LEN]; int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal @@ -351,10 +364,11 @@ typedef struct SMqTopicObj { // TODO: add cache and change name to id typedef struct SMqConsumerTopic { + char name[TSDB_TOPIC_FNAME_LEN]; int32_t epoch; - char name[TSDB_TOPIC_NAME_LEN]; //TODO: replace with something with ep SList *vgroups; // SList + SArray *pVgInfo; // SArray } SMqConsumerTopic; typedef struct SMqConsumerObj { @@ -362,7 +376,7 @@ typedef struct SMqConsumerObj { SRWLatch lock; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray *topics; // SArray - SHashObj *topicHash; + SHashObj *topicHash; //SHashObj } SMqConsumerObj; typedef struct SMqSubConsumerObj { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 54e640d8b7..d27bf53a90 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -204,34 +204,37 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; char *msgStr = pMsg->rpcMsg.pCont; - SCMSubscribeReq *pSubscribe; - tDeserializeSCMSubscribeReq(msgStr, pSubscribe); - int64_t consumerId = pSubscribe->consumerId; - char *consumerGroup = pSubscribe->consumerGroup; + SCMSubscribeReq subscribe; + tDeserializeSCMSubscribeReq(msgStr, &subscribe); + int64_t consumerId = subscribe.consumerId; + char *consumerGroup = subscribe.consumerGroup; int32_t cgroupLen = strlen(consumerGroup); SArray *newSub = NULL; - int newTopicNum = pSubscribe->topicNum; + int newTopicNum = subscribe.topicNum; if (newTopicNum) { newSub = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); } + SMqConsumerTopic *pConsumerTopics = calloc(newTopicNum, sizeof(SMqConsumerTopic)); + if (pConsumerTopics == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } for (int i = 0; i < newTopicNum; i++) { char *newTopicName = taosArrayGetP(newSub, i); - SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); - if (pConsumerTopic == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - // TODO: free - return -1; - } + SMqConsumerTopic *pConsumerTopic = &pConsumerTopics[i]; + strcpy(pConsumerTopic->name, newTopicName); pConsumerTopic->vgroups = tdListNew(sizeof(int64_t)); - taosArrayPush(newSub, pConsumerTopic); - free(pConsumerTopic); } + + taosArrayAddBatch(newSub, pConsumerTopics, newTopicNum); + free(pConsumerTopics); taosArraySortString(newSub, taosArrayCompareString); SArray *oldSub = NULL; int oldTopicNum = 0; + // create consumer if not exist SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); if (pConsumer == NULL) { // create consumer @@ -249,6 +252,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); if (pTrans == NULL) { + //TODO: free memory return -1; } @@ -286,6 +290,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } if (pOldTopic != NULL) { + //cancel subscribe of that old topic ASSERT(pNewTopic == NULL); char *oldTopicName = pOldTopic->name; SList *vgroups = pOldTopic->vgroups; @@ -298,13 +303,14 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); while ((pn = tdListNext(&iter)) != NULL) { int32_t vgId = *(int64_t *)pn->data; + // acquire and get epset SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); - // TODO release + // TODO what time to release? if (pVgObj == NULL) { // TODO handle error continue; } - // acquire and get epset + //build reset msg void *pMqVgSetReq = mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, consumerId, consumerGroup); // TODO:serialize if (pMsg == NULL) { @@ -323,10 +329,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } } + //delete data in mnode taosHashRemove(pTopic->cgroups, consumerGroup, cgroupLen); mndReleaseTopic(pMnode, pTopic); } else if (pNewTopic != NULL) { + // save subscribe info to mnode ASSERT(pOldTopic == NULL); char *newTopicName = pNewTopic->name; @@ -351,6 +359,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { // add into cgroups taosHashPut(pTopic->cgroups, consumerGroup, cgroupLen, pGroup, sizeof(SMqCGroup)); } + /*taosHashPut(pTopic->consumers, &pConsumer->consumerId, sizeof(int64_t), pConsumer, sizeof(SMqConsumerObj));*/ // put the consumer into list // rebalance will be triggered by timer diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 902eaa5c1c..3444a2409a 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -357,6 +357,11 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } } } + for (int i = 0; i < sz; i++) { + SClientHbReq* pHbReq = taosArrayGet(pArray, i); + tFreeClientHbReq(pHbReq); + } + taosArrayDestroy(pArray); int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); void* buf = rpcMallocCont(tlen); void* abuf = buf; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index d70c93e758..cab30702ea 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -69,6 +69,17 @@ static void mndTransReExecute(void *param, void *tmrId) { taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); } +static void mndCalMqRebalance(void* param, void* tmrId) { + SMnode* pMnode = param; + if (mndIsMaster(pMnode)) { + // iterate cgroup, cal rebalance + // sync with raft + // write sdb + } + + taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->transTimer); +} + static int32_t mndInitTimer(SMnode *pMnode) { if (pMnode->timer == NULL) { pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); From d14a599a32bda1bdad49e14b77c598a40d6531d5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Jan 2022 16:15:24 +0800 Subject: [PATCH 014/103] fix mem leak --- source/dnode/mnode/impl/src/mndProfile.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 3444a2409a..3773750ed3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -357,15 +357,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } } } - for (int i = 0; i < sz; i++) { - SClientHbReq* pHbReq = taosArrayGet(pArray, i); - tFreeClientHbReq(pHbReq); - } - taosArrayDestroy(pArray); + taosArrayDestroyEx(pArray, tFreeClientHbReq); + int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); void* buf = rpcMallocCont(tlen); void* abuf = buf; tSerializeSClientHbBatchRsp(&abuf, &batchRsp); + taosArrayDestroy(batchRsp.rsps); pReq->contLen = tlen; pReq->pCont = buf; return 0; From cc8a02aeb496bcb2901d3c9569f80c4f56265fb2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Jan 2022 16:17:58 +0800 Subject: [PATCH 015/103] fix mem leak --- source/libs/wal/src/walMeta.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index d630080086..a3894ceedd 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -149,6 +149,7 @@ int walCheckAndRepairMeta(SWal* pWal) { } } + closedir(dir); regfree(&logRegPattern); regfree(&idxRegPattern); From 623acecafdbfae6aa1471214040a001ea90b63fd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Jan 2022 16:24:54 +0800 Subject: [PATCH 016/103] init hb count --- source/client/src/clientHb.c | 3 +++ source/client/test/clientTests.cpp | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 97ef77abcc..0f4ff6f725 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -157,6 +157,9 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { } // init stat pAppHbMgr->startTime = taosGetTimestampMs(); + pAppHbMgr->connKeyCnt = 0; + pAppHbMgr->reportCnt = 0; + pAppHbMgr->reportBytes = 0; // init app info pAppHbMgr->pAppInstInfo = pAppInstInfo; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 13915fd85d..415d6a57ce 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,9 +53,7 @@ TEST(testCase, connect_Test) { if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } - while (1) { - sleep(3); - } + sleep(3); taos_close(pConn); } From 66f610f8df52cf8897452fb06da972066a369ad4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Jan 2022 16:34:09 +0800 Subject: [PATCH 017/103] make jenkins happy --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7da0f3826e..a2d6bbf4e6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -333,7 +333,7 @@ typedef struct SMqConsumerEp { typedef struct SMqCgroupTopicPair { char key[TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN]; - SArray* assigned; + SArray* assigned; // SArray SArray* unassignedConsumer; SArray* unassignedVg; } SMqCgroupTopicPair; From c4e0e94fc6f6e8a33739ffed6e066548941c8dd4 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Jan 2022 17:53:21 +0800 Subject: [PATCH 018/103] [add create table scriptes] --- tests/script/sh/massiveTable/cleanCluster.sh | 57 ++++++++ .../script/sh/massiveTable/compileVersion.sh | 81 +++++++++++ tests/script/sh/massiveTable/deployCluster.sh | 25 ++++ tests/script/sh/massiveTable/setupDnodes.sh | 136 ++++++++++++++++++ 4 files changed, 299 insertions(+) create mode 100644 tests/script/sh/massiveTable/cleanCluster.sh create mode 100644 tests/script/sh/massiveTable/compileVersion.sh create mode 100644 tests/script/sh/massiveTable/deployCluster.sh create mode 100644 tests/script/sh/massiveTable/setupDnodes.sh diff --git a/tests/script/sh/massiveTable/cleanCluster.sh b/tests/script/sh/massiveTable/cleanCluster.sh new file mode 100644 index 0000000000..af278933b2 --- /dev/null +++ b/tests/script/sh/massiveTable/cleanCluster.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# clean test environment + +set -e +#set -x + +# cleanCluster.sh +# -r [ dnode root dir] + + +dataRootDir="/data" + + +while getopts "hr:" arg +do + case $arg in + r) + dataRootDir=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -r [ dnode root dir] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + + +rmDnodesDataDir() { + if [ -d ${dataRootDir} ]; then + rm -rf ${dataRootDir}/dnode* + else + echo "${dataRootDir} not exist" + exit 1 + fi +} + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + kill -9 $pid || : + fi +} + +######################################################################################## +############################### main process ########################################## + +## kill all taosd process +kill_process taosd + +rmDnodesDataDir + + diff --git a/tests/script/sh/massiveTable/compileVersion.sh b/tests/script/sh/massiveTable/compileVersion.sh new file mode 100644 index 0000000000..1976e8f14a --- /dev/null +++ b/tests/script/sh/massiveTable/compileVersion.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# compile test version + +set -e +#set -x + +# compileVersion.sh +# -r [ TDengine project dir] +# -v [ TDengine branch version ] + + +projectDir=/root/TDengine +TDengineBrVer="3.0" + +while getopts "hr:v:" arg +do + case $arg in + r) + projectDir=$(echo $OPTARG) + ;; + v) + TDengineBrVer=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -r [ TDengine project dir] " + echo " -v [ TDengine branch version] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "projectDir=${projectDir} TDengineBrVer=${TDengineBrVer}" + +function gitPullBranchInfo () { + branch_name=$1 + + git checkout $branch_name + echo "==== git pull $branch_name start ====" +## git submodule update --init --recursive + git pull origin $branch_name ||: + echo "==== git pull $branch_name end ====" +} + +function compileTDengineVersion() { + debugDir=debug + if [ -d ${debugDir} ]; then + rm -rf ${debugDir}/* ||: + else + mkdir -p ${debugDir} + fi + + cd ${debugDir} + cmake .. + make -j24 +} +######################################################################################## +############################### main process ########################################## + +## checkout all branchs and git pull +cd ${projectDir} +gitPullBranchInfo $TDengineBrVer +compileTDengineVersion + +taos_dir=${projectDir}/debug/tools/shell +taosd_dir=${projectDir}/debug/source/dnode/mgmt/daemon +create_table_dir=${projectDir}/debug/tests/test/c + +rm -f /usr/bin/taos +rm -f /usr/bin/taosd +rm -f /usr/bin/create_table + +ln -s $taos_dir/taos /usr/bin/taos +ln -s $taosd_dir/taosd /usr/bin/taosd +ln -s $create_table_dir/create_table /usr/bin/create_table + + diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh new file mode 100644 index 0000000000..47802ea6a7 --- /dev/null +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +# deployCluster.sh + +curr_dir=$(pwd) + +source ./cleanCluster.sh -r /data +source ./cleanCluster.sh -r /data2 + +source ./compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" + +source ./setupDnodes.sh -r /data -n 1 -f trd02:7000 -p 7000 +source ./setupDnodes.sh -r /data2 -n 1 -f trd02:7000 -p 8000 + +#source ./setupDnodes.sh -r /data -n 2 -f trd02:7000 -p 7000 +#source ./setupDnodes.sh -r /data2 -n 2 -f trd02:7000 -p 8000 + + + + diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh new file mode 100644 index 0000000000..034c15c4eb --- /dev/null +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# setup test environment + +set -e +#set -x + +# setupDnodes.sh +# -e [ new | old] +# -n [ dnode number] +# -f [ first ep] +# -p [ start port] +# -r [ dnode root dir] + +# set parameters by default value +enviMode=new +dataRootDir="/data" +firstEp="localhost:7000" +startPort=7000 +dnodeNumber=1 + + +while getopts "he:f:n:r:p:" arg +do + case $arg in + e) + enviMode=$( echo $OPTARG ) + ;; + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + firstEp=$(echo $OPTARG) + ;; + p) + startPort=$(echo $OPTARG) + ;; + r) + dataRootDir=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -e [new | old] " + echo " -n [ dnode number] " + echo " -f [ first ep] " + echo " -p [ start port] " + echo " -r [ dnode root dir] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "enviMode=${enviMode} dnodeNumber=${dnodeNumber} dataRootDir=${dataRootDir} firstEp=${firstEp} startPort=${startPort}" + +#curr_dir=$(pwd) + + +createNewCfgFile() { + cfgFile=$1/taos.cfg + dataDir=$2 + logDir=$3 + firstEp=$4 + serverPort=$5 + + echo "debugFlag 131" > ${cfgFile} + echo "firstEp ${firstEp}" >> ${cfgFile} + echo "dataDir ${dataDir}" >> ${cfgFile} + echo "logDir ${logDir}" >> ${cfgFile} + echo "serverPort ${serverPort}" >> ${cfgFile} + + echo "supportVnodes 1024" >> ${cfgFile} + #echo "asyncLog 0" >> ${cfgFile} + echo "telemetryReporting 0" >> ${cfgFile} +} + +createNewDnodesDataDir() { + if [ -d ${dataRootDir} ]; then + rm -rf ${dataRootDir}/dnode* + else + echo "${dataRootDir} not exist" + exit 1 + fi + + dnodeNumber=$1 + firstEp=$2 + + serverPort=${startPort} + for ((i=0; i<${dnodeNumber}; i++)); do + mkdir -p ${dataRootDir}/dnode_${i}/cfg + mkdir -p ${dataRootDir}/dnode_${i}/log + mkdir -p ${dataRootDir}/dnode_${i}/data + + createNewCfgFile ${dataRootDir}/dnode_${i}/cfg ${dataRootDir}/dnode_${i}/data ${dataRootDir}/dnode_${i}/log ${firstEp} ${serverPort} + echo "create dnode: ${serverPort}, ${dataRootDir}/dnode_${i}" + serverPort=$((10#${serverPort}+100)) + done +} + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + kill -9 $pid || : + fi +} + +startDnodes() { + dnodeNumber=$1 + + for ((i=0; i<${dnodeNumber}; i++)); do + if [ -d ${dataRootDir}/dnode_${i} ]; then + nohup taosd -c ${dataRootDir}/dnode_${i}/cfg >/dev/null 2>&1 & + echo "start taosd ${dataRootDir}/dnode_${i}" + fi + done +} + +######################################################################################## +############################### main process ########################################## + +## kill all taosd process +kill_process taosd + +## create director for all dnode +if [[ "$enviMode" == "new" ]]; then + createNewDnodesDataDir ${dnodeNumber} ${firstEp} +fi + +## start all dnode by nohup +startDnodes ${dnodeNumber} + +echo " run setupDnodes.sh end !!!" + + From 45f7b12f83d08d0825aae66890e5d46fdf77910f Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Jan 2022 17:59:20 +0800 Subject: [PATCH 019/103] [modify] --- tests/script/sh/massiveTable/cleanCluster.sh | 0 tests/script/sh/massiveTable/compileVersion.sh | 0 tests/script/sh/massiveTable/deployCluster.sh | 0 tests/script/sh/massiveTable/setupDnodes.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/script/sh/massiveTable/cleanCluster.sh mode change 100644 => 100755 tests/script/sh/massiveTable/compileVersion.sh mode change 100644 => 100755 tests/script/sh/massiveTable/deployCluster.sh mode change 100644 => 100755 tests/script/sh/massiveTable/setupDnodes.sh diff --git a/tests/script/sh/massiveTable/cleanCluster.sh b/tests/script/sh/massiveTable/cleanCluster.sh old mode 100644 new mode 100755 diff --git a/tests/script/sh/massiveTable/compileVersion.sh b/tests/script/sh/massiveTable/compileVersion.sh old mode 100644 new mode 100755 diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh old mode 100644 new mode 100755 diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh old mode 100644 new mode 100755 From 0b2aea0f3519b519161fe8f420cafe786c936914 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Jan 2022 18:01:21 +0800 Subject: [PATCH 020/103] add client --- source/libs/transport/inc/transComm.h | 120 +++++ source/libs/transport/inc/transportInt.h | 88 ++- source/libs/transport/src/trans.c | 64 +++ source/libs/transport/src/transCli.c | 180 +++++++ source/libs/transport/src/transComm.c | 117 ++++ source/libs/transport/src/transSrv.c | 540 +++++++++++++++++++ source/libs/transport/test/transportTests.cc | 3 +- 7 files changed, 1066 insertions(+), 46 deletions(-) create mode 100644 source/libs/transport/inc/transComm.h create mode 100644 source/libs/transport/src/trans.c create mode 100644 source/libs/transport/src/transCli.c create mode 100644 source/libs/transport/src/transComm.c create mode 100644 source/libs/transport/src/transSrv.c diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h new file mode 100644 index 0000000000..792200639a --- /dev/null +++ b/source/libs/transport/inc/transComm.h @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifdef USE_UV + +#include +#include "lz4.h" +#include "os.h" +#include "rpcCache.h" +#include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tidpool.h" +#include "tmd5.h" +#include "tmempool.h" +#include "tmsg.h" +#include "transportInt.h" +#include "tref.h" +#include "trpc.h" +#include "ttimer.h" +#include "tutil.h" + +typedef void* queue[2]; +/* Private macros. */ +#define QUEUE_NEXT(q) (*(queue**)&((*(q))[0])) +#define QUEUE_PREV(q) (*(queue**)&((*(q))[1])) + +#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q))) +#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) +/* Initialize an empty queue. */ +#define QUEUE_INIT(q) \ + { \ + QUEUE_NEXT(q) = (q); \ + QUEUE_PREV(q) = (q); \ + } + +/* Return true if the queue has no element. */ +#define QUEUE_IS_EMPTY(q) ((const queue*)(q) == (const queue*)QUEUE_NEXT(q)) + +/* Insert an element at the back of a queue. */ +#define QUEUE_PUSH(q, e) \ + { \ + QUEUE_NEXT(e) = (q); \ + QUEUE_PREV(e) = QUEUE_PREV(q); \ + QUEUE_PREV_NEXT(e) = (e); \ + QUEUE_PREV(q) = (e); \ + } + +/* Remove the given element from the queue. Any element can be removed at any * + * time. */ +#define QUEUE_REMOVE(e) \ + { \ + QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \ + QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \ + } + +/* Return the element at the front of the queue. */ +#define QUEUE_HEAD(q) (QUEUE_NEXT(q)) + +/* Return the element at the back of the queue. */ +#define QUEUE_TAIL(q) (QUEUE_PREV(q)) + +/* Iterate over the element of a queue. * Mutating the queue while iterating + * results in undefined behavior. */ +#define QUEUE_FOREACH(q, e) for ((q) = QUEUE_NEXT(e); (q) != (e); (q) = QUEUE_NEXT(q)) + +/* Return the structure holding the given element. */ +#define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) + +typedef struct { + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + SEpSet* pSet; // for synchronous API + char msg[0]; // RpcHead starts from here +} SRpcReqContext; + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) + +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) +#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) +#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) +#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) +#define rpcIsReq(type) (type & 1U) + +int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); + +#endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index f93753cfe9..0addea5d1a 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,62 +16,60 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#include +#include "lz4.h" +#include "os.h" +#include "rpcCache.h" #include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tidpool.h" +#include "tmsg.h" +#include "transportInt.h" +#include "tref.h" +#include "trpc.h" +#include "ttimer.h" +#include "tutil.h" + #ifdef __cplusplus extern "C" { #endif #ifdef USE_UV -#include -typedef void *queue[2]; +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); -/* Private macros. */ -#define QUEUE_NEXT(q) (*(queue **)&((*(q))[0])) -#define QUEUE_PREV(q) (*(queue **)&((*(q))[1])) +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + int64_t index; + char label[TSDB_LABEL_LEN]; -#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q))) -#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key -/* Initialize an empty queue. */ -#define QUEUE_INIT(q) \ - { \ - QUEUE_NEXT(q) = (q); \ - QUEUE_PREV(q) = (q); \ - } + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); -/* Return true if the queue has no element. */ -#define QUEUE_IS_EMPTY(q) ((const queue *)(q) == (const queue *)QUEUE_NEXT(q)) - -/* Insert an element at the back of a queue. */ -#define QUEUE_PUSH(q, e) \ - { \ - QUEUE_NEXT(e) = (q); \ - QUEUE_PREV(e) = QUEUE_PREV(q); \ - QUEUE_PREV_NEXT(e) = (e); \ - QUEUE_PREV(q) = (e); \ - } - -/* Remove the given element from the queue. Any element can be removed at any * - * time. */ -#define QUEUE_REMOVE(e) \ - { \ - QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \ - QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \ - } - -/* Return the element at the front of the queue. */ -#define QUEUE_HEAD(q) (QUEUE_NEXT(q)) - -/* Return the element at the back of the queue. */ -#define QUEUE_TAIL(q) (QUEUE_PREV(q)) - -/* Iterate over the element of a queue. * Mutating the queue while iterating - * results in undefined behavior. */ -#define QUEUE_FOREACH(q, e) for ((q) = QUEUE_NEXT(e); (q) != (e); (q) = QUEUE_NEXT(q)) - -/* Return the structure holding the given element. */ -#define QUEUE_DATA(e, type, field) ((type *)((void *)((char *)(e)-offsetof(type, field)))) + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization + pthread_mutex_t mutex; +} SRpcInfo; #endif // USE_LIBUV diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c new file mode 100644 index 0000000000..a59f2f88c7 --- /dev/null +++ b/source/libs/transport/src/trans.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef USE_UV + +#include "transComm.h" + +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; +} SConnBuffer; + +void* (*taosHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { + taosInitServer, taosInitClient}; + +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + return NULL; + } + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + pRpc->connType = pInit->connType; + pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + + return pRpc; +} +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + +int32_t rpcInit(void) { + // impl later + return -1; +} + +void rpcCleanup(void) { + // impl later + return; +} +#endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c new file mode 100644 index 0000000000..f848932c0f --- /dev/null +++ b/source/libs/transport/src/transCli.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef USE_UV + +#include "transComm.h" + +typedef struct SCliConn { + uv_connect_t connReq; + uv_stream_t* stream; + void* data; + queue conn; +} SCliConn; +typedef struct SCliMsg { + SRpcReqContext* context; + queue q; +} SCliMsg; + +typedef struct SCliThrdObj { + pthread_t thread; + uv_loop_t* loop; + uv_async_t* cliAsync; // + void* cache; // conn pool + queue msg; + pthread_mutex_t msgMtx; + void* shandle; +} SCliThrdObj; + +typedef struct SClientObj { + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SCliThrdObj** pThreadObj; +} SClientObj; + +static void clientWriteCb(uv_write_t* req, int status); +static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void clientConnCb(struct uv_connect_s* req, int status); +static void clientAsyncCb(uv_async_t* handle); + +static void* clientThread(void* arg); + +static void clientWriteCb(uv_write_t* req, int status) { + // impl later +} +static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // impl later +} +static void clientConnCb(struct uv_connect_s* req, int status) { + // impl later +} + +static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { + // impl later + return NULL; +} +static void clientAsyncCb(uv_async_t* handle) { + SCliThrdObj* pThrd = handle->data; + SCliMsg* pMsg = NULL; + pthread_mutex_lock(&pThrd->msgMtx); + if (!QUEUE_IS_EMPTY(&pThrd->msg)) { + queue* head = QUEUE_HEAD(&pThrd->msg); + pMsg = QUEUE_DATA(head, SCliMsg, q); + QUEUE_REMOVE(head); + } + pthread_mutex_unlock(&pThrd->msgMtx); + + SEpSet* pEpSet = &pMsg->context->epSet; + char* fqdn = pEpSet->fqdn[pEpSet->inUse]; + uint32_t port = pEpSet->port[pEpSet->inUse]; + + SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); + if (conn != NULL) { + } else { + SCliConn* conn = malloc(sizeof(SCliConn)); + + conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); + + conn->connReq.data = conn; + conn->data = pMsg; + + struct sockaddr_in addr; + uv_ip4_addr(fqdn, port, &addr); + // handle error in callback if connect error + uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); + } + + // SRpcReqContext* pCxt = pMsg->context; + + // SRpcHead* pHead = rpcHeadFromCont(pCtx->pCont); + // char* msg = (char*)pHead; + // int len = rpcMsgLenFromCont(pCtx->contLen); + // tmsg_t msgType = pCtx->msgType; + + // impl later +} + +static void* clientThread(void* arg) { + SCliThrdObj* pThrd = (SCliThrdObj*)arg; + + QUEUE_INIT(&pThrd->msg); + pthread_mutex_init(&pThrd->msgMtx, NULL); + + // QUEUE_INIT(&pThrd->clientCache); + + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); + + pThrd->cliAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); + pThrd->cliAsync->data = pThrd; + + uv_run(pThrd->loop, UV_RUN_DEFAULT); +} + +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + SClientObj* cli = calloc(1, sizeof(SClientObj)); + memcpy(cli->label, label, strlen(label)); + cli->numOfThreads = numOfThreads; + cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + + for (int i = 0; i < cli->numOfThreads; i++) { + SCliThrdObj* thrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + + thrd->shandle = shandle; + int err = pthread_create(&thrd->thread, NULL, clientThread, (void*)(thrd)); + if (err == 0) { + tDebug("sucess to create tranport-client thread %d", i); + } + cli->pThreadObj[i] = thrd; + } + return cli; +} + +void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { + // impl later + SRpcInfo* pRpc = (SRpcInfo*)shandle; + + SRpcReqContext* pContext; + int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); + pContext = (SRpcReqContext*)((char*)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); + pContext->ahandle = pMsg->ahandle; + pContext->pRpc = (SRpcInfo*)shandle; + pContext->epSet = *pEpSet; + pContext->contLen = contLen; + pContext->pCont = pMsg->pCont; + pContext->msgType = pMsg->msgType; + pContext->oldInUse = pEpSet->inUse; + + assert(pRpc->connType == TAOS_CONN_CLIENT); + // atomic or not + int64_t index = pRpc->index; + if (pRpc->index++ >= pRpc->numOfThreads) { + pRpc->index = 0; + } + SCliMsg* msg = malloc(sizeof(SCliMsg)); + msg->context = pContext; + + SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; + + pthread_mutex_lock(&thrd->msgMtx); + QUEUE_PUSH(&thrd->msg, &msg->q); + pthread_mutex_unlock(&thrd->msgMtx); + + uv_async_send(thrd->cliAsync); +} +#endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c new file mode 100644 index 0000000000..f23cfb6e2d --- /dev/null +++ b/source/libs/transport/src/transComm.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifdef USE_UV + +#include "transComm.h" + +int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + int ret = -1; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; + + return ret; +} +void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + memcpy(pAuth, context.digest, sizeof(context.digest)); +} + +int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { + SRpcHead* pHead = rpcHeadFromCont(pCont); + int32_t finalLen = 0; + int overhead = sizeof(SRpcComp); + + if (!NEEDTO_COMPRESSS_MSG(contLen)) { + return contLen; + } + + char* buf = malloc(contLen + overhead + 8); // 8 extra bytes + if (buf == NULL) { + tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); + return contLen; + } + + int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); + tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); + + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (compLen > 0 && compLen < contLen - overhead) { + SRpcComp* pComp = (SRpcComp*)pCont; + pComp->reserved = 0; + pComp->contLen = htonl(contLen); + memcpy(pCont + overhead, buf, compLen); + + pHead->comp = 1; + tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); + finalLen = compLen + overhead; + } else { + finalLen = contLen; + } + + free(buf); + return finalLen; +} + +SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { + int overhead = sizeof(SRpcComp); + SRpcHead* pNewHead = NULL; + uint8_t* pCont = pHead->content; + SRpcComp* pComp = (SRpcComp*)pHead->content; + + if (pHead->comp) { + // decompress the content + assert(pComp->reserved == 0); + int contLen = htonl(pComp->contLen); + + // prepare the temporary buffer to decompress message + char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD); + pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext + + if (pNewHead) { + int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; + int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen); + assert(origLen == contLen); + + memcpy(pNewHead, pHead, sizeof(SRpcHead)); + pNewHead->msgLen = rpcMsgLenFromCont(origLen); + /// rpcFreeMsg(pHead); // free the compressed message buffer + pHead = pNewHead; + tTrace("decomp malloc mem:%p", temp); + } else { + tError("failed to allocate memory to decompress msg, contLen:%d", contLen); + } + } + + return pHead; +} + +#endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c new file mode 100644 index 0000000000..0bf39b9985 --- /dev/null +++ b/source/libs/transport/src/transSrv.c @@ -0,0 +1,540 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef USE_UV +#include "transComm.h" + +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; +} SConnBuffer; + +typedef struct SConn { + uv_tcp_t* pTcp; + uv_write_t* pWriter; + uv_timer_t* pTimer; + + uv_async_t* pWorkerAsync; + queue queue; + int ref; + int persist; // persist connection or not + SConnBuffer connBuf; // read buf, + SConnBuffer writeBuf; // write buf + int count; + void* shandle; // rpc init + void* ahandle; // + void* hostThrd; + // del later + char secured; + int spi; + char info[64]; + char user[TSDB_UNI_LEN]; // user ID for the link + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; // ciphering key +} SConn; + +typedef struct SWorkThrdObj { + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + uv_async_t* workerAsync; // + queue conn; + pthread_mutex_t connMtx; + void* shandle; +} SWorkThrdObj; + +typedef struct SServerObj { + pthread_t thread; + uv_tcp_t server; + uv_loop_t* loop; + int workerIdx; + int numOfThreads; + SWorkThrdObj** pThreadObj; + uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; +} SServerObj; + +static const char* notify = "a"; + +// refactor later +static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen); + +static int uvAuthMsg(SConn* pConn, char* msg, int msgLen); + +static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void uvOnTimeoutCb(uv_timer_t* handle); +static void uvOnWriteCb(uv_write_t* req, int status); +static void uvOnAcceptCb(uv_stream_t* stream, int status); +static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void uvWorkerAsyncCb(uv_async_t* handle); + +// already read complete packet +static bool readComplete(SConnBuffer* buf); + +static SConn* connCreate(); +static void connDestroy(SConn* conn); +static void uvConnDestroy(uv_handle_t* handle); + +// server worke thread +static void* workerThread(void* arg); +static void* acceptThread(void* arg); + +void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + /* + * formate of data buffer: + * |<-------SRpcReqContext------->|<------------data read from socket----------->| + */ + static const int CAPACITY = 1024; + + SConn* conn = handle->data; + SConnBuffer* pBuf = &conn->connBuf; + if (pBuf->cap == 0) { + pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); + pBuf->len = 0; + pBuf->cap = CAPACITY; + pBuf->left = -1; + + buf->base = pBuf->buf + RPC_RESERVE_SIZE; + buf->len = CAPACITY; + } else { + if (pBuf->len >= pBuf->cap) { + if (pBuf->left == -1) { + pBuf->cap *= 2; + pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); + } else if (pBuf->len + pBuf->left > pBuf->cap) { + pBuf->cap = pBuf->len + pBuf->left; + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); + } + } + buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; + buf->len = pBuf->cap - pBuf->len; + } +} + +// check data read from socket completely or not +// +static bool readComplete(SConnBuffer* data) { + // TODO(yihao): handle pipeline later + SRpcHead rpcHead; + int32_t headLen = sizeof(rpcHead); + if (data->len >= headLen) { + memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + if (msgLen > data->len) { + data->left = msgLen - data->len; + return false; + } else { + return true; + } + } else { + return false; + } +} + +static void uvDoProcess(SRecvInfo* pRecv) { + SRpcHead* pHead = (SRpcHead*)pRecv->msg; + SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; + SConn* pConn = pRecv->thandle; + + tDump(pRecv->msg, pRecv->msgLen); + + terrno = 0; + SRpcReqContext* pContest; + + // do auth and check +} + +static int uvAuthMsg(SConn* pConn, char* msg, int len) { + SRpcHead* pHead = (SRpcHead*)msg; + int code = 0; + + if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { + // secured link, or no authentication + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, secured link, no auth is required", pConn->info); + return 0; + } + + if (!rpcIsReq(pHead->msgType)) { + // for response, if code is auth failure, it shall bypass the auth process + code = htonl(pHead->code); + if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || + code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + return 0; + } + } + + code = 0; + if (pHead->spi == pConn->spi) { + // authentication + SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); + + int32_t delta; + delta = (int32_t)htonl(pDigest->timeStamp); + delta -= (int32_t)taosGetTimestampSec(); + if (abs(delta) > 900) { + tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); + code = TSDB_CODE_RPC_INVALID_TIME_STAMP; + } else { + if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + // tDebug("%s, authentication failed, msg discarded", pConn->info); + code = TSDB_CODE_RPC_AUTH_FAILURE; + } else { + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); + if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client + // tTrace("%s, message is authenticated", pConn->info); + } + } + } else { + tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); + code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; + } + + return code; +} + +// refers specifically to query or insert timeout +static void uvHandleActivityTimeout(uv_timer_t* handle) { + // impl later + SConn* conn = handle->data; +} + +static void uvProcessData(SConn* pConn) { + SRecvInfo info; + SRecvInfo* p = &info; + SConnBuffer* pBuf = &pConn->connBuf; + p->msg = pBuf->buf + RPC_RESERVE_SIZE; + p->msgLen = pBuf->len; + p->ip = 0; + p->port = 0; + p->shandle = pConn->shandle; // + p->thandle = pConn; + p->chandle = NULL; + + // + SRpcHead* pHead = (SRpcHead*)p->msg; + assert(rpcIsReq(pHead->msgType)); + + SRpcInfo* pRpc = (SRpcInfo*)p->shandle; + pConn->ahandle = (void*)pHead->ahandle; + // auth here + + int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); + if (code != 0) { + terrno = code; + return; + } + pHead->code = htonl(pHead->code); + + SRpcMsg rpcMsg; + + pHead = rpcDecompressRpcMsg(pHead); + rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); + rpcMsg.pCont = pHead->content; + rpcMsg.msgType = pHead->msgType; + rpcMsg.code = pHead->code; + rpcMsg.ahandle = pConn->ahandle; + rpcMsg.handle = pConn; + + (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); + uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0); + // auth + // validate msg type +} + +void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + SConn* ctx = cli->data; + SConnBuffer* pBuf = &ctx->connBuf; + if (nread > 0) { + pBuf->len += nread; + if (readComplete(pBuf)) { + tDebug("alread read complete packet"); + uvProcessData(ctx); + } else { + tDebug("read half packet, continue to read"); + } + return; + } + if (terrno != 0) { + // handle err code + } + + if (nread != UV_EOF) { + tDebug("Read error %s\n", uv_err_name(nread)); + } + uv_close((uv_handle_t*)cli, uvConnDestroy); +} +void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(sizeof(char)); + buf->len = 2; +} + +void uvOnTimeoutCb(uv_timer_t* handle) { + // opt + tDebug("time out"); +} + +void uvOnWriteCb(uv_write_t* req, int status) { + SConn* conn = req->data; + if (status == 0) { + tDebug("data already was written on stream"); + } else { + connDestroy(conn); + } + // opt +} + +void uvWorkerAsyncCb(uv_async_t* handle) { + SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync); + SConn* conn = NULL; + + // opt later + pthread_mutex_lock(&pThrd->connMtx); + if (!QUEUE_IS_EMPTY(&pThrd->conn)) { + queue* head = QUEUE_HEAD(&pThrd->conn); + conn = QUEUE_DATA(head, SConn, queue); + QUEUE_REMOVE(head); + } + pthread_mutex_unlock(&pThrd->connMtx); + if (conn == NULL) { + tError("except occurred, do nothing"); + return; + } + uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); + uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); +} + +void uvOnAcceptCb(uv_stream_t* stream, int status) { + if (status == -1) { + return; + } + SServerObj* pObj = container_of(stream, SServerObj, server); + + uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, cli); + + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + + uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); + + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThreads; + tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); + } else { + uv_close((uv_handle_t*)cli, NULL); + } +} +void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + tDebug("connection coming"); + if (nread < 0) { + if (nread != UV_EOF) { + tError("read error %s", uv_err_name(nread)); + } + // TODO(log other failure reason) + uv_close((uv_handle_t*)q, NULL); + return; + } + // free memory allocated by + assert(nread == strlen(notify)); + assert(buf->base[0] == notify[0]); + free(buf->base); + + SWorkThrdObj* pThrd = q->data; + + uv_pipe_t* pipe = (uv_pipe_t*)q; + if (!uv_pipe_pending_count(pipe)) { + tError("No pending count"); + return; + } + + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SConn* pConn = connCreate(); + pConn->shandle = pThrd->shandle; + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pConn->pTimer); + pConn->pTimer->data = pConn; + + pConn->hostThrd = pThrd; + pConn->pWorkerAsync = pThrd->workerAsync; // thread safty + + // init client handle + pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pThrd->loop, pConn->pTcp); + pConn->pTcp->data = pConn; + + // init write request, just + pConn->pWriter = calloc(1, sizeof(uv_write_t)); + pConn->pWriter->data = pConn; + + if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); + tDebug("new connection created: %d", fd); + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } else { + connDestroy(pConn); + } +} + +void* acceptThread(void* arg) { + // opt + SServerObj* srv = (SServerObj*)arg; + uv_tcp_init(srv->loop, &srv->server); + + struct sockaddr_in bind_addr; + + uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); +} +void* workerThread(void* arg) { + SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; + + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); + + // SRpcInfo* pRpc = pThrd->shandle; + uv_pipe_init(pThrd->loop, pThrd->pipe, 1); + uv_pipe_open(pThrd->pipe, pThrd->fd); + + pThrd->pipe->data = pThrd; + + QUEUE_INIT(&pThrd->conn); + pthread_mutex_init(&pThrd->connMtx, NULL); + + pThrd->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); + + uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + uv_run(pThrd->loop, UV_RUN_DEFAULT); +} + +static SConn* connCreate() { + SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); + return pConn; +} +static void connDestroy(SConn* conn) { + if (conn == NULL) { + return; + } + uv_timer_stop(conn->pTimer); + free(conn->pTimer); + uv_close((uv_handle_t*)conn->pTcp, NULL); + free(conn->connBuf.buf); + free(conn->pTcp); + free(conn->pWriter); + free(conn); + // handle +} +static void uvConnDestroy(uv_handle_t* handle) { + SConn* conn = handle->data; + connDestroy(conn); +} +static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen) { + SRpcHead* pHead = (SRpcHead*)msg; + + if (pConn->spi && pConn->secured == 0) { + // add auth part + pHead->spi = pConn->spi; + SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen); + pDigest->timeStamp = htonl(taosGetTimestampSec()); + msgLen += sizeof(SRpcDigest); + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + } else { + pHead->spi = 0; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + } + + return msgLen; +} + +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + SServerObj* srv = calloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + srv->numOfThreads = numOfThreads; + srv->workerIdx = 0; + srv->pThreadObj = (SWorkThrdObj**)calloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pipe = (uv_pipe_t**)calloc(srv->numOfThreads, sizeof(uv_pipe_t*)); + srv->ip = ip; + srv->port = port; + uv_loop_init(srv->loop); + + for (int i = 0; i < srv->numOfThreads; i++) { + SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + return NULL; + } + uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); + uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write + + thrd->shandle = shandle; + thrd->fd = fds[0]; + thrd->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); + if (err == 0) { + tDebug("sucess to create worker-thread %d", i); + // printf("thread %d create\n", i); + } else { + // TODO: clear all other resource later + tError("failed to create worker-thread %d", i); + } + srv->pThreadObj[i] = thrd; + } + + int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + if (err == 0) { + tDebug("success to create accept-thread"); + } else { + // clear all resource later + } + + return srv; +} + +void rpcSendResponse(const SRpcMsg* pMsg) { + SConn* pConn = pMsg->handle; + SWorkThrdObj* pThrd = pConn->hostThrd; + + // opt later + pthread_mutex_lock(&pThrd->connMtx); + QUEUE_PUSH(&pThrd->conn, &pConn->queue); + pthread_mutex_unlock(&pThrd->connMtx); + + uv_async_send(pConn->pWorkerAsync); +} + +#endif diff --git a/source/libs/transport/test/transportTests.cc b/source/libs/transport/test/transportTests.cc index 151deaf29b..53910aa30c 100644 --- a/source/libs/transport/test/transportTests.cc +++ b/source/libs/transport/test/transportTests.cc @@ -22,6 +22,7 @@ #include #include +#include "transComm.h" #include "transportInt.h" #include "trpc.h" @@ -46,7 +47,7 @@ class QueueObj { if (!IsEmpty()) { queue *h = QUEUE_HEAD(&head); el = QUEUE_DATA(h, QueueElem, q); - QUEUE_REMOVE(&el->q); + QUEUE_REMOVE(h); } return el; } From caeb8ae159a6a8180bb3fed9e6cf5dbeba919b69 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Jan 2022 18:14:43 +0800 Subject: [PATCH 021/103] add client --- source/libs/transport/inc/transportInt.h | 3 ++- source/libs/transport/src/transCli.c | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 0addea5d1a..e39e0d9273 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,7 +16,9 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#ifdef USE_UV #include +#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" @@ -29,7 +31,6 @@ #include "thash.h" #include "tidpool.h" #include "tmsg.h" -#include "transportInt.h" #include "tref.h" #include "trpc.h" #include "ttimer.h" diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f848932c0f..bd54e8e57b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -149,13 +149,14 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* // impl later SRpcInfo* pRpc = (SRpcInfo*)shandle; + int len = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); + SRpcReqContext* pContext; - int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); pContext = (SRpcReqContext*)((char*)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); pContext->ahandle = pMsg->ahandle; pContext->pRpc = (SRpcInfo*)shandle; pContext->epSet = *pEpSet; - pContext->contLen = contLen; + pContext->contLen = len; pContext->pCont = pMsg->pCont; pContext->msgType = pMsg->msgType; pContext->oldInUse = pEpSet->inUse; From ea57fba331c99db40d9cc3dc360d659105fb6cdc Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Jan 2022 18:22:13 +0800 Subject: [PATCH 022/103] [modify] --- tests/script/sh/massiveTable/deployCluster.sh | 15 ++++++++------- tests/script/sh/massiveTable/setupDnodes.sh | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index 47802ea6a7..2341d512c0 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -8,17 +8,18 @@ set -e # deployCluster.sh curr_dir=$(pwd) +echo "currect pwd: ${curr_dir}" -source ./cleanCluster.sh -r /data -source ./cleanCluster.sh -r /data2 +./cleanCluster.sh -r "/data" +./cleanCluster.sh -r "/data2" -source ./compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" +./compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" -source ./setupDnodes.sh -r /data -n 1 -f trd02:7000 -p 7000 -source ./setupDnodes.sh -r /data2 -n 1 -f trd02:7000 -p 8000 +./setupDnodes.sh -r "/data" -n 1 -f "trd02:7000" -p 7000 +./setupDnodes.sh -r "/data2" -n 1 -f "trd02:7000" -p 8000 -#source ./setupDnodes.sh -r /data -n 2 -f trd02:7000 -p 7000 -#source ./setupDnodes.sh -r /data2 -n 2 -f trd02:7000 -p 8000 +#./setupDnodes.sh -r "/data" -n 2 -f trd02:7000 -p 7000 +#./setupDnodes.sh -r "/data2" -n 2 -f trd02:7000 -p 8000 diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index 034c15c4eb..e09b2fb396 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -94,7 +94,7 @@ createNewDnodesDataDir() { mkdir -p ${dataRootDir}/dnode_${i}/data createNewCfgFile ${dataRootDir}/dnode_${i}/cfg ${dataRootDir}/dnode_${i}/data ${dataRootDir}/dnode_${i}/log ${firstEp} ${serverPort} - echo "create dnode: ${serverPort}, ${dataRootDir}/dnode_${i}" + #echo "create dnode: ${serverPort}, ${dataRootDir}/dnode_${i}" serverPort=$((10#${serverPort}+100)) done } @@ -131,6 +131,7 @@ fi ## start all dnode by nohup startDnodes ${dnodeNumber} -echo " run setupDnodes.sh end !!!" +echo "====run setupDnodes.sh end====" +echo " " From 04e3df71d35dd855c38cdee54ce8d0bdd1bbb4c4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Jan 2022 18:39:10 +0800 Subject: [PATCH 023/103] add client --- source/libs/transport/src/transCli.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bd54e8e57b..8ba1f1a0f6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -59,6 +59,18 @@ static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // impl later } static void clientConnCb(struct uv_connect_s* req, int status) { + SCliConn* pConn = req->data; + SCliMsg* pMsg = pConn->data; + SEpSet* pEpSet = &pMsg->context->epSet; + + char* fqdn = pEpSet->fqdn[pEpSet->inUse]; + uint32_t port = pEpSet->port[pEpSet->inUse]; + if (status != 0) { + // call user fp later + tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status)); + return; + } + // impl later } @@ -83,6 +95,7 @@ static void clientAsyncCb(uv_async_t* handle) { SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); if (conn != NULL) { + // impl later } else { SCliConn* conn = malloc(sizeof(SCliConn)); From 6d9ae1af999861f771aca6ad78f669f741cf173c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Jan 2022 19:08:07 +0800 Subject: [PATCH 024/103] [td-11818]Support select * from super_table. --- source/client/src/clientImpl.c | 17 +- source/client/test/clientTests.cpp | 192 ++++++++++----------- source/dnode/vnode/inc/meta.h | 2 +- source/dnode/vnode/inc/tsdb.h | 19 ++ source/dnode/vnode/src/meta/metaBDBImpl.c | 14 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 42 ++--- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorMain.c | 50 +++++- source/libs/executor/src/executorimpl.c | 13 +- source/libs/planner/src/physicalPlanJson.c | 28 ++- source/libs/qworker/CMakeLists.txt | 16 -- source/libs/scheduler/inc/schedulerInt.h | 2 +- source/libs/scheduler/src/scheduler.c | 20 ++- 13 files changed, 240 insertions(+), 177 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 51f267e884..df0f060fbe 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -257,7 +257,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return pRequest->code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); + SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr)); + + SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 0}; + addr.epAddr[0].port = 6030; + strcpy(addr.epAddr[0].fqdn, "ubuntu"); + + taosArrayPush(execNode, &addr); + return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob); } typedef struct tmq_t tmq_t; @@ -706,9 +713,13 @@ void* doFetchRow(SRequestObj* pRequest) { return NULL; } - scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); - setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + return NULL; + } + setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); if (pResultInfo->numOfRows == 0) { return NULL; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 415d6a57ce..59b133338e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -49,7 +49,7 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); } TEST(testCase, connect_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } @@ -295,24 +295,24 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); - ASSERT_EQ(taos_errno(pRes), 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); - ASSERT_NE(taos_errno(pRes), 0); - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); +// ASSERT_EQ(taos_errno(pRes), 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); +// ASSERT_NE(taos_errno(pRes), 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} //TEST(testCase, create_ctable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -333,36 +333,36 @@ TEST(testCase, create_table_Test) { // taos_close(pConn); //} -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != nullptr); - -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != nullptr); +// +//// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +// +// TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); // if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); // } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// // taos_free_result(pRes); - - TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +// taos_close(pConn); +//} // //TEST(testCase, show_vgroup_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -521,29 +521,29 @@ TEST(testCase, show_stable_Test) { // taosHashCleanup(phash); //} // -TEST(testCase, create_topic_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_topic_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == nullptr); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// char* sql = "select * from tu"; +// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +// taos_free_result(pRes); +// taos_close(pConn); +//} //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -614,30 +614,30 @@ TEST(testCase, create_topic_Test) { // taos_close(pConn); //} -//TEST(testCase, projection_query_stables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "select ts,k from m1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, projection_query_stables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "select ts from m1"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 75ddc37a03..383073871e 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -67,7 +67,7 @@ char * metaTbCursorNext(SMTbCursor *pTbCur); SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); -char * metaCtbCursorNext(SMCtbCursor *pCtbCur); +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index f3d9f71346..25e7c8cffa 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -92,6 +92,7 @@ int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); typedef void* tsdbReadHandleT; + /** * Get the data block iterator, starting from position according to the query condition * @@ -123,6 +124,24 @@ tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGro bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); +/** + * + * @param tsdb + * @param uid + * @param skey + * @param pTagCond + * @param len + * @param tagNameRelType + * @param tbnameCond + * @param pGroupInfo + * @param pColIndex + * @param numOfCols + * @param reqId + * @return + */ +int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, + int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId); /** * get num of rows in mem table * diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 490333397f..bbceaa3389 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -658,7 +658,7 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { } } -char *metaCtbCursorNext(SMCtbCursor *pCtbCur) { +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { DBT skey = {0}; DBT pkey = {0}; DBT pval = {0}; @@ -669,11 +669,13 @@ char *metaCtbCursorNext(SMCtbCursor *pCtbCur) { skey.data = &(pCtbCur->suid); skey.size = sizeof(pCtbCur->suid); - if (pCtbCur->pCur->get(pCtbCur->pCur, &skey, &pval, DB_NEXT) == 0) { - pBuf = pval.data; - metaDecodeTbInfo(pBuf, &tbCfg); - return tbCfg.name; + if (pCtbCur->pCur->pget(pCtbCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) { + tb_uid_t id = *(tb_uid_t *)pkey.data; + assert(id != 0); + return id; +// metaDecodeTbInfo(pBuf, &tbCfg); +// return tbCfg.; } else { - return NULL; + return 0; } } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d07a5ffc77..15748118d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -152,7 +152,7 @@ typedef struct STsdbReadHandle { typedef struct STableGroupSupporter { int32_t numOfCols; SColIndex* pCols; - STSchema* pTagSchema; + SSchema* pTagSchema; } STableGroupSupporter; static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); @@ -466,7 +466,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, return (tsdbReadHandleT)pReadHandle; _end: -// tsdbCleanupQueryHandle(pTsdbReadHandle); + tsdbCleanupQueryHandle(pReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } @@ -2630,18 +2630,20 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int return numOfRows; } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); +static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { + SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid); - STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode); + while (1) { + tb_uid_t id = metaCtbCursorNext(pCur); + if (id == 0) { + break; + } - STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = NULL, .lastKey = TSKEY_INITIAL_VAL, uid = id}; taosArrayPush(list, &info); } - tSkipListDestroyIter(iter); + metaCloseCtbCurosr(pCur); return TSDB_CODE_SUCCESS; } @@ -3553,7 +3555,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } -SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { +SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3564,25 +3566,18 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + SArray* sa = taosArrayDup(pTableList); if (sa == NULL) { taosArrayDestroy(pTableGroup); return NULL; } - for(int32_t i = 0; i < size; ++i) { - STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); - - STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; - taosArrayPush(sa, &info); - } - taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { STableGroupSupporter sup = {0}; sup.numOfCols = numOfOrderCols; - sup.pTagSchema = pTagSchema; + sup.pTagSchema = pTagSchema->pSchema; sup.pCols = pCols; // taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); @@ -3710,12 +3705,11 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - STSchema* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - assert(false); - int32_t ret = 0;//getAllTableList(pTable, res); + int32_t ret = getAllTableList(tsdb->pMeta, uid, res); if (ret != TSDB_CODE_SUCCESS) { goto _error; } @@ -3854,7 +3848,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGro return TSDB_CODE_SUCCESS; } - +#endif static void* doFreeColumnInfoData(SArray* pColumnInfoData) { if (pColumnInfoData == NULL) { return NULL; @@ -3883,6 +3877,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { return NULL; } + void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; if (pTsdbReadHandle == NULL) { @@ -3921,6 +3916,7 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { tfree(pTsdbReadHandle); } +#if 0 void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { assert(pGroupList != NULL); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index adb305ab09..255b000d10 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -649,6 +649,6 @@ int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); -int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, STableGroupInfo* pGroupInfo, void* readerHandle); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 1f5d0cd059..df28972645 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tarray.h" +#include #include "dataSinkMgt.h" #include "exception.h" +#include "os.h" +#include "tarray.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" @@ -72,7 +73,45 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ assert(tsdb != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); + int32_t code = 0; + uint64_t uid = 0; + STimeWindow window = TSWINDOW_INITIALIZER; + int32_t tableType = 0; + + SPhyNode *pPhyNode = pSubplan->pNode; + if (pPhyNode->info.type == OP_TableScan || pPhyNode->info.type == OP_DataBlocksOptScan) { + STableScanPhyNode* pTableScanNode = (STableScanPhyNode*)pPhyNode; + uid = pTableScanNode->scan.uid; + window = pTableScanNode->window; + tableType = pTableScanNode->scan.tableType; + } else { + assert(0); + } + + STableGroupInfo groupInfo = {0}; + if (tableType == TSDB_SUPER_TABLE) { + code = tsdbQuerySTableByTagCond(tsdb, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, pSubplan->id.queryId); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } else { // Create one table group. + groupInfo.numOfTables = 1; + groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES); + + SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; + taosArrayPush(pa, &info); + taosArrayPush(groupInfo.pGroupList, &pa); + } + + if (groupInfo.numOfTables == 0) { + code = 0; +// qDebug("no table qualified for query, reqId:0x%"PRIx64, (*pTask)->id.queryId); + goto _error; + } + + code = doCreateExecTaskInfo(pSubplan, pTask, &groupInfo, tsdb); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -141,6 +180,11 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); + // todo: remove it. + if (tinfo == NULL) { + return NULL; + } + *pRes = NULL; int64_t curOwner = 0; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b6df0c527e..9daf3298e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7369,15 +7369,13 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask } } -int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, STableGroupInfo* pGroupInfo, void* readerHandle) { STsdbQueryCond cond = {.loadExternalRows = false}; - uint64_t uid = 0; SPhyNode* pPhyNode = pPlan->pNode; if (pPhyNode->info.type == OP_TableScan || pPhyNode->info.type == OP_DataBlocksOptScan) { STableScanPhyNode* pTableScanNode = (STableScanPhyNode*) pPhyNode; - uid = pTableScanNode->scan.uid; cond.order = pTableScanNode->scan.order; cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); @@ -7397,15 +7395,8 @@ int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* r assert(0); } - STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; - SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; - taosArrayPush(pa, &info); - - taosArrayPush(group.pGroupList, &pa); - *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); - tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); + tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, pGroupInfo, (*pTaskInfo)->id.queryId, NULL); (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); if ((*pTaskInfo)->pRoot == NULL) { diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index c7a4e438ba..9aeae55c4f 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -558,11 +558,15 @@ static bool timeWindowToJson(const void* obj, cJSON* json) { static bool timeWindowFromJson(const cJSON* json, void* obj) { STimeWindow* win = (STimeWindow*)obj; - char* p = getString(json, jkTimeWindowStartKey); - win->skey = strtoll(p, NULL, 10); + char* pStartKey = getString(json, jkTimeWindowStartKey); + win->skey = strtoll(pStartKey, NULL, 10); + + char* pEndKey = getString(json, jkTimeWindowEndKey); + win->ekey = strtoll(pEndKey, NULL, 10); + + tfree(pStartKey); + tfree(pEndKey); - p = getString(json, jkTimeWindowEndKey); - win->ekey = strtoll(p, NULL, 10); return true; } @@ -574,14 +578,19 @@ static const char* jkScanNodeTableRevCount = "Reverse"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* pNode = (const SScanPhyNode*)obj; - bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, pNode->uid); + + char uid[40] = {0}; + snprintf(uid, tListLen(uid), "%"PRIu64, pNode->uid); + bool res = cJSON_AddStringToObject(json, jkScanNodeTableId, uid); if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, pNode->tableType); } + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, pNode->order); } + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, pNode->count); } @@ -589,12 +598,17 @@ static bool scanNodeToJson(const void* obj, cJSON* json) { if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableRevCount, pNode->reverse); } + return res; } static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* pNode = (SScanPhyNode*)obj; - pNode->uid = getNumber(json, jkScanNodeTableId); + + char* val = getString(json, jkScanNodeTableId); + pNode->uid = strtoull(val, NULL, 10); + tfree(val); + pNode->tableType = getNumber(json, jkScanNodeTableType); pNode->count = getNumber(json, jkScanNodeTableCount); pNode->order = getNumber(json, jkScanNodeTableOrder); @@ -726,7 +740,7 @@ static bool nodeAddrToJson(const void* obj, cJSON* json) { res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, ep->inUse); } if (res) { - res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, ep->epAddr, ep->numOfEps, sizeof(SEpAddr)); + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, ep->epAddr, sizeof(SEpAddr), ep->numOfEps); } return res; } diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index a3db9c6992..9ada451c61 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -1,15 +1,4 @@ aux_source_directory(src QWORKER_SRC) -#add_library(qworker ${QWORKER_SRC}) -#target_include_directories( -# qworker -# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" -# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -#) -# -#target_link_libraries( -# qworker -# PRIVATE os util transport planner qcom executor -#) add_library(qworker STATIC ${QWORKER_SRC}) target_include_directories( @@ -18,11 +7,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -#set_target_properties(qworker PROPERTIES -# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a" -# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker" -# ) - target_link_libraries(qworker PRIVATE os util transport planner qcom executor ) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 661beee5d5..6b047eb96e 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -29,7 +29,7 @@ extern "C" { #define SCHEDULE_DEFAULT_JOB_NUMBER 1000 #define SCHEDULE_DEFAULT_TASK_NUMBER 1000 -#define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA +#define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA enum { SCH_READ = 1, diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c53926f8c1..ffb602bd36 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -26,9 +26,9 @@ int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel * pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); - pTask->execAddrs = taosArrayInit(SCH_MAX_CONDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->execAddrs) { - SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CONDIDATE_EP_NUM); + SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -383,9 +383,9 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } pTask->candidateIdx = 0; - pTask->candidateAddrs = taosArrayInit(SCH_MAX_CONDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + pTask->candidateAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { - SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CONDIDATE_EP_NUM); + SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -405,10 +405,10 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CONDIDATE_EP_NUM; ++i) { + for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { SQueryNodeAddr *naddr = taosArrayGet(pJob->nodeList, i); - if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { + if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -418,12 +418,12 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } if (addNum <= 0) { - SCH_TASK_ELOG("no available execNode as condidate addr, nodeNum:%d", nodeNum); + SCH_TASK_ELOG("no available execNode as candidate addr, nodeNum:%d", nodeNum); return TSDB_CODE_QRY_INVALID_INPUT; } /* - for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CONDIDATE_EP_NUM; ++i) { + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; @@ -734,7 +734,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } /* - if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CONDIDATE_EP_NUM) { + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CANDIDATE_EP_NUM) { strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; @@ -1165,6 +1165,8 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("subplanToString error, code:%x, msg:%p, len:%d", code, pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); } + + printf("physical plan:%s\n", pTask->msg); } SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); From e2d8dfafe8b17272f9baee1b9fb102ff36770ce5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 18 Jan 2022 19:32:29 +0800 Subject: [PATCH 025/103] feature/qnode --- include/libs/scheduler/scheduler.h | 8 + include/util/taoserror.h | 1 + source/libs/scheduler/src/scheduler.c | 123 +++++++-- source/libs/scheduler/test/schedulerTests.cpp | 249 +++++++++++++++++- source/util/src/terror.c | 1 + 5 files changed, 359 insertions(+), 23 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 1f369067d6..3262b9437c 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -114,6 +114,14 @@ void schedulerDestroy(void); */ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks); +/** + * make one task info's multiple copies + * @param src + * @param dst SArray** + * @return + */ +int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum); + void schedulerFreeTaskList(SArray *taskList); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a855e6d881..fc895069af 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -361,6 +361,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) //"Task dropping") #define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation") #define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error") +#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) //"Job freed") // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c53926f8c1..ccf0c58ac0 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -20,12 +20,38 @@ static SSchedulerMgmt schMgmt = {0}; +uint64_t schGenTaskId(void) { + return atomic_add_fetch_64(&schMgmt.taskId, 1); +} + +uint64_t schGenUUID(void) { + static uint64_t hashId = 0; + static int32_t requestSerialId = 0; + + if (hashId == 0) { + char uid[64]; + int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + if (code != TSDB_CODE_SUCCESS) { + qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + hashId = MurmurHash3_32(uid, strlen(uid)); + } + } + + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + + uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + return id; +} + int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { pTask->plan = pPlan; pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); - pTask->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + pTask->taskId = schGenTaskId(); pTask->execAddrs = taosArrayInit(SCH_MAX_CONDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->execAddrs) { SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CONDIDATE_EP_NUM); @@ -41,7 +67,7 @@ void schFreeTask(SSchTask* pTask) { } // TODO NEED TO VERFY WITH ASYNC_SEND MEMORY FREE - //tfree(pTask->msg); + tfree(pTask->msg); if (pTask->children) { taosArrayDestroy(pTask->children); @@ -141,7 +167,7 @@ int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); break; default: @@ -541,12 +567,9 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b return TSDB_CODE_SUCCESS; } - - -// Note: no more error processing, handled in function internal -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { +int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) { // if already FAILED, no more processing - SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, status)); if (errCode) { atomic_store_32(&pJob->errCode, errCode); @@ -563,6 +586,17 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { +// Note: no more error processing, handled in function internal +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_FAILED, errCode)); +} + +// Note: no more error processing, handled in function internal +int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { + SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_DROPPING, errCode)); +} + + // Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; @@ -871,7 +905,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); if (NULL == job || NULL == (*job)) { qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); - SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } pJob = *job; @@ -880,13 +914,13 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in int32_t s = taosHashGetSize(pJob->execTasks); if (s <= 0) { - qError("QID:%"PRIx64",TID:%"PRIx64" no task in execTask list", pParam->queryId, pParam->taskId); + qError("QID:%"PRIx64",TID:%"PRId64" no task in execTask list", pParam->queryId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { - qError("QID:%"PRIx64",TID:%"PRIx64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); + qError("QID:%"PRIx64",TID:%"PRId64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -1434,7 +1468,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(plan->id.queryId); - pMsg->taskId = htobe64(atomic_add_fetch_64(&schMgmt.taskId, 1)); + pMsg->taskId = htobe64(schGenUUID()); pMsg->contentLen = htonl(msgLen); memcpy(pMsg->msg, msg, msgLen); @@ -1457,6 +1491,52 @@ _return: SCH_RET(code); } +int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { + if (NULL == src || NULL == dst || copyNum <= 0) { + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + + *dst = taosArrayInit(copyNum, sizeof(STaskInfo)); + if (NULL == *dst) { + qError("taosArrayInit %d taskInfo failed", copyNum); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t msgSize = src->msg->contentLen + sizeof(*src->msg); + STaskInfo info = {0}; + + info.addr = src->addr; + + for (int32_t i = 0; i < copyNum; ++i) { + info.msg = malloc(msgSize); + if (NULL == info.msg) { + qError("malloc %d failed", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(info.msg, src->msg, msgSize); + + info.msg->taskId = schGenUUID(); + + if (NULL == taosArrayPush(*dst, &info)) { + qError("taosArrayPush failed, idx:%d", i); + free(info.msg); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + return TSDB_CODE_SUCCESS; + +_return: + + schedulerFreeTaskList(*dst); + *dst = NULL; + + SCH_RET(code); +} + int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { @@ -1464,14 +1544,15 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { } int32_t code = 0; + atomic_add_fetch_32(&pJob->ref, 1); + int8_t status = SCH_GET_JOB_STATUS(pJob); if (status == JOB_TASK_STATUS_DROPPING) { SCH_JOB_ELOG("job is dropping, status:%d", status); - return TSDB_CODE_SCH_STATUS_ERROR; + atomic_sub_fetch_32(&pJob->ref, 1); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - atomic_add_fetch_32(&pJob->ref, 1); - if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); atomic_sub_fetch_32(&pJob->ref, 1); @@ -1484,7 +1565,7 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (status == JOB_TASK_STATUS_FAILED) { + if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); @@ -1500,7 +1581,7 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { status = SCH_GET_JOB_STATUS(pJob); - if (status == JOB_TASK_STATUS_FAILED) { + if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { code = atomic_load_32(&pJob->errCode); SCH_ERR_JRET(code); } @@ -1547,6 +1628,7 @@ void scheduleFreeJob(void *job) { SSchJob *pJob = job; uint64_t queryId = pJob->queryId; + bool setJobFree = false; if (SCH_GET_JOB_STATUS(pJob) > 0) { if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { @@ -1554,8 +1636,6 @@ void scheduleFreeJob(void *job) { return; } - schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); - SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); while (true) { @@ -1563,6 +1643,11 @@ void scheduleFreeJob(void *job) { if (0 == ref) { break; } else if (ref > 0) { + if (1 == ref && atomic_load_8(&pJob->userFetch) > 0 && !setJobFree) { + schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); + setJobFree = true; + } + usleep(1); } else { assert(0); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 5332c6fcd1..58159f2306 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -38,6 +38,20 @@ namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); +extern "C" int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode); + +struct SSchJob *pInsertJob = NULL; +struct SSchJob *pQueryJob = NULL; + +uint64_t schtMergeTemplateId = 0x4; +uint64_t schtFetchTaskId = 0; +uint64_t schtQueryId = 1; + +bool schtTestStop = false; +bool schtTestDeadLoop = false; +int32_t schtTestMTRunSec = 60; +int32_t schtTestPrintNum = 1000; +int32_t schtStartFetch = 0; void schtInitLogFile() { @@ -57,7 +71,7 @@ void schtInitLogFile() { void schtBuildQueryDag(SQueryDag *dag) { - uint64_t qId = 0x0000000000000001; + uint64_t qId = schtQueryId; dag->queryId = qId; dag->numOfSubplans = 2; @@ -84,7 +98,7 @@ void schtBuildQueryDag(SQueryDag *dag) { scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; - mergePlan->id.templateId = 0x4444444444; + mergePlan->id.templateId = schtMergeTemplateId; mergePlan->id.subplanId = 0x5555555555; mergePlan->type = QUERY_TYPE_MERGE; mergePlan->level = 0; @@ -269,15 +283,225 @@ void *schtCreateFetchRspThread(void *param) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; + code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(rsp), 0); assert(code == 0); } +void *schtFetchRspThread(void *aa) { + SDataBuf dataBuf = {0}; + SSchCallbackParam* param = NULL; + + while (!schtTestStop) { + if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) { + continue; + } + + usleep(1); + + param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + + param->queryId = schtQueryId; + param->taskId = schtFetchTaskId; + + int32_t code = 0; + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + rsp->completed = 1; + rsp->numOfRows = 10; + + dataBuf.pData = rsp; + dataBuf.len = sizeof(*rsp); + + code = schHandleCallback(param, &dataBuf, TDMT_VND_FETCH_RSP, 0); + + assert(code == 0 || code); + } +} + +void schtFreeQueryJob(int32_t freeThread) { + static uint32_t freeNum = 0; + SSchJob *job = atomic_load_ptr(&pQueryJob); + + if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) { + scheduleFreeJob(job); + if (freeThread) { + if (++freeNum % schtTestPrintNum == 0) { + printf("FreeNum:%d\n", freeNum); + } + } + } +} + +void* schtRunJobThread(void *aa) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + SQueryDag dag = {0}; + + schtInitLogFile(); + + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + SEpAddr qnodeAddr = {0}; + strcpy(qnodeAddr.fqdn, "qnode0.ep"); + qnodeAddr.port = 6031; + taosArrayPush(qnodeList, &qnodeAddr); + + int32_t code = schedulerInit(NULL); + assert(code == 0); + + + schtSetPlanToString(); + schtSetExecNode(); + schtSetAsyncSendMsgToServer(); + + SSchJob *job = NULL; + SSchCallbackParam *param = NULL; + SHashObj *execTasks = NULL; + SDataBuf dataBuf = {0}; + uint32_t jobFinished = 0; + + while (!schtTestStop) { + schtBuildQueryDag(&dag); + + code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &job); + assert(code == 0); + + execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + void *pIter = taosHashIterate(job->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + schtFetchTaskId = task->taskId - 1; + + taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task)); + pIter = taosHashIterate(job->execTasks, pIter); + } + + param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param->queryId = schtQueryId; + + pQueryJob = job; + + + pIter = taosHashIterate(execTasks, NULL); + while (pIter) { + SSchTask *task = (SSchTask *)pIter; + + param->taskId = task->taskId; + SQueryTableRsp rsp = {0}; + dataBuf.pData = &rsp; + dataBuf.len = sizeof(rsp); + + code = schHandleCallback(param, &dataBuf, TDMT_VND_QUERY_RSP, 0); + assert(code == 0 || code); + + pIter = taosHashIterate(execTasks, pIter); + } + + + param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param->queryId = schtQueryId; + + pIter = taosHashIterate(execTasks, NULL); + while (pIter) { + SSchTask *task = (SSchTask *)pIter; + + param->taskId = task->taskId; + SResReadyRsp rsp = {0}; + dataBuf.pData = &rsp; + dataBuf.len = sizeof(rsp); + + code = schHandleCallback(param, &dataBuf, TDMT_VND_RES_READY_RSP, 0); + assert(code == 0 || code); + + pIter = taosHashIterate(execTasks, pIter); + } + + + param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param->queryId = schtQueryId; + + pIter = taosHashIterate(execTasks, NULL); + while (pIter) { + SSchTask *task = (SSchTask *)pIter; + + param->taskId = task->taskId - 1; + SQueryTableRsp rsp = {0}; + dataBuf.pData = &rsp; + dataBuf.len = sizeof(rsp); + + code = schHandleCallback(param, &dataBuf, TDMT_VND_QUERY_RSP, 0); + assert(code == 0 || code); + + pIter = taosHashIterate(execTasks, pIter); + } + + + param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param->queryId = schtQueryId; + + pIter = taosHashIterate(execTasks, NULL); + while (pIter) { + SSchTask *task = (SSchTask *)pIter; + + param->taskId = task->taskId - 1; + SResReadyRsp rsp = {0}; + dataBuf.pData = &rsp; + dataBuf.len = sizeof(rsp); + + code = schHandleCallback(param, &dataBuf, TDMT_VND_RES_READY_RSP, 0); + assert(code == 0 || code); + + pIter = taosHashIterate(execTasks, pIter); + } + + atomic_store_32(&schtStartFetch, 1); + + void *data = NULL; + code = scheduleFetchRows(pQueryJob, &data); + assert(code == 0 || code); + + if (0 == code) { + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; + assert(pRsp->completed == 1); + assert(pRsp->numOfRows == 10); + } + + data = NULL; + code = scheduleFetchRows(pQueryJob, &data); + assert(code == 0 || code); + + assert(data == (void*)NULL); + + schtFreeQueryJob(0); + + taosHashCleanup(execTasks); + + schtFreeQueryDag(&dag); + + if (++jobFinished % schtTestPrintNum == 0) { + printf("jobFinished:%d\n", jobFinished); + } + + ++schtQueryId; + } + + schedulerDestroy(); + +} + +void* schtFreeJobThread(void *aa) { + while (!schtTestStop) { + usleep(rand() % 2000); + schtFreeQueryJob(1); + } +} -struct SSchJob *pInsertJob = NULL; } TEST(queryTest, normalCase) { @@ -427,13 +651,30 @@ TEST(insertTest, normalCase) { } TEST(multiThread, forceFree) { + pthread_attr_t thattr; + pthread_attr_init(&thattr); - schtInitLogFile(); + pthread_t thread1, thread2, thread3; + pthread_create(&(thread1), &thattr, schtRunJobThread, NULL); + pthread_create(&(thread2), &thattr, schtFreeJobThread, NULL); + pthread_create(&(thread3), &thattr, schtFetchRspThread, NULL); + while (true) { + if (schtTestDeadLoop) { + sleep(1); + } else { + sleep(schtTestMTRunSec); + break; + } + } + + schtTestStop = true; + sleep(3); } int main(int argc, char** argv) { + srand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 144de08cd0..f520585315 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -360,6 +360,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPING, "Task dropping") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUPLICATTED_OPERATION, "Duplicatted operation") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED, "Job already freed") From 3f4551fa01b219fb6d7a94566df8cb40a3f470d6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 11:44:03 +0000 Subject: [PATCH 026/103] fix ctb cursor issue --- source/dnode/vnode/src/meta/metaBDBImpl.c | 25 ++++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index bbceaa3389..d8ee48ff23 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -374,22 +374,27 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey DBT * pDbt; if (pTbCfg->type == META_CHILD_TABLE) { - pDbt = calloc(2, sizeof(DBT)); + // pDbt = calloc(2, sizeof(DBT)); - // First key is suid - pDbt[0].data = &(pTbCfg->ctbCfg.suid); - pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); + // // First key is suid + // pDbt[0].data = &(pTbCfg->ctbCfg.suid); + // pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); - // Second key is the first tag - void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - pDbt[1].data = pTagVal; - pDbt[1].size = sizeof(int32_t); + // // Second key is the first tag + // void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); + // pDbt[1].data = pTagVal; + // pDbt[1].size = sizeof(int32_t); // Set index key memset(pSKey, 0, sizeof(*pSKey)); +#if 0 pSKey->flags = DB_DBT_MULTIPLE | DB_DBT_APPMALLOC; pSKey->data = pDbt; pSKey->size = 2; +#else + pSKey->data = &(pTbCfg->ctbCfg.suid); + pSKey->size = sizeof(pTbCfg->ctbCfg.suid); +#endif return 0; } else { @@ -673,8 +678,8 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { tb_uid_t id = *(tb_uid_t *)pkey.data; assert(id != 0); return id; -// metaDecodeTbInfo(pBuf, &tbCfg); -// return tbCfg.; + // metaDecodeTbInfo(pBuf, &tbCfg); + // return tbCfg.; } else { return 0; } From df62f45256b3c884ffe7e4c15710fd49fa37f1ab Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Jan 2022 21:23:40 +0800 Subject: [PATCH 027/103] add client --- source/libs/transport/src/trans.c | 13 +- source/libs/transport/src/transCli.c | 38 +- source/libs/transport/src/transport.c | 773 -------------------------- source/libs/transport/test/rclient.c | 6 +- 4 files changed, 37 insertions(+), 793 deletions(-) delete mode 100644 source/libs/transport/src/transport.c diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index a59f2f88c7..89361b13ad 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -42,7 +42,18 @@ void* rpcOpen(const SRpcInit* pInit) { return pRpc; } void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } +void* rpcMallocCont(int contLen) { + int size = contLen + RPC_MSG_OVERHEAD; + + char* start = (char*)calloc(1, (size_t)size); + if (start == NULL) { + tError("failed to malloc msg, size:%d", size); + return NULL; + } else { + tTrace("malloc mem:%p size:%d", start, size); + } + return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); +} void rpcFreeCont(void* cont) { return; } void* rpcReallocCont(void* ptr, int contLen) { return NULL; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8ba1f1a0f6..29f3361b10 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -55,6 +55,10 @@ static void* clientThread(void* arg); static void clientWriteCb(uv_write_t* req, int status) { // impl later } +static void clientFailedCb(uv_handle_t* handle) { + // impl later + tDebug("close handle"); +} static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // impl later } @@ -68,8 +72,10 @@ static void clientConnCb(struct uv_connect_s* req, int status) { if (status != 0) { // call user fp later tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status)); + uv_close((uv_handle_t*)req->handle, clientFailedCb); return; } + assert(pConn->stream == req->handle); // impl later } @@ -123,19 +129,6 @@ static void clientAsyncCb(uv_async_t* handle) { static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; - - QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); - - // QUEUE_INIT(&pThrd->clientCache); - - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pThrd->loop); - - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; - uv_run(pThrd->loop, UV_RUN_DEFAULT); } @@ -146,14 +139,25 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* thrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + QUEUE_INIT(&pThrd->msg); + pthread_mutex_init(&pThrd->msgMtx, NULL); - thrd->shandle = shandle; - int err = pthread_create(&thrd->thread, NULL, clientThread, (void*)(thrd)); + // QUEUE_INIT(&pThrd->clientCache); + + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); + + pThrd->cliAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); + pThrd->cliAsync->data = pThrd; + + pThrd->shandle = shandle; + int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); if (err == 0) { tDebug("sucess to create tranport-client thread %d", i); } - cli->pThreadObj[i] = thrd; + cli->pThreadObj[i] = pThrd; } return cli; } diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c deleted file mode 100644 index 6cc2ca8c49..0000000000 --- a/source/libs/transport/src/transport.c +++ /dev/null @@ -1,773 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifdef USE_UV - -#include -#include "lz4.h" -#include "os.h" -#include "rpcCache.h" -#include "rpcHead.h" -#include "rpcLog.h" -#include "rpcTcp.h" -#include "rpcUdp.h" -#include "taoserror.h" -#include "tglobal.h" -#include "thash.h" -#include "tidpool.h" -#include "tmd5.h" -#include "tmempool.h" -#include "tmsg.h" -#include "transportInt.h" -#include "tref.h" -#include "trpc.h" -#include "ttimer.h" -#include "tutil.h" - -#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) -static const char* notify = "a"; - -typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; - uint16_t localPort; - int8_t connType; - int index; // for UDP server only, round robin for multiple threads - char label[TSDB_LABEL_LEN]; - - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - - void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); - - int32_t refCount; - void* parent; - void* idPool; // handle to ID pool - void* tmrCtrl; // handle to timer - SHashObj* hash; // handle returned by hash utility - void* tcphandle; // returned handle from TCP initialization - void* udphandle; // returned handle from UDP initialization - void* pCache; // connection cache - pthread_mutex_t mutex; - struct SRpcConn* connList; // connection list -} SRpcInfo; - -typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - struct SRpcConn* pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - SEpSet* pSet; // for synchronous API - char msg[0]; // RpcHead starts from here -} SRpcReqContext; - -typedef struct SThreadObj { - pthread_t thread; - uv_pipe_t* pipe; - int fd; - uv_loop_t* loop; - uv_async_t* workerAsync; // - queue conn; - pthread_mutex_t connMtx; - void* shandle; -} SThreadObj; - -typedef struct SClientObj { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; - SThreadObj** pThreadObj; -} SClientObj; - -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) -#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) -#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) -#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) -#define rpcIsReq(type) (type & 1U) - -typedef struct SServerObj { - pthread_t thread; - uv_tcp_t server; - uv_loop_t* loop; - int workerIdx; - int numOfThreads; - SThreadObj** pThreadObj; - uv_pipe_t** pipe; - uint32_t ip; - uint32_t port; -} SServerObj; - -typedef struct SConnBuffer { - char* buf; - int len; - int cap; - int left; -} SConnBuffer; - -typedef struct SRpcConn { - uv_tcp_t* pTcp; - uv_write_t* pWriter; - uv_timer_t* pTimer; - - uv_async_t* pWorkerAsync; - queue queue; - int ref; - int persist; // persist connection or not - SConnBuffer connBuf; // read buf, - SConnBuffer writeBuf; // write buf - int count; - void* shandle; // rpc init - void* ahandle; // - void* hostThread; - // del later - char secured; - int spi; - char info[64]; - char user[TSDB_UNI_LEN]; // user ID for the link - char secret[TSDB_PASSWORD_LEN]; - char ckey[TSDB_PASSWORD_LEN]; // ciphering key -} SRpcConn; - -// auth function -static int uvAuthMsg(SRpcConn* pConn, char* msg, int msgLen); -static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); -static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); -static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen); -// compress data -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); -static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); - -static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -static void uvOnTimeoutCb(uv_timer_t* handle); -static void uvOnWriteCb(uv_write_t* req, int status); -static void uvOnAcceptCb(uv_stream_t* stream, int status); -static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); -static void uvWorkerAsyncCb(uv_async_t* handle); - -static SRpcConn* connCreate(); -static void connDestroy(SRpcConn* conn); -static void uvConnDestroy(uv_handle_t* handle); - -static void* workerThread(void* arg); -static void* acceptThread(void* arg); - -void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); -void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); - -void* (*taosHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = {taosInitServer, taosInitClient}; - -void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SClientObj* cli = calloc(1, sizeof(SClientObj)); - memcpy(cli->label, label, strlen(label)); - cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SThreadObj**)calloc(cli->numOfThreads, sizeof(SThreadObj*)); - - for (int i = 0; i < cli->numOfThreads; i++) { - SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); - - int err = pthread_create(&thrd->thread, NULL, workerThread, (void*)(thrd)); - if (err == 0) { - tDebug("sucess to create tranport-client thread %d", i); - } - } - return cli; -} - -void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SServerObj* srv = calloc(1, sizeof(SServerObj)); - srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - srv->numOfThreads = numOfThreads; - srv->workerIdx = 0; - srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThreads, sizeof(SThreadObj*)); - srv->pipe = (uv_pipe_t**)calloc(srv->numOfThreads, sizeof(uv_pipe_t*)); - srv->ip = ip; - srv->port = port; - uv_loop_init(srv->loop); - - for (int i = 0; i < srv->numOfThreads; i++) { - SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); - int fds[2]; - if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { - return NULL; - } - uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); - uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - - thrd->shandle = shandle; - thrd->fd = fds[0]; - thrd->pipe = &(srv->pipe[i][1]); // init read - int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); - if (err == 0) { - tDebug("sucess to create worker-thread %d", i); - // printf("thread %d create\n", i); - } else { - // TODO: clear all other resource later - tError("failed to create worker-thread %d", i); - } - srv->pThreadObj[i] = thrd; - } - - int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); - if (err == 0) { - tDebug("success to create accept-thread"); - } else { - // clear all resource later - } - - return srv; -} -void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - /* - * formate of data buffer: - * |<-------SRpcReqContext------->|<------------data read from socket----------->| - */ - static const int CAPACITY = 1024; - - SRpcConn* ctx = handle->data; - SConnBuffer* pBuf = &ctx->connBuf; - if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); - pBuf->len = 0; - pBuf->cap = CAPACITY; - pBuf->left = -1; - - buf->base = pBuf->buf + RPC_RESERVE_SIZE; - buf->len = CAPACITY; - } else { - if (pBuf->len >= pBuf->cap) { - if (pBuf->left == -1) { - pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); - } else if (pBuf->len + pBuf->left > pBuf->cap) { - pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); - } - } - buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; - buf->len = pBuf->cap - pBuf->len; - } -} -// check data read from socket completely or not -// -static bool isReadAll(SConnBuffer* data) { - // TODO(yihao): handle pipeline later - SRpcHead rpcHead; - int32_t headLen = sizeof(rpcHead); - if (data->len >= headLen) { - memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - if (msgLen > data->len) { - data->left = msgLen - data->len; - return false; - } else { - return true; - } - } else { - return false; - } -} -static void uvDoProcess(SRecvInfo* pRecv) { - SRpcHead* pHead = (SRpcHead*)pRecv->msg; - SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; - SRpcConn* pConn = pRecv->thandle; - - tDump(pRecv->msg, pRecv->msgLen); - - terrno = 0; - SRpcReqContext* pContest; - - // do auth and check -} -static int uvAuthMsg(SRpcConn* pConn, char* msg, int len) { - SRpcHead* pHead = (SRpcHead*)msg; - int code = 0; - - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { - // secured link, or no authentication - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, secured link, no auth is required", pConn->info); - return 0; - } - - if (!rpcIsReq(pHead->msgType)) { - // for response, if code is auth failure, it shall bypass the auth process - code = htonl(pHead->code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || - code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - return 0; - } - } - - code = 0; - if (pHead->spi == pConn->spi) { - // authentication - SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); - - int32_t delta; - delta = (int32_t)htonl(pDigest->timeStamp); - delta -= (int32_t)taosGetTimestampSec(); - if (abs(delta) > 900) { - tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); - code = TSDB_CODE_RPC_INVALID_TIME_STAMP; - } else { - if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { - // tDebug("%s, authentication failed, msg discarded", pConn->info); - code = TSDB_CODE_RPC_AUTH_FAILURE; - } else { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client - // tTrace("%s, message is authenticated", pConn->info); - } - } - } else { - tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); - code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; - } - - return code; -} -// refers specifically to query or insert timeout -static void uvHandleActivityTimeout(uv_timer_t* handle) { - // impl later - SRpcConn* conn = handle->data; -} -static void uvProcessData(SRpcConn* pConn) { - SRecvInfo info; - SRecvInfo* p = &info; - SConnBuffer* pBuf = &pConn->connBuf; - p->msg = pBuf->buf + RPC_RESERVE_SIZE; - p->msgLen = pBuf->len; - p->ip = 0; - p->port = 0; - p->shandle = pConn->shandle; // - p->thandle = pConn; - p->chandle = NULL; - - // - SRpcHead* pHead = (SRpcHead*)p->msg; - assert(rpcIsReq(pHead->msgType)); - - SRpcInfo* pRpc = (SRpcInfo*)p->shandle; - pConn->ahandle = (void*)pHead->ahandle; - // auth here - - int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); - if (code != 0) { - terrno = code; - return; - } - pHead->code = htonl(pHead->code); - - SRpcMsg rpcMsg; - - pHead = rpcDecompressRpcMsg(pHead); - rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); - rpcMsg.pCont = pHead->content; - rpcMsg.msgType = pHead->msgType; - rpcMsg.code = pHead->code; - rpcMsg.ahandle = pConn->ahandle; - rpcMsg.handle = pConn; - - (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); - uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0); - // auth - // validate msg type -} -void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { - // opt - SRpcConn* ctx = cli->data; - SConnBuffer* pBuf = &ctx->connBuf; - if (nread > 0) { - pBuf->len += nread; - if (isReadAll(pBuf)) { - tDebug("alread read complete packet"); - uvProcessData(ctx); - } else { - tDebug("read half packet, continue to read"); - } - return; - } - if (terrno != 0) { - // handle err code - } - - if (nread != UV_EOF) { - tDebug("Read error %s\n", uv_err_name(nread)); - } - uv_close((uv_handle_t*)cli, uvConnDestroy); -} -void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - buf->base = malloc(sizeof(char)); - buf->len = 2; -} - -void uvOnTimeoutCb(uv_timer_t* handle) { - // opt - tDebug("time out"); -} - -void uvOnWriteCb(uv_write_t* req, int status) { - SRpcConn* conn = req->data; - if (status == 0) { - tDebug("data already was written on stream"); - } else { - connDestroy(conn); - } - // opt -} - -void uvWorkerAsyncCb(uv_async_t* handle) { - SThreadObj* pThrd = container_of(handle, SThreadObj, workerAsync); - SRpcConn* conn = NULL; - - // opt later - pthread_mutex_lock(&pThrd->connMtx); - if (!QUEUE_IS_EMPTY(&pThrd->conn)) { - queue* head = QUEUE_HEAD(&pThrd->conn); - conn = QUEUE_DATA(head, SRpcConn, queue); - QUEUE_REMOVE(&conn->queue); - } - pthread_mutex_unlock(&pThrd->connMtx); - if (conn == NULL) { - tError("except occurred, do nothing"); - return; - } - uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); - uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); -} - -void uvOnAcceptCb(uv_stream_t* stream, int status) { - if (status == -1) { - return; - } - SServerObj* pObj = container_of(stream, SServerObj, server); - - uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pObj->loop, cli); - - if (uv_accept(stream, (uv_stream_t*)cli) == 0) { - uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); - - uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); - - pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThreads; - tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); - uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); - } else { - uv_close((uv_handle_t*)cli, NULL); - } -} -void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { - tDebug("connection coming"); - if (nread < 0) { - if (nread != UV_EOF) { - tError("read error %s", uv_err_name(nread)); - } - // TODO(log other failure reason) - uv_close((uv_handle_t*)q, NULL); - return; - } - // free memory allocated by - assert(nread == strlen(notify)); - assert(buf->base[0] == notify[0]); - free(buf->base); - - SThreadObj* pThrd = q->data; - - uv_pipe_t* pipe = (uv_pipe_t*)q; - if (!uv_pipe_pending_count(pipe)) { - tError("No pending count"); - return; - } - - uv_handle_type pending = uv_pipe_pending_type(pipe); - assert(pending == UV_TCP); - - SRpcConn* pConn = connCreate(); - pConn->shandle = pThrd->shandle; - /* init conn timer*/ - pConn->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pThrd->loop, pConn->pTimer); - pConn->pTimer->data = pConn; - - pConn->hostThread = pThrd; - pConn->pWorkerAsync = pThrd->workerAsync; // thread safty - - // init client handle - pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pThrd->loop, pConn->pTcp); - pConn->pTcp->data = pConn; - - // init write request, just - pConn->pWriter = calloc(1, sizeof(uv_write_t)); - pConn->pWriter->data = pConn; - - if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { - uv_os_fd_t fd; - uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); - tDebug("new connection created: %d", fd); - uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); - } else { - connDestroy(pConn); - } -} - -void* acceptThread(void* arg) { - // opt - SServerObj* srv = (SServerObj*)arg; - uv_tcp_init(srv->loop, &srv->server); - - struct sockaddr_in bind_addr; - - uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); - uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); - int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { - tError("Listen error %s\n", uv_err_name(err)); - return NULL; - } - uv_run(srv->loop, UV_RUN_DEFAULT); -} -void* workerThread(void* arg) { - SThreadObj* pThrd = (SThreadObj*)arg; - - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pThrd->loop); - - uv_pipe_init(pThrd->loop, pThrd->pipe, 1); - uv_pipe_open(pThrd->pipe, pThrd->fd); - - pThrd->pipe->data = pThrd; - - QUEUE_INIT(&pThrd->conn); - - pThrd->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); - - uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); - uv_run(pThrd->loop, UV_RUN_DEFAULT); -} -static SRpcConn* connCreate() { - SRpcConn* pConn = (SRpcConn*)calloc(1, sizeof(SRpcConn)); - return pConn; -} -static void connDestroy(SRpcConn* conn) { - if (conn == NULL) { - return; - } - uv_timer_stop(conn->pTimer); - free(conn->pTimer); - uv_close((uv_handle_t*)conn->pTcp, NULL); - free(conn->connBuf.buf); - free(conn->pTcp); - free(conn->pWriter); - free(conn); - // handle -} -static void uvConnDestroy(uv_handle_t* handle) { - SRpcConn* conn = handle->data; - connDestroy(conn); -} -void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); - if (pRpc == NULL) { - return NULL; - } - if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); - } - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - pRpc->connType = pInit->connType; - pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); - // pRpc->taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); - return pRpc; -} -void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } -void rpcFreeCont(void* cont) { return; } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } - -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { - // impl later - return; -} - -void rpcSendResponse(const SRpcMsg* pMsg) { - SRpcConn* pConn = pMsg->handle; - SThreadObj* pThrd = pConn->hostThread; - - // opt later - pthread_mutex_lock(&pThrd->connMtx); - QUEUE_PUSH(&pThrd->conn, &pConn->queue); - pthread_mutex_unlock(&pThrd->connMtx); - - uv_async_send(pConn->pWorkerAsync); -} - -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } -int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } -void rpcCancelRequest(int64_t rid) { return; } - -static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - int ret = -1; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; - - return ret; -} -static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { - T_MD5_CTX context; - - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Update(&context, (uint8_t*)pMsg, msgLen); - tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); - tMD5Final(&context); - - memcpy(pAuth, context.digest, sizeof(context.digest)); -} - -static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen) { - SRpcHead* pHead = (SRpcHead*)msg; - - if (pConn->spi && pConn->secured == 0) { - // add auth part - pHead->spi = pConn->spi; - SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen); - pDigest->timeStamp = htonl(taosGetTimestampSec()); - msgLen += sizeof(SRpcDigest); - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); - } else { - pHead->spi = 0; - pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - } - - return msgLen; -} - -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { - SRpcHead* pHead = rpcHeadFromCont(pCont); - int32_t finalLen = 0; - int overhead = sizeof(SRpcComp); - - if (!NEEDTO_COMPRESSS_MSG(contLen)) { - return contLen; - } - - char* buf = malloc(contLen + overhead + 8); // 8 extra bytes - if (buf == NULL) { - tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); - return contLen; - } - - int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); - tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); - - /* - * only the compressed size is less than the value of contLen - overhead, the compression is applied - * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message - */ - if (compLen > 0 && compLen < contLen - overhead) { - SRpcComp* pComp = (SRpcComp*)pCont; - pComp->reserved = 0; - pComp->contLen = htonl(contLen); - memcpy(pCont + overhead, buf, compLen); - - pHead->comp = 1; - tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); - finalLen = compLen + overhead; - } else { - finalLen = contLen; - } - - free(buf); - return finalLen; -} - -static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { - int overhead = sizeof(SRpcComp); - SRpcHead* pNewHead = NULL; - uint8_t* pCont = pHead->content; - SRpcComp* pComp = (SRpcComp*)pHead->content; - - if (pHead->comp) { - // decompress the content - assert(pComp->reserved == 0); - int contLen = htonl(pComp->contLen); - - // prepare the temporary buffer to decompress message - char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD); - pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext - - if (pNewHead) { - int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; - int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen); - assert(origLen == contLen); - - memcpy(pNewHead, pHead, sizeof(SRpcHead)); - pNewHead->msgLen = rpcMsgLenFromCont(origLen); - /// rpcFreeMsg(pHead); // free the compressed message buffer - pHead = pNewHead; - tTrace("decomp malloc mem:%p", temp); - } else { - tError("failed to allocate memory to decompress msg, contLen:%d", contLen); - } - } - - return pHead; -} -int32_t rpcInit(void) { - // impl later - return -1; -} - -void rpcCleanup(void) { - // impl later - return; -} -#endif diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 045fb8520e..58fbf6ae85 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -34,7 +34,8 @@ typedef struct { static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); + tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -185,7 +186,8 @@ int main(int argc, char *argv[]) { // float usedTime = (endTime - startTime) / 1000.0f; // mseconds // tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - // tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); + // tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, + // msgSize); int ch = getchar(); UNUSED(ch); From ee7393e589d8f81ee2893abc732d8daa6aee36fe Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 19 Jan 2022 10:00:24 +0800 Subject: [PATCH 028/103] [modify] --- tests/script/sh/massiveTable/deployCluster.sh | 15 +++++++-------- tests/script/sh/massiveTable/setupDnodes.sh | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index 2341d512c0..a86e9220cd 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -6,17 +6,16 @@ set -e #set -x # deployCluster.sh +curr_dir=$(readlink -f "$(dirname "$0")") +echo $curr_dir -curr_dir=$(pwd) -echo "currect pwd: ${curr_dir}" +${curr_dir}/cleanCluster.sh -r "/data" +${curr_dir}/cleanCluster.sh -r "/data2" -./cleanCluster.sh -r "/data" -./cleanCluster.sh -r "/data2" +${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" -./compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" - -./setupDnodes.sh -r "/data" -n 1 -f "trd02:7000" -p 7000 -./setupDnodes.sh -r "/data2" -n 1 -f "trd02:7000" -p 8000 +${curr_dir}/setupDnodes.sh -r "/data" -n 1 -f "trd02:7000" -p 7000 +${curr_dir}/setupDnodes.sh -r "/data2" -n 1 -f "trd02:7000" -p 8000 #./setupDnodes.sh -r "/data" -n 2 -f trd02:7000 -p 7000 #./setupDnodes.sh -r "/data2" -n 2 -f trd02:7000 -p 8000 diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index e09b2fb396..e45c7724ba 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -121,7 +121,7 @@ startDnodes() { ############################### main process ########################################## ## kill all taosd process -kill_process taosd +#kill_process taosd ## create director for all dnode if [[ "$enviMode" == "new" ]]; then From 76b8abe2df00a758b4649945db805ec4b9750365 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 19 Jan 2022 10:18:31 +0800 Subject: [PATCH 029/103] feature/qnode --- source/libs/scheduler/src/scheduler.c | 54 +++++++++++++------ source/libs/scheduler/test/schedulerTests.cpp | 35 ++++++------ 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ccf0c58ac0..de295f77c6 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -66,7 +66,6 @@ void schFreeTask(SSchTask* pTask) { taosArrayDestroy(pTask->candidateAddrs); } - // TODO NEED TO VERFY WITH ASYNC_SEND MEMORY FREE tfree(pTask->msg); if (pTask->children) { @@ -97,7 +96,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%d, rspType:%d", lastMsgType, msgType); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); @@ -868,8 +867,18 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } + if (pJob->res) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->res); + tfree(rsp); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + atomic_store_ptr(&pJob->res, rsp); atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); + + if (rsp->completed) { + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + } SCH_ERR_JRET(schProcessOnDataFetched(pJob)); @@ -1067,7 +1076,13 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; - msg = pTask->msg; + msg = calloc(1, msgSize); + if (NULL == msg) { + SCH_TASK_ELOG("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(msg, pTask->msg, msgSize); break; } @@ -1549,29 +1564,24 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { int8_t status = SCH_GET_JOB_STATUS(pJob); if (status == JOB_TASK_STATUS_DROPPING) { SCH_JOB_ELOG("job is dropping, status:%d", status); - atomic_sub_fetch_32(&pJob->ref, 1); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); - atomic_sub_fetch_32(&pJob->ref, 1); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); - atomic_sub_fetch_32(&pJob->ref, 1); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - *pData = atomic_load_ptr(&pJob->res); - atomic_store_ptr(&pJob->res, NULL); + SCH_JOB_ELOG("job failed or dropping, status:%d", status); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } else if (status == JOB_TASK_STATUS_SUCCEED) { - *pData = atomic_load_ptr(&pJob->res); - atomic_store_ptr(&pJob->res, NULL); + SCH_JOB_ELOG("job already succeed, status:%d", status); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_ERR_JRET(schFetchFromRemote(pJob)); @@ -1582,14 +1592,16 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { status = SCH_GET_JOB_STATUS(pJob); if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - code = atomic_load_32(&pJob->errCode); - SCH_ERR_JRET(code); + SCH_JOB_ELOG("job failed or dropping, status:%d", status); + SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } +_return: + while (true) { *pData = atomic_load_ptr(&pJob->res); @@ -1600,10 +1612,19 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { break; } -_return: + if (NULL == *pData) { + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + if (rsp) { + rsp->completed = 1; + } + + *pData = rsp; + } atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); + SCH_JOB_DLOG("fetch done, code:%x", code); + atomic_sub_fetch_32(&pJob->ref, 1); SCH_RET(code); @@ -1683,6 +1704,7 @@ void scheduleFreeJob(void *job) { taosHashCleanup(pJob->succTasks); taosArrayDestroy(pJob->levels); + taosArrayDestroy(pJob->nodeList); tfree(pJob->res); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 58159f2306..1425ac0e6c 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -49,7 +49,7 @@ uint64_t schtQueryId = 1; bool schtTestStop = false; bool schtTestDeadLoop = false; -int32_t schtTestMTRunSec = 60; +int32_t schtTestMTRunSec = 10; int32_t schtTestPrintNum = 1000; int32_t schtStartFetch = 0; @@ -187,8 +187,6 @@ void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int6 } - - void schtSetPlanToString() { static Stub stub; stub.set(qSubPlanToString, schtPlanToString); @@ -228,7 +226,12 @@ void schtSetRpcSendRequest() { } } -int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { +int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { + if (pInfo) { + tfree(pInfo->param); + tfree(pInfo->msgInfo.pData); + free(pInfo); + } return 0; } @@ -284,7 +287,7 @@ void *schtCreateFetchRspThread(void *param) { rsp->completed = 1; rsp->numOfRows = 10; - code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, job->fetchTask, TDMT_VND_FETCH_RSP, (char *)rsp, sizeof(*rsp), 0); assert(code == 0); } @@ -344,12 +347,6 @@ void* schtRunJobThread(void *aa) { schtInitLogFile(); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); - - SEpAddr qnodeAddr = {0}; - strcpy(qnodeAddr.fqdn, "qnode0.ep"); - qnodeAddr.port = 6031; - taosArrayPush(qnodeList, &qnodeAddr); int32_t code = schedulerInit(NULL); assert(code == 0); @@ -368,6 +365,13 @@ void* schtRunJobThread(void *aa) { while (!schtTestStop) { schtBuildQueryDag(&dag); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + SEpAddr qnodeAddr = {0}; + strcpy(qnodeAddr.fqdn, "qnode0.ep"); + qnodeAddr.port = 6031; + taosArrayPush(qnodeList, &qnodeAddr); + code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &job); assert(code == 0); @@ -475,8 +479,6 @@ void* schtRunJobThread(void *aa) { code = scheduleFetchRows(pQueryJob, &data); assert(code == 0 || code); - assert(data == (void*)NULL); - schtFreeQueryJob(0); taosHashCleanup(execTasks); @@ -496,7 +498,7 @@ void* schtRunJobThread(void *aa) { void* schtFreeJobThread(void *aa) { while (!schtTestStop) { - usleep(rand() % 2000); + usleep(rand() % 100); schtFreeQueryJob(1); } } @@ -592,11 +594,12 @@ TEST(queryTest, normalCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); + tfree(data); data = NULL; code = scheduleFetchRows(job, &data); ASSERT_EQ(code, 0); - ASSERT_EQ(data, (void*)NULL); + ASSERT_TRUE(data); scheduleFreeJob(pJob); @@ -607,7 +610,6 @@ TEST(queryTest, normalCase) { - TEST(insertTest, normalCase) { void *mockPointer = (void *)0x1; char *clusterId = "cluster1"; @@ -672,7 +674,6 @@ TEST(multiThread, forceFree) { sleep(3); } - int main(int argc, char** argv) { srand(time(NULL)); testing::InitGoogleTest(&argc, argv); From 9a5f4f2f449e3deaab5d5ef90375bf4721c99243 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Jan 2022 19:50:18 -0800 Subject: [PATCH 030/103] fix create table error while use multi dnodes --- source/dnode/mnode/impl/src/mndAuth.c | 15 +++- tests/test/c/create_table.c | 103 ++++++++++++-------------- 2 files changed, 60 insertions(+), 58 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index a2fc7a57b3..52730412e9 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -16,9 +16,18 @@ #define _DEFAULT_SOURCE #include "mndAuth.h" -int32_t mndInitAuth(SMnode *pMnode) { return 0; } -void mndCleanupAuth(SMnode *pMnode) {} +static int32_t mndProcessAuthReq(SMnodeMsg *pReq); -int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { +int32_t mndInitAuth(SMnode *pMnode) { + mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); + return 0; +} + +void mndCleanupAuth(SMnode *pMnode) {} + +int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } + +static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { + mDebug("user:%s, auth req is processed", pReq->user); return 0; } \ No newline at end of file diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 0376ab70ad..60d2321a7d 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -46,9 +46,9 @@ typedef struct { pthread_t thread; } SThreadInfo; -//void parseArgument(int32_t argc, char *argv[]); -//void *threadFunc(void *param); -//void createDbAndStb(); +// void parseArgument(int32_t argc, char *argv[]); +// void *threadFunc(void *param); +// void createDbAndStb(); void createDbAndStb() { pPrint("start to create db and stable"); @@ -64,7 +64,8 @@ void createDbAndStb() { TAOS_RES *pRes = taos_query(con, qstr); int32_t code = taos_errno(pRes); if (code != 0) { - pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), taos_errstr(pRes)); + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), + taos_errstr(pRes)); exit(0); } taos_free_result(pRes); @@ -129,10 +130,9 @@ static int64_t getResult(TAOS_RES *tres) { return numOfRows; } - -void showTables() { +void showTables() { pPrint("start to show tables"); - char qstr[32]; + char qstr[128]; TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { @@ -140,9 +140,9 @@ void showTables() { exit(1); } - sprintf(qstr, "use %s", dbName); + snprintf(qstr, 128, "use %s", dbName); TAOS_RES *pRes = taos_query(con, qstr); - int code = taos_errno(pRes); + int code = taos_errno(pRes); if (code != 0) { pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(1); @@ -160,12 +160,11 @@ void showTables() { int64_t totalTableNum = getResult(pRes); taos_free_result(pRes); - pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); + pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); taos_close(con); } - void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; char *qstr = malloc(2000 * 1000); @@ -177,48 +176,48 @@ void *threadFunc(void *param) { exit(1); } - //printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); + // printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + // pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); taos_free_result(pRes); - if (createTable) { int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); pInfo->startMs = beginMs; - int64_t t = pInfo->tableBeginIndex; + int64_t t = pInfo->tableBeginIndex; for (; t <= pInfo->tableEndIndex;) { - //int64_t batch = (pInfo->tableEndIndex - t); - //batch = MIN(batch, batchNum); + // int64_t batch = (pInfo->tableEndIndex - t); + // batch = MIN(batch, batchNum); int32_t len = sprintf(qstr, "create table"); for (int32_t i = 0; i < batchNum;) { len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); - t++; - i++; + t++; + i++; if (t > pInfo->tableEndIndex) { - break; - } + break; + } } - int64_t startTs = taosGetTimestampUs(); + int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); } taos_free_result(pRes); - int64_t endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - //printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); - if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; if (delay < pInfo->minDelay) pInfo->minDelay = delay; - curMs = taosGetTimestampMs(); - if (curMs - beginMs > 10000) { - beginMs = curMs; - //printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); printCreateProgress(pInfo, t); } } @@ -227,7 +226,8 @@ void *threadFunc(void *param) { if (insertData) { int64_t curMs = 0; - int64_t beginMs = taosGetTimestampMs();; + int64_t beginMs = taosGetTimestampMs(); + ; pInfo->startMs = taosGetTimestampMs(); for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { @@ -247,7 +247,7 @@ void *threadFunc(void *param) { taos_free_result(pRes); curMs = taosGetTimestampMs(); - if (curMs - beginMs > 10000) { + if (curMs - beginMs > 10000) { printInsertProgress(pInfo, t); } t += (batch - 1); @@ -335,33 +335,32 @@ int32_t main(int32_t argc, char *argv[]) { parseArgument(argc, argv); if (showTablesFlag) { - showTables(); - return 0; + showTables(); + return 0; } - + createDbAndStb(); - pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); + pPrint("%d threads are spawned to create %" PRId64 " tables", numOfThreads, numOfTables); pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - //int64_t numOfTablesPerThread = numOfTables / numOfThreads; - //numOfTables = numOfTablesPerThread * numOfThreads; - + // int64_t numOfTablesPerThread = numOfTables / numOfThreads; + // numOfTables = numOfTablesPerThread * numOfThreads; if (numOfThreads < 1) { numOfThreads = 1; } - + int64_t a = numOfTables / numOfThreads; if (a < 1) { - numOfThreads = numOfTables; - a = 1; + numOfThreads = numOfTables; + a = 1; } - + int64_t b = 0; b = numOfTables % numOfThreads; @@ -371,7 +370,7 @@ int32_t main(int32_t argc, char *argv[]) { pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; - pInfo[i].minDelay = INT64_MAX; + pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); @@ -390,7 +389,7 @@ int32_t main(int32_t argc, char *argv[]) { createTableSpeed += pInfo[i].createTableSpeed; if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; - if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; } float insertDataSpeed = 0; @@ -398,21 +397,15 @@ int32_t main(int32_t argc, char *argv[]) { insertDataSpeed += pInfo[i].insertDataSpeed; } - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", - GREEN, - numOfTables, - createTableSpeed, - numOfThreads, - maxDelay, - minDelay, - NC); + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 + "us %s", + GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); if (insertData) { - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, - numOfThreads, NC); + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + numOfThreads, NC); } pthread_attr_destroy(&thattr); free(pInfo); } - From ed9709c3f0a593cb358a73cede29f041d0e0cadd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Jan 2022 21:39:32 -0800 Subject: [PATCH 031/103] refact tfs module --- include/libs/tfs/tfs.h | 255 +++++++++--- include/util/tdef.h | 12 +- source/common/src/tglobal.c | 2 +- source/dnode/mgmt/impl/inc/dndEnv.h | 1 + source/dnode/mgmt/impl/inc/dndInt.h | 1 + source/dnode/mgmt/impl/src/dndEnv.c | 33 +- source/dnode/vnode/inc/tsdb.h | 2 +- source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/inc/tsdbDef.h | 1 + source/dnode/vnode/src/inc/tsdbFile.h | 16 +- source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +- source/dnode/vnode/src/tsdb/tsdbCompact.c | 3 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 62 +-- source/dnode/vnode/src/tsdb/tsdbMain.c | 11 +- source/dnode/vnode/src/vnd/vnodeMain.c | 3 +- source/libs/tfs/inc/tfsInt.h | 82 ++-- source/libs/tfs/src/tfs.c | 485 +++++++++------------- source/libs/tfs/src/tfsDisk.c | 8 +- source/libs/tfs/src/tfsTier.c | 41 +- 20 files changed, 581 insertions(+), 447 deletions(-) diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 3828a93144..e1f28092cf 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -16,77 +16,226 @@ #ifndef _TD_TFS_H_ #define _TD_TFS_H_ -#include "tglobal.h" +#include "tcfg.h" #ifdef __cplusplus extern "C" { #endif +/* ------------------------ TYPES EXPOSED ------------------------ */ +typedef struct STfs STfs; +typedef struct STfsDir STfsDir; + typedef struct { int32_t level; int32_t id; } SDiskID; -#define TFS_UNDECIDED_LEVEL -1 -#define TFS_UNDECIDED_ID -1 -#define TFS_PRIMARY_LEVEL 0 -#define TFS_PRIMARY_ID 0 -#define TFS_MIN_LEVEL 0 -#define TFS_MAX_LEVEL (TSDB_MAX_TIERS - 1) - -// FS APIs ==================================== typedef struct { - int64_t total; - int64_t used; - int64_t avail; -} SFSMeta; - -int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk); -void tfsCleanup(); -void tfsUpdateSize(SFSMeta *pFSMeta); -void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id); - -const char *TFS_PRIMARY_PATH(); -const char *TFS_DISK_PATH(int32_t level, int32_t id); - -// TFILE APIs ==================================== -typedef struct { - int32_t level; - int32_t id; - char rname[TSDB_FILENAME_LEN]; // REL name + SDiskID did; char aname[TSDB_FILENAME_LEN]; // ABS name -} TFILE; + char rname[TSDB_FILENAME_LEN]; // REL name + STfs *pTfs; +} STfsFile; -#define TFILE_LEVEL(pf) ((pf)->level) -#define TFILE_ID(pf) ((pf)->id) -#define TFILE_NAME(pf) ((pf)->aname) -#define TFILE_REL_NAME(pf) ((pf)->rname) +/** + * @brief Open a fs. + * + * @param pCfg Config of the fs. + * @param ndisk Length of the config. + * @return STfs* The fs object. + */ +STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk); -#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags) -#define tfsclose(fd) close(fd) -#define tfsremove(pf) remove(TFILE_NAME(pf)) -#define tfscopy(sf, df) taosCopyFile(TFILE_NAME(sf), TFILE_NAME(df)) -#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) +/** + * @brief Close a fs. + * + * @param pTfs The fs object to close. + */ +void tfsClose(STfs *pTfs); -void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname); -bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); -int32_t tfsEncodeFile(void **buf, TFILE *pf); -void *tfsDecodeFile(void *buf, TFILE *pf); -void tfsbasename(const TFILE *pf, char *dest); -void tfsdirname(const TFILE *pf, char *dest); +/** + * @brief Update the disk size. + * + * @param pTfs The fs object. + */ +void tfsUpdateSize(STfs *pTfs); -// DIR APIs ==================================== -int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id); -int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id); -int32_t tfsMkdir(const char *rname); -int32_t tfsRmdir(const char *rname); -int32_t tfsRename(char *orname, char *nrname); +/** + * @brief Get the disk size. + * + * @param pTfs The fs object. + */ +SDiskSize tfsGetSize(STfs *pTfs); -typedef struct TDIR TDIR; +/** + * @brief Allocate an existing available tier level from fs. + * + * @param pTfs The fs object. + * @param expLevel Disk level want to allocate. + * @param pDiskId The disk ID after allocation. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId); -TDIR *tfsOpendir(const char *rname); -const TFILE *tfsReaddir(TDIR *tdir); -void tfsClosedir(TDIR *tdir); +/** + * @brief Get the primary path. + * + * @param pTfs The fs object. + * @return const char * The primary path. + */ +const char *tfsGetPrimaryPath(STfs *pTfs); + +/** + * @brief Get the disk path. + * + * @param pTfs The fs object. + * @param diskId The diskId. + * @return const char * The primary path. + */ +const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId); + +/** + * @brief Make directory at all levels in tfs. + * + * @param pTfs The fs object. + * @param rname The rel name of directory. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsMkdir(STfs *pTfs, const char *rname); + +/** + * @brief Create directories in tfs. + * + * @param pTfs The fs object. + * @param rname The rel name of directory. + * @param diskId The disk ID. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId); + +/** + * @brief Recursive create directories in tfs. + * + * @param pTfs The fs object. + * @param rname The rel name of directory. + * @param diskId The disk ID. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId); + +/** + * @brief Remove directory at all levels in tfs. + * + * @param pTfs The fs object. + * @param rname The rel name of directory. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsRmdir(STfs *pTfs, const char *rname); + +/** + * @brief Rename file/directory in tfs. + * + * @param pTfs The fs object. + * @param orname The rel name of old file. + * @param nrname The rel name of new file. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsRename(STfs *pTfs, char *orname, char *nrname); + +/** + * @brief Init file object in tfs. + * + * @param pTfs The fs object. + * @param pFile The file object. + * @param diskId The disk ID. + * @param rname The rel name of file. + */ +void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname); + +/** + * @brief Determine whether they are the same file. + * + * @param pFile1 The file object. + * @param pFile2 The file object. + * @param bool The compare result. + */ +bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2); + +/** + * @brief Encode file name to a buffer. + * + * @param buf The buffer where file name are saved. + * @param pFile The file object. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsEncodeFile(void **buf, STfsFile *pFile); + +/** + * @brief Decode file name from a buffer. + * + * @param pTfs The fs object. + * @param buf The buffer where file name are saved. + * @param pFile The file object. + * @return void * Buffer address after decode. + */ +void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile); + +/** + * @brief Get the basename of the file. + * + * @param pFile The file object. + * @param dest The buffer where basename will be saved. + */ +void tfsBasename(const STfsFile *pFile, char *dest); + +/** + * @brief Get the dirname of the file. + * + * @param pFile The file object. + * @param dest The buffer where dirname will be saved. + */ +void tfsDirname(const STfsFile *pFile, char *dest); + +/** + * @brief Remove file in tfs. + * + * @param pFile The file to be removed. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsRemoveFile(const STfsFile *pFile); + +/** + * @brief Copy file in tfs. + * + * @param pFile1 The src file. + * @param pFile2 The dest file. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2); + +/** + * @brief Open a directory for traversal. + * + * @param rname The rel name of file. + * @return STfsDir* The dir object. + */ +STfsDir *tfsOpendir(STfs *pTfs, const char *rname); + +/** + * @brief Get a file from dir and move to next pos. + * + * @param pDir The dir object. + * @return STfsFile* The file in dir. + */ +const STfsFile *tfsReaddir(STfsDir *pDir); + +/** + * @brief Close a directory. + * + * @param pDir The dir object. + */ +void tfsClosedir(STfsDir *pDir); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index 428de5d171..1847432cc9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -372,10 +372,14 @@ do { \ #define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P -#define TSDB_MAX_TIERS 3 -#define TSDB_MAX_DISKS_PER_TIER 16 -#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) - +#define TFS_MAX_TIERS 3 +#define TFS_MAX_DISKS_PER_TIER 16 +#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) +#define TFS_MIN_LEVEL 0 +#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 +#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK }; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 463b107aa7..40e278f54a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -137,7 +137,7 @@ int32_t tsDiskCfgNum = 0; #ifndef _STORAGE SDiskCfg tsDiskCfg[1]; #else -SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; +SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; #endif /* diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index e80237bb9c..7ab3f46fdb 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -134,6 +134,7 @@ typedef struct SDnode { SBnodeMgmt bmgmt; SVnodesMgmt vmgmt; STransMgmt tmgmt; + STfs *pTfs; SStartupReq startup; } SDnode; diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index c5ba494e5e..829fabd006 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -43,6 +43,7 @@ extern "C" { #include "qnode.h" #include "snode.h" #include "vnode.h" +#include "tfs.h" extern int32_t dDebugFlag; diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 23fc643abe..74fb5f1437 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -173,11 +173,12 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { return NULL; } - SDiskCfg dCfg; - strcpy(dCfg.dir, pDnode->cfg.dataDir); + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN); dCfg.level = 0; dCfg.primary = 1; - if (tfsInit(&dCfg, 1) != 0) { + pDnode->pTfs = tfsOpen(&dCfg, 1); + if (pDnode->pTfs == NULL) { dError("failed to init tfs since %s", terrstr()); dndClose(pDnode); return NULL; @@ -251,7 +252,7 @@ void dndClose(SDnode *pDnode) { dndCleanupQnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupMgmt(pDnode); - tfsCleanup(); + tfsClose(pDnode->pTfs); dndCloseImp(pDnode); free(pDnode); @@ -313,4 +314,28 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); +} + +// OTHER FUNCTIONS =================================== +void taosGetDisk() { +#if 0 + const double unit = 1024 * 1024 * 1024; + + SDiskSize diskSize = tfsGetSize(pTfs); + + tfsUpdateSize(&fsMeta); + tsTotalDataDirGB = (float)(fsMeta.total / unit); + tsUsedDataDirGB = (float)(fsMeta.used / unit); + tsAvailDataDirGB = (float)(fsMeta.avail / unit); + + if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { + tsTotalLogDirGB = (float)(diskSize.total / unit); + tsAvailLogDirGB = (float)(diskSize.avail / unit); + } + + if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { + tsTotalTmpDirGB = (float)(diskSize.total / unit); + tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); + } +#endif } \ No newline at end of file diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index f3d9f71346..e56a6bece3 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -80,7 +80,7 @@ typedef struct { } STableKeyInfo; // STsdb -STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); +STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs); void tsdbClose(STsdb *); void tsdbRemove(const char *path); int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d8de94b204..cd34262457 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -36,7 +36,8 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { int32_t vgId; - SDnode * pDnode; + SDnode *pDnode; + STfs *pTfs; uint64_t wsize; uint64_t ssize; uint64_t lsize; diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 51ddbe80aa..55e16218ad 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -50,6 +50,7 @@ struct STsdb { SMemAllocatorFactory *pmaf; STsdbFS * fs; SMeta * pMeta; + STfs * pTfs; }; #define REPO_ID(r) ((r)->vgId) diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index fb21ef56a7..3d30013382 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -54,10 +54,10 @@ typedef struct { } SMFInfo; typedef struct { - SMFInfo info; - TFILE f; - int fd; - uint8_t state; + SMFInfo info; + STfsFile f; + int fd; + uint8_t state; } SMFile; void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); @@ -175,10 +175,10 @@ typedef struct { } SDFInfo; typedef struct { - SDFInfo info; - TFILE f; - int fd; - uint8_t state; + SDFInfo info; + STfsFile f; + int fd; + uint8_t state; } SDFile; void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 7f43a93ab7..6b99b71b62 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -78,6 +78,7 @@ struct SVnode { tsem_t canCommit; SQHandle* pQuery; SDnode* pDnode; + STfs* pTfs; }; int vnodeScheduleTask(SVnodeTask* task); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5aab343c4b..1e39cc6711 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -97,8 +97,7 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { level = tsdbGetFidLevel(pSet->fid, pRtn); - tfsAllocDisk(level, &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { + if (tfsAllocDisk(pRepo->pTfs, level, &did) < 0) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; return -1; } @@ -456,8 +455,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); - tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { + if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index d890faea9b..59b680ec15 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -186,8 +186,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { } } else { // Create new fset as compacted fset - tfsAllocDisk(tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { + if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) { terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno)); tsdbCompactFSetEnd(pComph); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index b874da03f4..217fe568cc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -23,14 +23,14 @@ static const char *tsdbTxnFname[] = {"current.t", "current"}; static int tsdbComparFidFSet(const void *arg1, const void *arg2); static void tsdbResetFSStatus(SFSStatus *pStatus); -static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid); +static int tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid); static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo); -static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]); +static void tsdbGetTxnFname(STfs *pTfs, int repoid, TSDB_TXN_FILE_T ftype, char fname[]); static int tsdbOpenFSFromCurrent(STsdb *pRepo); static int tsdbScanAndTryFixFS(STsdb *pRepo); static int tsdbScanRootDir(STsdb *pRepo); static int tsdbScanDataDir(STsdb *pRepo); -static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); +static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf); static int tsdbRestoreCurrent(STsdb *pRepo); static int tsdbComparTFILE(const void *arg1, const void *arg2); static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); @@ -311,7 +311,7 @@ int tsdbOpenFS(STsdb *pRepo) { ASSERT(pfs != NULL); - tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + tsdbGetTxnFname(pRepo->pTfs, REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); tsdbGetRtnSnap(pRepo, &pRepo->rtn); if (access(current, F_OK) == 0) { @@ -375,7 +375,7 @@ int tsdbEndFSTxn(STsdb *pRepo) { SFSStatus *pStatus; // Write current file system snapshot - if (tsdbSaveFSStatus(pfs->nstatus, REPO_ID(pRepo)) < 0) { + if (tsdbSaveFSStatus(pRepo->pTfs, pfs->nstatus, REPO_ID(pRepo)) < 0) { tsdbEndFSTxnWithError(pfs); return -1; } @@ -405,7 +405,7 @@ int tsdbEndFSTxnWithError(STsdbFS *pfs) { int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); } -static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { +static int tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid) { SFSHeader fsheader; void * pBuf = NULL; void * ptr; @@ -413,8 +413,8 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { char tfname[TSDB_FILENAME_LEN] = "\0"; char cfname[TSDB_FILENAME_LEN] = "\0"; - tsdbGetTxnFname(vid, TSDB_TXN_TEMP_FILE, tfname); - tsdbGetTxnFname(vid, TSDB_TXN_CURR_FILE, cfname); + tsdbGetTxnFname(pTfs, vid, TSDB_TXN_TEMP_FILE, tfname); + tsdbGetTxnFname(pTfs, vid, TSDB_TXN_CURR_FILE, cfname); int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); if (fd < 0) { @@ -645,8 +645,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } } -static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]); +static void tsdbGetTxnFname(STfs *pTfs, int repoid, TSDB_TXN_FILE_T ftype, char fname[]) { + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(pTfs), repoid, tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -657,7 +657,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { char current[TSDB_FILENAME_LEN] = "\0"; void * ptr; - tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + tsdbGetTxnFname(pRepo->pTfs, REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); // current file exists, try to recover fd = open(current, O_RDONLY | O_BINARY); @@ -910,17 +910,17 @@ static int tsdbScanRootDir(STsdb *pRepo) { char rootDir[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN]; STsdbFS * pfs = REPO_FS(pRepo); - const TFILE *pf; + const STfsFile *pf; tsdbGetRootDir(REPO_ID(pRepo), rootDir); - TDIR *tdir = tfsOpendir(rootDir); + STfsDir *tdir = tfsOpendir(rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); return -1; } while ((pf = tfsReaddir(tdir))) { - tfsbasename(pf, bname); + tfsBasename(pf, bname); if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) { // Skip current file and data directory @@ -944,17 +944,17 @@ static int tsdbScanDataDir(STsdb *pRepo) { char dataDir[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN]; STsdbFS * pfs = REPO_FS(pRepo); - const TFILE *pf; + const STfsFile *pf; tsdbGetDataDir(REPO_ID(pRepo), dataDir); - TDIR *tdir = tfsOpendir(dataDir); + STfsDir *tdir = tfsOpendir(dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); return -1; } while ((pf = tfsReaddir(tdir))) { - tfsbasename(pf, bname); + tfsBasename(pf, bname); if (!tsdbIsTFileInFS(pfs, pf)) { (void)tfsremove(pf); @@ -967,7 +967,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { return 0; } -static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { +static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) { SFSIter fsiter; tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); SDFileSet *pSet; @@ -987,8 +987,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { // static int tsdbRestoreMeta(STsdb *pRepo) { // char rootDir[TSDB_FILENAME_LEN]; // char bname[TSDB_FILENAME_LEN]; -// TDIR * tdir = NULL; -// const TFILE *pf = NULL; +// STfsDir * tdir = NULL; +// const STfsFile *pf = NULL; // const char * pattern = "^meta(-ver[0-9]+)?$"; // regex_t regex; // STsdbFS * pfs = REPO_FS(pRepo); @@ -1007,7 +1007,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { // } // while ((pf = tfsReaddir(tdir))) { -// tfsbasename(pf, bname); +// tfsBasename(pf, bname); // if (strcmp(bname, "data") == 0) { // // Skip the data/ directory @@ -1108,8 +1108,8 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { static int tsdbRestoreDFileSet(STsdb *pRepo) { char dataDir[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN]; - TDIR * tdir = NULL; - const TFILE *pf = NULL; + STfsDir * tdir = NULL; + const STfsFile *pf = NULL; const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$"; SArray * fArray = NULL; regex_t regex; @@ -1120,7 +1120,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); - fArray = taosArrayInit(1024, sizeof(TFILE)); + fArray = taosArrayInit(1024, sizeof(STfsFile)); if (fArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, @@ -1139,7 +1139,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } while ((pf = tfsReaddir(tdir))) { - tfsbasename(pf, bname); + tfsBasename(pf, bname); int code = regexec(®ex, bname, 0, NULL, 0); if (code == 0) { @@ -1200,7 +1200,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { uint32_t tversion; char _bname[TSDB_FILENAME_LEN]; - tfsbasename(pf, _bname); + tfsBasename(pf, _bname); tsdbParseDFilename(_bname, &tvid, &tfid, &ttype, &tversion); ASSERT(tvid == REPO_ID(pRepo)); @@ -1287,7 +1287,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { return -1; } - if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) { + if (tsdbSaveFSStatus(pRepo->pTfs, pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) { tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } @@ -1296,8 +1296,8 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { } static int tsdbComparTFILE(const void *arg1, const void *arg2) { - TFILE *pf1 = (TFILE *)arg1; - TFILE *pf2 = (TFILE *)arg2; + STfsFile *pf1 = (STfsFile *)arg1; + STfsFile *pf2 = (STfsFile *)arg2; int vid1, fid1, vid2, fid2; TSDB_FILE_T ftype1, ftype2; @@ -1305,8 +1305,8 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) { char bname1[TSDB_FILENAME_LEN]; char bname2[TSDB_FILENAME_LEN]; - tfsbasename(pf1, bname1); - tfsbasename(pf2, bname2); + tfsBasename(pf1, bname1); + tfsBasename(pf2, bname2); tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1); tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2); diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index ab1f0294bf..adc7726780 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -16,12 +16,12 @@ #include "tsdbDef.h" static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, - SMeta *pMeta); + SMeta *pMeta, STfs *pTfs); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); -STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta) { +STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs) { STsdb *pTsdb = NULL; // Set default TSDB Options @@ -36,7 +36,7 @@ STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAl } // Create the handle - pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta); + pTsdb = tsdbNew(path, vgId, pTsdbCfg, pMAF, pMeta, pTfs); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -64,7 +64,7 @@ void tsdbRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, - SMeta *pMeta) { + SMeta *pMeta, STfs *pTfs) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); @@ -78,6 +78,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg); pTsdb->pmaf = pMAF; pTsdb->pMeta = pMeta; + pTsdb->pTfs = pTfs; pTsdb->fs = tsdbNewFS(pTsdbCfg); @@ -494,7 +495,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t } } else { // get the named file at the specified index. If not there, return 0 fname = malloc(256); - sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); + sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name); if (access(fname, F_OK) != 0) { tfree(fname); return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index ae17c9887d..76ec0c0594 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -75,6 +75,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { pVnode->vgId = pVnodeCfg->vgId; pVnode->pDnode = pVnodeCfg->pDnode; + pVnode->pTfs = pVnodeCfg->pTfs; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); @@ -109,7 +110,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta); + pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index ce1436eb29..d9134982f2 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -33,28 +33,73 @@ extern int32_t fsDebugFlag; #define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }} #define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }} #define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }} -#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} -#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} +#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }} +#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }} -// Global Definitions -#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 - -typedef struct SDisk { +typedef struct { int32_t level; int32_t id; char *path; SDiskSize size; -} SDisk; +} STfsDisk; -typedef struct STier { +typedef struct { pthread_spinlock_t lock; int32_t level; - int16_t nextid; // next disk id to allocate - int16_t ndisk; // # of disks mounted to this tier - int16_t nAvailDisks; // # of Available disks - SDisk *disks[TSDB_MAX_DISKS_PER_TIER]; + int32_t nextid; // next disk id to allocate + int32_t ndisk; // # of disks mounted to this tier + int32_t nAvailDisks; // # of Available disks + STfsDisk *disks[TFS_MAX_DISKS_PER_TIER]; SDiskSize size; -} STier; +} STfsTier; + +typedef struct { + STfsDisk *pDisk; +} SDiskIter; + +typedef struct STfsDir { + SDiskIter *iter; + SDiskID did; + char dirname[TSDB_FILENAME_LEN]; + STfsFile tfile; + DIR *dir; + STfs *pTfs; +} STfsDir; + +typedef struct STfs { + pthread_spinlock_t lock; + SDiskSize size; + int32_t nlevel; + STfsTier tiers[TFS_MAX_TIERS]; + SHashObj *hash; // name to did map +} STfs; + +STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); +STfsDisk *tfsFreeDisk(STfsDisk *pDisk); +int32_t tfsUpdateDiskSize(STfsDisk *pDisk); + +int32_t tfsInitTier(STfsTier *pTier, int32_t level); +void tfsDestroyTier(STfsTier *pTier); +STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg); +void tfsUpdateTierSize(STfsTier *pTier); +int32_t tfsAllocDiskOnTier(STfsTier *pTier); +void tfsPosNextId(STfsTier *pTier); + +#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) +#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) + +#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) + +#define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock) +#define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock) + +#define TFS_TIER_AT(pTfs, level) (&pTfs->tiers[level]) +#define TFS_DISK_AT(pTfs, did) (pTfs->tiers[(did).level].disks[(did).id]) +#define TFS_PRIMARY_DISK(pTfs) (pTfs->tiers[0].disks[0]) + +#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL())) +#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level)))) +#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id)) #define TIER_LEVEL(pt) ((pt)->level) #define TIER_NDISKS(pt) ((pt)->ndisk) @@ -64,17 +109,6 @@ typedef struct STier { #define DISK_AT_TIER(pt, id) ((pt)->disks[id]) #define DISK_DIR(pd) ((pd)->path) -SDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir); -SDisk *tfsFreeDisk(SDisk *pDisk); -int32_t tfsUpdateDiskSize(SDisk *pDisk); - -int32_t tfsInitTier(STier *pTier, int32_t level); -void tfsDestroyTier(STier *pTier); -SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); -void tfsUpdateTierSize(STier *pTier); -int32_t tfsAllocDiskOnTier(STier *pTier); -void tfsPosNextId(STier *pTier); - #ifdef __cplusplus } #endif diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 1d11cd6df2..37a1b58247 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -16,227 +16,193 @@ #define _DEFAULT_SOURCE #include "tfsInt.h" -#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) +static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg); +static int32_t tfsCheck(STfs *pTfs); +static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg); +static int32_t tfsFormatDir(char *idir, char *odir); +static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir); +static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir); +static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter); -typedef struct { - pthread_spinlock_t lock; - SFSMeta meta; - int32_t nlevel; - STier tiers[TSDB_MAX_TIERS]; - SHashObj *map; // name to did map -} SFS; - -typedef struct { - SDisk *pDisk; -} SDiskIter; - -#define TFS_META() (pfs->meta) -#define TFS_NLEVEL() (pfs->nlevel) -#define TFS_TIERS() (pfs->tiers) -#define TFS_TIER_AT(level) (TFS_TIERS() + (level)) -#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id) -#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID) -#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL())) -#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level)))) -#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id)) - -#define tfsLock() pthread_spin_lock(&(pfs->lock)) -#define tfsUnLock() pthread_spin_unlock(&(pfs->lock)) - -static SFS tfs = {0}; -static SFS *pfs = &tfs; - -// STATIC DECLARATION -static int32_t tfsMount(SDiskCfg *pCfg); -static int32_t tfsCheck(); -static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg); -static int32_t tfsFormatDir(char *idir, char *odir); -static SDisk *tfsGetDiskByID(SDiskID did); -static SDisk *tfsGetDiskByName(const char *dir); -static int32_t tfsOpendirImpl(TDIR *tdir); -static void tfsInitDiskIter(SDiskIter *pIter); -static SDisk *tfsNextDisk(SDiskIter *pIter); - -// FS APIs ==================================== -int32_t tfsInit(SDiskCfg *pDiskCfg, int32_t ndisk) { - if (ndisk < 0) { +STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { + if (ndisk < 0 || ndisk > TFS_MAX_DISKS) { terrno = TSDB_CODE_INVALID_PARA; - return -1; + return NULL; } - for (int32_t level = 0; level < TSDB_MAX_TIERS; level++) { - if (tfsInitTier(TFS_TIER_AT(level), level) < 0) { - while (true) { - level--; - if (level < 0) break; + STfs *pTfs = calloc(1, sizeof(STfs)); + if (pTfs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - tfsDestroyTier(TFS_TIER_AT(level)); - } + if (pthread_spin_init(&pTfs->lock, 0) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tfsClose(pTfs); + return NULL; + } - return -1; + for (int32_t level = 0; level < TFS_MAX_TIERS; level++) { + STfsTier *pTier = &pTfs->tiers[level]; + if (tfsInitTier(pTier, level) < 0) { + tfsClose(pTfs); + return NULL; } } - pthread_spin_init(&(pfs->lock), 0); - - pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (pfs->map == NULL) { + pTfs->hash = taosHashInit(TFS_MAX_DISKS * 2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pTfs->hash == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tfsCleanup(); - return -1; + tfsClose(pTfs); + return NULL; } for (int32_t idisk = 0; idisk < ndisk; idisk++) { - if (tfsMount(pDiskCfg + idisk) < 0) { - tfsCleanup(); - return -1; + if (tfsMount(pTfs, &pCfg[idisk]) < 0) { + tfsClose(pTfs); + return NULL; } } - if (tfsCheck() < 0) { - tfsCleanup(); - return -1; + if (tfsCheck(pTfs) < 0) { + tfsClose(pTfs); + return NULL; } - tfsUpdateSize(NULL); - for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - tfsPosNextId(TFS_TIER_AT(level)); + tfsUpdateSize(pTfs); + for (int32_t level = 0; level < pTfs->nlevel; level++) { + tfsPosNextId(&pTfs->tiers[level]); } - return 0; + return pTfs; } -void tfsCleanup() { - taosHashCleanup(pfs->map); - pfs->map = NULL; +void tfsClose(STfs *pTfs) { + if (pTfs == NULL) return; - pthread_spin_destroy(&(pfs->lock)); - for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - tfsDestroyTier(TFS_TIER_AT(level)); + for (int32_t level = 0; level < TFS_MAX_LEVEL; level++) { + tfsDestroyTier(&pTfs->tiers[level]); } + + taosHashCleanup(pTfs->hash); + pthread_spin_destroy(&pTfs->lock); + free(pTfs); } -void tfsUpdateSize(SFSMeta *pFSMeta) { - SFSMeta fsMeta = {0}; +void tfsUpdateSize(STfs *pTfs) { SDiskSize size = {0}; - if (pFSMeta == NULL) { - pFSMeta = &fsMeta; - } - - memset(pFSMeta, 0, sizeof(SFSMeta)); - - for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - STier *pTier = TFS_TIER_AT(level); + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = &pTfs->tiers[level]; tfsUpdateTierSize(pTier); - pFSMeta->total += pTier->size.total; - pFSMeta->avail += pTier->size.avail; - pFSMeta->used += pTier->size.used; + size.total += pTier->size.total; + size.avail += pTier->size.avail; + size.used += pTier->size.used; } - tfsLock(); - pfs->meta = *pFSMeta; - tfsUnLock(); + tfsLock(pTfs); + pTfs->size = size; + tfsUnLock(pTfs); } -/* Allocate an existing available tier level - */ -void tfsAllocDisk(int32_t expLevel, int32_t *level, int32_t *id) { - ASSERT(expLevel >= 0); +SDiskSize tfsGetSize(STfs *pTfs) { + tfsLock(pTfs); + SDiskSize size = pTfs->size; + tfsUnLock(pTfs); - *level = expLevel; - *id = TFS_UNDECIDED_ID; + return size; +} - if (*level >= TFS_NLEVEL()) { - *level = TFS_NLEVEL() - 1; +int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { + pDiskId->level = expLevel; + pDiskId->id = -1; + + if (pDiskId->level >= pTfs->nlevel) { + pDiskId->level--; } - while (*level >= 0) { - *id = tfsAllocDiskOnTier(TFS_TIER_AT(*level)); - if (*id == TFS_UNDECIDED_ID) { - (*level)--; + while (pDiskId->level >= 0) { + pDiskId->id = tfsAllocDiskOnTier(&pTfs->tiers[pDiskId->level]); + if (pDiskId->id < 0) { + pDiskId->level--; continue; } - return; + return 0; } - *level = TFS_UNDECIDED_LEVEL; - *id = TFS_UNDECIDED_ID; + terrno = TSDB_CODE_FS_NO_VALID_DISK; + return -1; } -const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); } -const char *TFS_DISK_PATH(int32_t level, int32_t id) { return DISK_DIR(TFS_DISK_AT(level, id)); } +const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; } -// TFILE APIs ==================================== -void tfsInitFile(TFILE *pf, int32_t level, int32_t id, const char *bname) { - ASSERT(TFS_IS_VALID_DISK(level, id)); +const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId) { return TFS_DISK_AT(pTfs, diskId)->path; } - SDisk *pDisk = TFS_DISK_AT(level, id); +void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname) { + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); - pf->level = level; - pf->id = id; - tstrncpy(pf->rname, bname, TSDB_FILENAME_LEN); + pFile->did = diskId; + tstrncpy(pFile->rname, rname, TSDB_FILENAME_LEN); char tmpName[TMPNAME_LEN] = {0}; - snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname); - tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN); + snprintf(tmpName, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TD_DIRSEP, rname); + tstrncpy(pFile->aname, tmpName, TSDB_FILENAME_LEN); + pFile->pTfs = pTfs; } -bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) { - ASSERT(pf1 != NULL || pf2 != NULL); - if (pf1 == NULL || pf2 == NULL) return false; - if (pf1->level != pf2->level) return false; - if (pf1->id != pf2->id) return false; - if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false; +bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { + ASSERT(pFile1 != NULL || pFile2 != NULL); + if (pFile1 == NULL || pFile2 == NULL || pFile1->pTfs != pFile2->pTfs) return false; + if (pFile1->did.level != pFile2->did.level) return false; + if (pFile1->did.id != pFile2->did.id) return false; + if (strncmp(pFile1->rname, pFile2->rname, TSDB_FILENAME_LEN) != 0) return false; return true; } -int32_t tfsEncodeFile(void **buf, TFILE *pf) { +int32_t tfsEncodeFile(void **buf, STfsFile *pFile) { int32_t tlen = 0; - tlen += taosEncodeVariantI32(buf, pf->level); - tlen += taosEncodeVariantI32(buf, pf->id); - tlen += taosEncodeString(buf, pf->rname); + tlen += taosEncodeVariantI32(buf, pFile->did.level); + tlen += taosEncodeVariantI32(buf, pFile->did.id); + tlen += taosEncodeString(buf, pFile->rname); return tlen; } -void *tfsDecodeFile(void *buf, TFILE *pf) { - int32_t level, id; - char *rname; +void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile) { + SDiskID diskId = {0}; + char *rname = NULL; - buf = taosDecodeVariantI32(buf, &(level)); - buf = taosDecodeVariantI32(buf, &(id)); + buf = taosDecodeVariantI32(buf, &diskId.level); + buf = taosDecodeVariantI32(buf, &diskId.id); buf = taosDecodeString(buf, &rname); - tfsInitFile(pf, level, id, rname); + tfsInitFile(pTfs, pFile, diskId, rname); tfree(rname); return buf; } -void tfsbasename(const TFILE *pf, char *dest) { +void tfsBasename(const STfsFile *pFile, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; - tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); + tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN); } -void tfsdirname(const TFILE *pf, char *dest) { +void tfsDirname(const STfsFile *pFile, char *dest) { char tname[TSDB_FILENAME_LEN] = "\0"; - tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); + tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } -// DIR APIs ==================================== -int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) { - SDisk *pDisk = TFS_DISK_AT(level, id); - char aname[TMPNAME_LEN]; +int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) { + STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + char aname[TMPNAME_LEN]; - snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); + snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); if (taosMkDir(aname) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -245,8 +211,8 @@ int32_t tfsMkdirAt(const char *rname, int32_t level, int32_t id) { return 0; } -int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { - if (tfsMkdirAt(rname, level, id) < 0) { +int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { + if (tfsMkdirAt(pTfs, rname, diskId) < 0) { if (errno == ENOENT) { // Try to create upper char *s = strdup(rname); @@ -259,7 +225,7 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html char *dir = strdup(dirname(s)); - if (tfsMkdirRecurAt(dir, level, id) < 0) { + if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { free(s); free(dir); return -1; @@ -267,7 +233,7 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { free(s); free(dir); - if (tfsMkdirAt(rname, level, id) < 0) { + if (tfsMkdirAt(pTfs, rname, diskId) < 0) { return -1; } } else { @@ -278,11 +244,12 @@ int32_t tfsMkdirRecurAt(const char *rname, int32_t level, int32_t id) { return 0; } -int32_t tfsMkdir(const char *rname) { - for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - STier *pTier = TFS_TIER_AT(level); - for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { - if (tfsMkdirAt(rname, level, id) < 0) { +int32_t tfsMkdir(STfs *pTfs, const char *rname) { + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + SDiskID did = {.id = id, .level = level}; + if (tfsMkdirAt(pTfs, rname, did) < 0) { return -1; } } @@ -291,16 +258,14 @@ int32_t tfsMkdir(const char *rname) { return 0; } -int32_t tfsRmdir(const char *rname) { +int32_t tfsRmdir(STfs *pTfs, const char *rname) { char aname[TMPNAME_LEN] = "\0"; - for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - STier *pTier = TFS_TIER_AT(level); - for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { - SDisk *pDisk = pTier->disks[id]; - - snprintf(aname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TS_PATH_DELIMITER, rname); - + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + STfsDisk *pDisk = pTier->disks[id]; + snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); taosRemoveDir(aname); } } @@ -308,117 +273,108 @@ int32_t tfsRmdir(const char *rname) { return 0; } -#if 0 -int32_t tfsRename(char *orname, char *nrname) { +int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { char oaname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0"; - for (int32_t level = 0; level < pfs->nlevel; level++) { - STier *pTier = TFS_TIER_AT(level); - for (int32_t id = 0; id < TIER_NDISKS(pTier); id++) { - SDisk *pDisk = DISK_AT_TIER(pTier, id); - - snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname); - snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname); - - taosRenameFile(oaname, naname); + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = TFS_TIER_AT(pTfs, level); + for (int32_t id = 0; id < pTier->ndisk; id++) { + STfsDisk *pDisk = pTier->disks[id]; + snprintf(oaname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TD_DIRSEP, orname); + snprintf(naname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TD_DIRSEP, nrname); + if (taosRenameFile(oaname, naname) != 0) { + return -1; + } } } return 0; } -#endif -struct TDIR { - SDiskIter iter; - int32_t level; - int32_t id; - char dirname[TSDB_FILENAME_LEN]; - TFILE tfile; - DIR *dir; -}; - -TDIR *tfsOpendir(const char *rname) { - TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir)); - if (tdir == NULL) { +STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { + STfsDir *pDir = calloc(1, sizeof(STfsDir)); + if (pDir == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - tfsInitDiskIter(&(tdir->iter)); - tstrncpy(tdir->dirname, rname, TSDB_FILENAME_LEN); + SDiskID diskId = {.id = 0, .level = 0}; + pDir->iter->pDisk = TFS_DISK_AT(pTfs, diskId); + pDir->pTfs = pTfs; + tstrncpy(pDir->dirname, rname, TSDB_FILENAME_LEN); - if (tfsOpendirImpl(tdir) < 0) { - free(tdir); + if (tfsOpendirImpl(pTfs, pDir) < 0) { + free(pDir); return NULL; } - return tdir; + return pDir; } -const TFILE *tfsReaddir(TDIR *tdir) { - if (tdir == NULL || tdir->dir == NULL) return NULL; +const STfsFile *tfsReaddir(STfsDir *pDir) { + if (pDir == NULL || pDir->dir == NULL) return NULL; char bname[TMPNAME_LEN * 2] = "\0"; while (true) { struct dirent *dp = NULL; - dp = readdir(tdir->dir); + dp = readdir(pDir->dir); if (dp != NULL) { // Skip . and .. if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; - snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name); - tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname); - return &(tdir->tfile); + snprintf(bname, TMPNAME_LEN * 2, "%s/%s", pDir->dirname, dp->d_name); + tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname); + return &pDir->tfile; } - if (tfsOpendirImpl(tdir) < 0) { + if (tfsOpendirImpl(pDir->pTfs, pDir) < 0) { return NULL; } - if (tdir->dir == NULL) { + if (pDir->dir == NULL) { terrno = TSDB_CODE_SUCCESS; return NULL; } } } -void tfsClosedir(TDIR *tdir) { - if (tdir) { - if (tdir->dir != NULL) { - closedir(tdir->dir); - tdir->dir = NULL; +void tfsClosedir(STfsDir *pDir) { + if (pDir) { + if (pDir->dir != NULL) { + closedir(pDir->dir); + pDir->dir = NULL; } - free(tdir); + free(pDir); } } -// private -static int32_t tfsMount(SDiskCfg *pCfg) { - SDiskID did; - SDisk *pDisk = NULL; +static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { + if (tfsCheckAndFormatCfg(pTfs, pCfg) < 0) { + return -1; + } - if (tfsCheckAndFormatCfg(pCfg) < 0) return -1; - - did.level = pCfg->level; - pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg); + SDiskID did = {.level = pCfg->level}; + STfsDisk *pDisk = tfsMountDiskToTier(TFS_TIER_AT(pTfs, did.level), pCfg); if (pDisk == NULL) { - fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); + fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, terrstr()); return -1; } did.id = pDisk->id; - taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); - if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1; + taosHashPut(pTfs->hash, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); + if (pTfs->nlevel < pCfg->level + 1) { + pTfs->nlevel = pCfg->level + 1; + } return 0; } -static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { +static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { char dirName[TSDB_FILENAME_LEN] = "\0"; struct stat pstat; - if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) { + if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; @@ -431,7 +387,7 @@ static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { return -1; } - if (TFS_PRIMARY_DISK() != NULL) { + if (TFS_PRIMARY_DISK(pTfs) != NULL) { fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir); terrno = TSDB_CODE_FS_DUP_PRIMARY; return -1; @@ -444,7 +400,7 @@ static int32_t tfsCheckAndFormatCfg(SDiskCfg *pCfg) { return -1; } - if (tfsGetDiskByName(dirName) != NULL) { + if (tfsGetDiskByName(pTfs, dirName) != NULL) { fError("failed to mount %s to FS since duplicate mount", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; @@ -494,15 +450,15 @@ static int32_t tfsFormatDir(char *idir, char *odir) { return 0; } -static int32_t tfsCheck() { - if (TFS_PRIMARY_DISK() == NULL) { +static int32_t tfsCheck(STfs *pTfs) { + if (TFS_PRIMARY_DISK(pTfs) == NULL) { fError("no primary disk is set"); terrno = TSDB_CODE_FS_NO_PRIMARY_DISK; return -1; } - for (int32_t level = 0; level < TFS_NLEVEL(); level++) { - if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) { + for (int32_t level = 0; level < pTfs->nlevel; level++) { + if (TFS_TIER_AT(pTfs, level)->ndisk == 0) { fError("no disk at level %d", level); terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; return -1; @@ -512,66 +468,53 @@ static int32_t tfsCheck() { return 0; } -static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); } -static SDisk *tfsGetDiskByName(const char *dir) { - SDiskID did; - SDisk *pDisk = NULL; - void *pr = NULL; - - pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN)); +static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) { + void *pr = taosHashGet(pTfs->hash, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN)); if (pr == NULL) return NULL; - did = *(SDiskID *)pr; - pDisk = tfsGetDiskByID(did); - ASSERT(pDisk != NULL); + SDiskID did = *(SDiskID *)pr; + STfsDisk *pDisk = TFS_DISK_AT(pTfs, did); return pDisk; } -static int32_t tfsOpendirImpl(TDIR *tdir) { - SDisk *pDisk = NULL; - char adir[TMPNAME_LEN * 2] = "\0"; +static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) { + STfsDisk *pDisk = NULL; + char adir[TMPNAME_LEN * 2] = "\0"; - if (tdir->dir != NULL) { - closedir(tdir->dir); - tdir->dir = NULL; + if (pDir->dir != NULL) { + closedir(pDir->dir); + pDir->dir = NULL; } while (true) { - pDisk = tfsNextDisk(&(tdir->iter)); + pDisk = tfsNextDisk(pTfs, pDir->iter); if (pDisk == NULL) return 0; - tdir->level = pDisk->level; - tdir->id = pDisk->id; + pDir->did.level = pDisk->level; + pDir->did.id = pDisk->id; - snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TS_PATH_DELIMITER,tdir->dirname); - tdir->dir = opendir(adir); - if (tdir->dir != NULL) break; + snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + pDir->dir = opendir(adir); + if (pDir->dir != NULL) break; } return 0; } -static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); } +static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) { + if (pIter == NULL) return NULL; -static SDisk *tfsNextDisk(SDiskIter *pIter) { - SDisk *pDisk = pIter->pDisk; + STfsDisk *pDisk = pIter->pDisk; + SDiskID did = {.level = pDisk->level, .id = pDisk->id + 1}; - if (pDisk == NULL) return NULL; - - int32_t level = pDisk->level; - int32_t id = pDisk->id; - - id++; - if (id < TIER_NDISKS(TFS_TIER_AT(level))) { - pIter->pDisk = TFS_DISK_AT(level, id); - ASSERT(pIter->pDisk != NULL); + if (did.id < TFS_TIER_AT(pTfs, did.level)->ndisk) { + pIter->pDisk = TFS_DISK_AT(pTfs, did); } else { - level++; - id = 0; - if (level < TFS_NLEVEL()) { - pIter->pDisk = TFS_DISK_AT(level, id); - ASSERT(pIter->pDisk != NULL); + did.level++; + did.id = 0; + if (did.level < pTfs->nlevel) { + pIter->pDisk = TFS_DISK_AT(pTfs, did); } else { pIter->pDisk = NULL; } @@ -579,25 +522,3 @@ static SDisk *tfsNextDisk(SDiskIter *pIter) { return pDisk; } - -// OTHER FUNCTIONS =================================== -void taosGetDisk() { - const double unit = 1024 * 1024 * 1024; - SDiskSize diskSize; - SFSMeta fsMeta; - - tfsUpdateSize(&fsMeta); - tsTotalDataDirGB = (float)(fsMeta.total / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - - if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.total / unit); - tsAvailLogDirGB = (float)(diskSize.avail / unit); - } - - if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.total / unit); - tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); - } -} diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index a5ef1121ff..7046530aeb 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "tfsInt.h" -SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { - SDisk *pDisk = calloc(1, sizeof(SDisk)); +STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { + STfsDisk *pDisk = calloc(1, sizeof(STfsDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -36,7 +36,7 @@ SDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { return pDisk; } -SDisk *tfsFreeDisk(SDisk *pDisk) { +STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { if (pDisk != NULL) { free(pDisk->path); free(pDisk); @@ -45,7 +45,7 @@ SDisk *tfsFreeDisk(SDisk *pDisk) { return NULL; } -int32_t tfsUpdateDiskSize(SDisk *pDisk) { +int32_t tfsUpdateDiskSize(STfsDisk *pDisk) { if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr()); diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 2a26c23ead..a28cd74f1a 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -16,11 +16,8 @@ #define _DEFAULT_SOURCE #include "tfsInt.h" -#define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) -#define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) - -int32_t tfsInitTier(STier *pTier, int32_t level) { - memset(pTier, 0, sizeof(STier)); +int32_t tfsInitTier(STfsTier *pTier, int32_t level) { + memset(pTier, 0, sizeof(STfsTier)); if (pthread_spin_init(&pTier->lock, 0) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -31,17 +28,17 @@ int32_t tfsInitTier(STier *pTier, int32_t level) { return 0; } -void tfsDestroyTier(STier *pTier) { - for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { +void tfsDestroyTier(STfsTier *pTier) { + for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; id++) { pTier->disks[id] = tfsFreeDisk(pTier->disks[id]); } pTier->ndisk = 0; - pthread_spin_destroy(&(pTier->lock)); + pthread_spin_destroy(&pTier->lock); } -SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { - if (pTier->ndisk >= TSDB_MAX_DISKS_PER_TIER) { +STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { + if (pTier->ndisk >= TFS_MAX_DISKS_PER_TIER) { terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; return NULL; } @@ -61,12 +58,12 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { id = pTier->ndisk; } - if (id >= TSDB_MAX_DISKS_PER_TIER) { + if (id >= TFS_MAX_DISKS_PER_TIER) { terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; return NULL; } - SDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); + STfsDisk *pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); if (pDisk == NULL) return NULL; pTier->disks[id] = pDisk; @@ -76,14 +73,14 @@ SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { return pTier->disks[id]; } -void tfsUpdateTierSize(STier *pTier) { +void tfsUpdateTierSize(STfsTier *pTier) { SDiskSize size = {0}; - int16_t nAvailDisks = 0; + int32_t nAvailDisks = 0; tfsLockTier(pTier); for (int32_t id = 0; id < pTier->ndisk; id++) { - SDisk *pDisk = pTier->disks[id]; + STfsDisk *pDisk = pTier->disks[id]; if (pDisk == NULL) continue; size.total += pDisk->size.total; @@ -99,7 +96,7 @@ void tfsUpdateTierSize(STier *pTier) { } // Round-Robin to allocate disk on a tier -int32_t tfsAllocDiskOnTier(STier *pTier) { +int32_t tfsAllocDiskOnTier(STfsTier *pTier) { terrno = TSDB_CODE_FS_NO_VALID_DISK; tfsLockTier(pTier); @@ -110,9 +107,9 @@ int32_t tfsAllocDiskOnTier(STier *pTier) { } int32_t retId = -1; - for (int32_t id = 0; id < TSDB_MAX_DISKS_PER_TIER; ++id) { - int32_t diskId = (pTier->nextid + id) % pTier->ndisk; - SDisk *pDisk = pTier->disks[diskId]; + for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) { + int32_t diskId = (pTier->nextid + id) % pTier->ndisk; + STfsDisk *pDisk = pTier->disks[diskId]; if (pDisk == NULL) continue; @@ -128,12 +125,12 @@ int32_t tfsAllocDiskOnTier(STier *pTier) { return retId; } -void tfsPosNextId(STier *pTier) { +void tfsPosNextId(STfsTier *pTier) { int32_t nextid = 0; for (int32_t id = 1; id < pTier->ndisk; id++) { - SDisk *pLDisk = pTier->disks[nextid]; - SDisk *pDisk = pTier->disks[id]; + STfsDisk *pLDisk = pTier->disks[nextid]; + STfsDisk *pDisk = pTier->disks[id]; if (pDisk->size.avail > TFS_MIN_DISK_FREE_SIZE && pDisk->size.avail > pLDisk->size.avail) { nextid = id; } From cc5563716c5eccc47638c0fa9dca5efa59e9c830 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Jan 2022 22:04:01 -0800 Subject: [PATCH 032/103] minor changes --- source/dnode/vnode/inc/tsdb.h | 1 + source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/inc/tsdbFile.h | 10 +++++----- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 ++++---- source/dnode/vnode/src/tsdb/tsdbFS.c | 14 +++++++------- source/dnode/vnode/src/tsdb/tsdbFile.c | 18 +++++++++--------- source/libs/tfs/src/tfs.c | 2 +- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index e56a6bece3..cfd8e501f6 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -19,6 +19,7 @@ #include "mallocator.h" #include "meta.h" #include "common.h" +#include "tfs.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cd34262457..b66fc413db 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -21,6 +21,7 @@ #include "meta.h" #include "tarray.h" +#include "tfs.h" #include "tq.h" #include "tsdb.h" #include "wal.h" diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 3d30013382..8827af5b0a 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -29,7 +29,7 @@ #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) #define TSDB_FILE_FD(tf) ((tf)->fd) -#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf)) +#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) #define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) #define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) @@ -181,10 +181,10 @@ typedef struct { uint8_t state; } SDFile; -void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); +void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); int tsdbEncodeSDFile(void** buf, SDFile* pDFile); -void* tsdbDecodeSDFile(void* buf, SDFile* pDFile); +void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile); int tsdbCreateDFile(SDFile* pDFile, bool updateHeader); int tsdbUpdateDFileHeader(SDFile* pDFile); int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); @@ -311,10 +311,10 @@ typedef struct { } \ } while (0); -void tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver); +void tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver); void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet); int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet); -void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet); +void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet); int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1e39cc6711..ef7ef60506 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -104,7 +104,7 @@ int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { if (did.level > TSDB_FSET_LEVEL(pSet)) { // Need to move the FSET to higher level - tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); + tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); if (tsdbCopyDFileSet(pSet, &nSet) < 0) { tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, @@ -482,7 +482,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // Set and open commit FSET if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { // Create a new FSET to write data - tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); + tsdbInitDFileSet(pRepo, pWSet, did, fid, FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateDFileSet(pWSet, true) < 0) { tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), @@ -507,7 +507,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // TSDB_FILE_HEAD SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); - tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); + tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); if (tsdbCreateDFile(pWHeadf, true) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), tstrerror(terrno)); @@ -554,7 +554,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid } } } else { - tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); + tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); pCommith->isLFileSame = false; if (tsdbCreateDFile(pWLastf, true) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 217fe568cc..54d6caaa1d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -97,7 +97,7 @@ static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) { return tlen; } -static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) { +static void *tsdbDecodeDFileSetArray(STsdb*pRepo, void *buf, SArray *pArray) { uint64_t nset; SDFileSet dset; @@ -105,7 +105,7 @@ static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) { buf = taosDecodeFixedU64(buf, &nset); for (size_t i = 0; i < nset; i++) { - buf = tsdbDecodeDFileSet(buf, &dset); + buf = tsdbDecodeDFileSet(pRepo, buf, &dset); taosArrayPush(pArray, (void *)(&dset)); } return buf; @@ -122,13 +122,13 @@ static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) { return tlen; } -static void *tsdbDecodeFSStatus(void *buf, SFSStatus *pStatus) { +static void *tsdbDecodeFSStatus(STsdb*pRepo, void *buf, SFSStatus *pStatus) { tsdbResetFSStatus(pStatus); // pStatus->pmf = &(pStatus->mf); // buf = tsdbDecodeSMFile(buf, pStatus->pmf); - buf = tsdbDecodeDFileSetArray(buf, pStatus->df); + buf = tsdbDecodeDFileSetArray(pRepo, buf, pStatus->df); return buf; } @@ -725,7 +725,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { } ptr = buffer; - ptr = tsdbDecodeFSStatus(ptr, pStatus); + ptr = tsdbDecodeFSStatus(pRepo, ptr, pStatus); } else { tsdbResetFSStatus(pStatus); } @@ -913,7 +913,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { const STfsFile *pf; tsdbGetRootDir(REPO_ID(pRepo), rootDir); - STfsDir *tdir = tfsOpendir(rootDir); + STfsDir *tdir = tfsOpendir(pRepo->pTfs, rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); return -1; @@ -947,7 +947,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { const STfsFile *pf; tsdbGetDataDir(REPO_ID(pRepo), dataDir); - STfsDir *tdir = tfsOpendir(dataDir); + STfsDir *tdir = tfsOpendir(pRepo->pTfs, dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index a1c1b57b44..1d5c39b41d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -295,7 +295,7 @@ static int tsdbRollBackMFile(SMFile *pMFile) { #endif // ============== Operations on SDFile -void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) { +void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) { char fname[TSDB_FILENAME_LEN]; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK); @@ -305,8 +305,8 @@ void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, memset(&(pDFile->info), 0, sizeof(pDFile->info)); pDFile->info.magic = TSDB_FILE_INIT_MAGIC; - tsdbGetFilename(vid, fid, ver, ftype, fname); - tfsInitFile(&(pDFile->f), did.level, did.id, fname); + tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname); + tfsInitFile(pRepo->pTfs, &(pDFile->f), did, fname); } void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) { @@ -323,9 +323,9 @@ int tsdbEncodeSDFile(void **buf, SDFile *pDFile) { return tlen; } -void *tsdbDecodeSDFile(void *buf, SDFile *pDFile) { +void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile) { buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); - buf = tfsDecodeFile(buf, &(pDFile->f)); + buf = tfsDecodeFile(pRepo->pTfs, buf, &(pDFile->f)); TSDB_FILE_SET_CLOSED(pDFile); return buf; @@ -559,13 +559,13 @@ static int tsdbRollBackDFile(SDFile *pDFile) { } // ============== Operations on SDFileSet -void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver) { +void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver) { pSet->fid = fid; pSet->state = 0; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); - tsdbInitDFile(pDFile, did, vid, fid, ver, ftype); + tsdbInitDFile(pRepo->pTfs, pDFile, did, fid, ver, ftype); } } @@ -587,14 +587,14 @@ int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { return tlen; } -void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet) { +void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) { int32_t fid; buf = taosDecodeFixedI32(buf, &(fid)); pSet->state = 0; pSet->fid = fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); + buf = tsdbDecodeSDFile(pRepo->pTfs, buf, TSDB_DFILE_IN_SET(pSet, ftype)); } return buf; } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 37a1b58247..bcccf87ef7 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -146,7 +146,7 @@ void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname) tstrncpy(pFile->rname, rname, TSDB_FILENAME_LEN); char tmpName[TMPNAME_LEN] = {0}; - snprintf(tmpName, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TD_DIRSEP, rname); + snprintf(tmpName, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); tstrncpy(pFile->aname, tmpName, TSDB_FILENAME_LEN); pFile->pTfs = pTfs; } From ae9dc0be57ef69e8272859c554d4c93207ca2463 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Jan 2022 22:15:59 -0800 Subject: [PATCH 033/103] minor changes --- source/dnode/mgmt/impl/src/dndVnodes.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index c174de9893..01179b38ae 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -841,6 +841,7 @@ static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) { + dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId); if (pMsg->msgType & 1u) { SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rsp); From 4b29cf3748dc87ed17207b1fe54afccaafc8c9d7 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 19 Jan 2022 14:32:13 +0800 Subject: [PATCH 034/103] [modify] --- tests/test/c/create_table.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 60d2321a7d..d8fced3011 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -204,7 +204,7 @@ void *threadFunc(void *param) { int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); - if (code != 0) { + if ((code != 0) && (code != 0x0002)) { pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); } taos_free_result(pRes); @@ -227,7 +227,6 @@ void *threadFunc(void *param) { if (insertData) { int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); - ; pInfo->startMs = taosGetTimestampMs(); for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { From b86bda4b41da273f5321ffc5fd82d959927c1212 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Jan 2022 22:40:28 -0800 Subject: [PATCH 035/103] refact tfs module --- include/os/osFile.h | 4 ++-- source/dnode/vnode/inc/vnode.h | 6 +++--- source/dnode/vnode/src/inc/tsdbFile.h | 15 +++++++++------ source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 +++--- source/dnode/vnode/src/tsdb/tsdbFS.c | 20 ++++++++++---------- source/dnode/vnode/src/tsdb/tsdbFile.c | 14 +++++++------- source/libs/tfs/inc/tfsInt.h | 20 ++++---------------- source/libs/tfs/src/tfs.c | 12 ++++++++++-- source/os/src/osFile.c | 4 ++-- 9 files changed, 50 insertions(+), 51 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 2b0abc60ae..ac399fa3be 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -58,8 +58,8 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); void taosCloseFile(FileFd fd); -int32_t taosRenameFile(char *oldName, char *newName); -int64_t taosCopyFile(char *from, char *to); +int32_t taosRenameFile(const char *oldName, const char *newName); +int64_t taosCopyFile(const char *from, const char *to); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b66fc413db..499972f476 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -54,9 +54,9 @@ typedef struct SVnodeCfg { typedef struct { int32_t sver; - char * timezone; - char * locale; - char * charset; + const char *timezone; + const char *locale; + const char *charset; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeOpt; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 8827af5b0a..4a79c6358d 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -33,8 +33,11 @@ #define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) #define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) -#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf)) -#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf)) +#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) +#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) +#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) +#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) +#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) #define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) #define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) @@ -185,7 +188,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); int tsdbEncodeSDFile(void** buf, SDFile* pDFile); void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile); -int tsdbCreateDFile(SDFile* pDFile, bool updateHeader); +int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader); int tsdbUpdateDFileHeader(SDFile* pDFile); int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); @@ -263,7 +266,7 @@ static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte return (int)nbyte; } -static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); } +static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); @@ -278,7 +281,7 @@ static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nby } static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { - if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { + if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -318,7 +321,7 @@ void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet); int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); -int tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader); +int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader); int tsdbUpdateDFileSetHeader(SDFileSet* pSet); int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ef7ef60506..0f2d711a79 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -484,7 +484,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // Create a new FSET to write data tsdbInitDFileSet(pRepo, pWSet, did, fid, FS_TXN_VERSION(REPO_FS(pRepo))); - if (tsdbCreateDFileSet(pWSet, true) < 0) { + if (tsdbCreateDFileSet(pRepo, pWSet, true) < 0) { tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); if (pCommith->isRFileSet) { @@ -508,7 +508,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // TSDB_FILE_HEAD SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); - if (tsdbCreateDFile(pWHeadf, true) < 0) { + if (tsdbCreateDFile(pRepo, pWHeadf, true) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), tstrerror(terrno)); @@ -557,7 +557,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFile(pRepo, pWLastf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); pCommith->isLFileSame = false; - if (tsdbCreateDFile(pWLastf, true) < 0) { + if (tsdbCreateDFile(pRepo, pWLastf, true) < 0) { tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 54d6caaa1d..d787eebcff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -931,8 +931,8 @@ static int tsdbScanRootDir(STsdb *pRepo) { // continue; // } - (void)tfsremove(pf); - tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); + (void)tfsRemoveFile(pf); + tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), pf->aname); } tfsClosedir(tdir); @@ -957,8 +957,8 @@ static int tsdbScanDataDir(STsdb *pRepo) { tfsBasename(pf, bname); if (!tsdbIsTFileInFS(pfs, pf)) { - (void)tfsremove(pf); - tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); + (void)tfsRemoveFile(pf); + tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), pf->aname); } } @@ -1016,7 +1016,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) { // if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) { // // Skip current.t file -// tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); +// tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), pf->aname); // (void)tfsremove(pf); // continue; // } @@ -1026,7 +1026,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) { // // Match // if (pfs->cstatus->pmf != NULL) { // tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo), -// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf)); +// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), pf->aname); // terrno = TSDB_CODE_TDB_FILE_CORRUPTED; // tfsClosedir(tdir); // regfree(®ex); @@ -1081,7 +1081,7 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) { // } // } else if (code == REG_NOMATCH) { // // Not match -// tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); +// tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname); // tfsremove(pf); // continue; // } else { @@ -1129,7 +1129,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { return -1; } - tdir = tfsOpendir(dataDir); + tdir = tfsOpendir(pRepo->pTfs, dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -1152,8 +1152,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } } else if (code == REG_NOMATCH) { // Not match - tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); - (void)tfsremove(pf); + tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname); + (void)tfsRemoveFile(pf); continue; } else { // Has other error diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 1d5c39b41d..579160bcb2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -352,15 +352,15 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { return buf; } -int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) { +int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); if (pDFile->fd < 0) { if (errno == ENOENT) { // Try to create directory recursively - char *s = strdup(TFILE_REL_NAME(&(pDFile->f))); - if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) { + char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); + if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) { tfree(s); return -1; } @@ -565,7 +565,7 @@ void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint3 for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); - tsdbInitDFile(pRepo->pTfs, pDFile, did, fid, ver, ftype); + tsdbInitDFile(pRepo, pDFile, did, fid, ver, ftype); } } @@ -594,7 +594,7 @@ void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) { pSet->state = 0; pSet->fid = fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - buf = tsdbDecodeSDFile(pRepo->pTfs, buf, TSDB_DFILE_IN_SET(pSet, ftype)); + buf = tsdbDecodeSDFile(pRepo, buf, TSDB_DFILE_IN_SET(pSet, ftype)); } return buf; } @@ -633,9 +633,9 @@ int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) { return 0; } -int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) { +int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { + if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { tsdbCloseDFileSet(pSet); tsdbRemoveDFileSet(pSet); return -1; diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index d9134982f2..5ca93db47b 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -88,26 +88,14 @@ void tfsPosNextId(STfsTier *pTier); #define tfsLockTier(pTier) pthread_spin_lock(&(pTier)->lock) #define tfsUnLockTier(pTier) pthread_spin_unlock(&(pTier)->lock) -#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) - #define tfsLock(pTfs) pthread_spin_lock(&(pTfs)->lock) #define tfsUnLock(pTfs) pthread_spin_unlock(&(pTfs)->lock) -#define TFS_TIER_AT(pTfs, level) (&pTfs->tiers[level]) -#define TFS_DISK_AT(pTfs, did) (pTfs->tiers[(did).level].disks[(did).id]) -#define TFS_PRIMARY_DISK(pTfs) (pTfs->tiers[0].disks[0]) +#define TFS_TIER_AT(pTfs, level) (&(pTfs)->tiers[level]) +#define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id]) +#define TFS_PRIMARY_DISK(pTfs) ((pTfs)->tiers[0].disks[0]) -#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL())) -#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level)))) -#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id)) - -#define TIER_LEVEL(pt) ((pt)->level) -#define TIER_NDISKS(pt) ((pt)->ndisk) -#define TIER_SIZE(pt) ((pt)->tmeta.size) -#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free) - -#define DISK_AT_TIER(pt, id) ((pt)->disks[id]) -#define DISK_DIR(pd) ((pd)->path) +#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) #ifdef __cplusplus } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index bcccf87ef7..8ceae8ac96 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -198,6 +198,14 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } +int32_t tfsRemoveFile(const STfsFile *pFile) { + return remove(pFile->aname); +} + +int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { + return taosCopyFile(pFile1->aname, pFile2->aname); +} + int32_t tfsMkdirAt(STfs *pTfs, const char *rname, SDiskID diskId) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); char aname[TMPNAME_LEN]; @@ -281,8 +289,8 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { STfsTier *pTier = TFS_TIER_AT(pTfs, level); for (int32_t id = 0; id < pTier->ndisk; id++) { STfsDisk *pDisk = pTier->disks[id]; - snprintf(oaname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TD_DIRSEP, orname); - snprintf(naname, TMPNAME_LEN, "%s%s%s", DISK_DIR(pDisk), TD_DIRSEP, nrname); + snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); + snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); if (taosRenameFile(oaname, naname) != 0) { return -1; } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 60155c8fb9..5a67ab73eb 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -142,7 +142,7 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } -int64_t taosCopyFile(char *from, char *to) { +int64_t taosCopyFile(const char *from, const char *to) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else @@ -400,7 +400,7 @@ int32_t taosFsyncFile(FileFd fd) { #endif } -int32_t taosRenameFile(char *oldName, char *newName) { +int32_t taosRenameFile(const char *oldName, const char *newName) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (code < 0) { From 0cc4362837c017b7dceb59669076e23f58b64afc Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 19 Jan 2022 14:46:37 +0800 Subject: [PATCH 036/103] [modify] --- tests/script/sh/massiveTable/deployCluster.sh | 61 +++++++++++++++++-- tests/test/c/create_table.c | 2 +- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index a86e9220cd..4febe40054 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -5,6 +5,49 @@ set -e #set -x +masterDnode=slave +dataRootDir="/data" +firstEp="trd02:7000" +startPort=7000 +dnodeNumber=1 +updateSrc=no + +while getopts "hm:f:n:r:p:u:" arg +do + case $arg in + m) + masterDnode=$( echo $OPTARG ) + ;; + n) + dnodeNumber=$(echo $OPTARG) + ;; + u) + updateSrc=$(echo $OPTARG) + ;; + f) + firstEp=$(echo $OPTARG) + ;; + p) + startPort=$(echo $OPTARG) + ;; + r) + dataRootDir=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -m [if master dnode] " + echo " -n [ dnode number] " + echo " -f [ first ep] " + echo " -p [ start port] " + echo " -r [ dnode root dir] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + # deployCluster.sh curr_dir=$(readlink -f "$(dirname "$0")") echo $curr_dir @@ -12,13 +55,21 @@ echo $curr_dir ${curr_dir}/cleanCluster.sh -r "/data" ${curr_dir}/cleanCluster.sh -r "/data2" -${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" +if [[ "${updateSrc}" == "yes" ]]; then + ${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" +fi -${curr_dir}/setupDnodes.sh -r "/data" -n 1 -f "trd02:7000" -p 7000 -${curr_dir}/setupDnodes.sh -r "/data2" -n 1 -f "trd02:7000" -p 8000 +${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000 +${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 -#./setupDnodes.sh -r "/data" -n 2 -f trd02:7000 -p 7000 -#./setupDnodes.sh -r "/data2" -n 2 -f trd02:7000 -p 8000 +if [[ "${masterDnode}" == "master" ]]; then + # create all dnode into cluster + taos -s "create dnode trd02 port 8000;" + taos -s "create dnode trd03 port 7000;" + taos -s "create dnode trd03 port 8000;" + taos -s "create dnode trd04 port 7000;" + taos -s "create dnode trd04 port 8000;" +fi diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index d8fced3011..d835760423 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -205,7 +205,7 @@ void *threadFunc(void *param) { TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); if ((code != 0) && (code != 0x0002)) { - pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); + pError("failed to create table t%" PRId64 ", code: %d, reason:%s", t, code, tstrerror(code)); } taos_free_result(pRes); int64_t endTs = taosGetTimestampUs(); From b3526ee271070d4e84c4029fc072d6cc7b1e27fd Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 19 Jan 2022 15:02:31 +0800 Subject: [PATCH 037/103] [modify] --- tests/test/c/create_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index d835760423..6df31c7b8b 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -204,7 +204,7 @@ void *threadFunc(void *param) { int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); - if ((code != 0) && (code != 0x0002)) { + if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { pError("failed to create table t%" PRId64 ", code: %d, reason:%s", t, code, tstrerror(code)); } taos_free_result(pRes); From 1145d1a0ea1248f9de562400f3b9d8baf7b53bcd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 19 Jan 2022 16:20:13 +0800 Subject: [PATCH 038/103] [td-11818] select * --- source/client/src/clientImpl.c | 2 +- source/libs/executor/src/executorMain.c | 20 ++++++++++---------- source/libs/executor/src/executorimpl.c | 7 ++++++- source/libs/planner/src/physicalPlanJson.c | 8 ++++---- source/libs/qworker/src/qworker.c | 2 ++ 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8fdde9f4e9..10d1d09c85 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -259,7 +259,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr)); - SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 1}; + SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 2}; addr.epAddr[0].port = 6030; strcpy(addr.epAddr[0].fqdn, "localhost"); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 9fad5242ba..56e2977753 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -73,12 +73,12 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ assert(tsdb != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = 0; - uint64_t uid = 0; - STimeWindow window = TSWINDOW_INITIALIZER; - int32_t tableType = 0; + int32_t code = 0; + uint64_t uid = 0; + STimeWindow window = TSWINDOW_INITIALIZER; + int32_t tableType = 0; - SPhyNode *pPhyNode = pSubplan->pNode; + SPhyNode* pPhyNode = pSubplan->pNode; STableGroupInfo groupInfo = {0}; int32_t type = pPhyNode->info.type; @@ -112,10 +112,10 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ } } - code = doCreateExecTaskInfo(pSubplan, pTask, &groupInfo, tsdb); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = doCreateExecTaskInfo(pSubplan, pTask, &groupInfo, tsdb); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; code = dsDataSinkMgtInit(&cfg); @@ -127,7 +127,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ *handle = (*pTask)->dsHandle; - _error: +_error: // if failed to add ref for all tables in this query, abort current query return code; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1b3c93ab61..4cb233f71e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4952,7 +4952,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { } static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { - SOperatorInfo* pOperator = (SOperatorInfo*) param; + SOperatorInfo *pOperator = (SOperatorInfo*) param; SExchangeInfo *pExchangeInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; @@ -5012,9 +5012,14 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { size_t len = pExchangeInfo->pRsp->numOfRows * pColInfoData->info.bytes; memcpy(tmp, pData, len); + + pColInfoData->pData = tmp; pData += len; } + pRes->info.numOfCols = pOperator->numOfOutput; + pRes->info.rows = pExchangeInfo->pRsp->numOfRows; + return pExchangeInfo->pResult; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 1b6cf89c56..bf38712b13 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -559,10 +559,10 @@ static bool timeWindowFromJson(const cJSON* json, void* obj) { STimeWindow* win = (STimeWindow*)obj; char* pStartKey = getString(json, jkTimeWindowStartKey); - win->skey = strtoll(pStartKey, NULL, 10); + win->skey = strtoul(pStartKey, NULL, 10); char* pEndKey = getString(json, jkTimeWindowEndKey); - win->ekey = strtoll(pEndKey, NULL, 10); + win->ekey = strtoul(pEndKey, NULL, 10); tfree(pStartKey); tfree(pEndKey); @@ -783,7 +783,7 @@ static bool nodeAddrFromJson(const cJSON* json, void* obj) { pSource->taskId = getNumber(json, jkNodeTaskId); char* pSchedId = getString(json, jkNodeTaskSchedId); - pSource->schedId = strtoll(pSchedId, NULL, 10); + pSource->schedId = strtoul(pSchedId, NULL, 10); tfree(pSchedId); bool res = fromObject(json, jkNodeAddr, queryNodeAddrFromJson, &pSource->addr, true); @@ -1032,7 +1032,7 @@ static bool subplanIdFromJson(const cJSON* json, void* obj) { SSubplanId* id = (SSubplanId*)obj; char* queryId = getString(json, jkIdQueryId); - id->queryId = strtoll(queryId, NULL, 0); + id->queryId = strtoul(queryId, NULL, 0); tfree(queryId); id->templateId = getNumber(json, jkIdTemplateId); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index a0beaba61d..d01f4f4e52 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -258,6 +258,8 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); + printf("%"PRIx64", tid:%"PRIx64"\n", qId, tId); + SQWTaskCtx nctx = {0}; QW_LOCK(QW_WRITE, &mgmt->ctxLock); From 00ccbb4a3a87bd9d91c0cbbdd724ca36713fb2a8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 19 Jan 2022 16:28:13 +0800 Subject: [PATCH 039/103] add subscribe for sdb --- include/common/tmsg.h | 24 + include/dnode/mnode/sdb/sdb.h | 4 +- include/util/tdef.h | 1 + source/dnode/mnode/impl/inc/mndConsumer.h | 3 + source/dnode/mnode/impl/inc/mndDef.h | 238 +++++++- source/dnode/mnode/impl/inc/mndSubscribe.h | 38 ++ source/dnode/mnode/impl/src/mndConsumer.c | 267 ++------- source/dnode/mnode/impl/src/mndProfile.c | 3 + source/dnode/mnode/impl/src/mndSubscribe.c | 597 +++++++++++++++++++++ source/dnode/mnode/impl/src/mndTopic.c | 14 +- source/dnode/mnode/sdb/src/sdbHash.c | 4 +- 11 files changed, 934 insertions(+), 259 deletions(-) create mode 100644 source/dnode/mnode/impl/inc/mndSubscribe.h create mode 100644 source/dnode/mnode/impl/src/mndSubscribe.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index dfd376f1e9..7d9fd53e69 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -324,6 +324,30 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { return buf; } +typedef struct SMqSetCVgReq { + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; +} SMqSetCVgReq; + +static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pReq->vgId); + tlen += taosEncodeFixedI64(buf, pReq->consumerId); + tlen += taosEncodeString(buf, pReq->topicName); + tlen += taosEncodeString(buf, pReq->cGroup); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { + buf = taosDecodeFixedI32(buf, &pReq->vgId); + buf = taosDecodeFixedI64(buf, &pReq->consumerId); + buf = taosDecodeStringTo(buf, pReq->topicName); + buf = taosDecodeStringTo(buf, pReq->cGroup); + return buf; +} + typedef struct { int32_t vgId; char* dbName; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 5a4ac6a96f..7b022dd7c7 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,8 +113,8 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_CONSUMER = 10, - SDB_CGROUP = 11, + SDB_SUBSCRIBE = 10, + SDB_CONSUMER = 11, SDB_TOPIC = 12, SDB_VGROUP = 13, SDB_STB = 14, diff --git a/include/util/tdef.h b/include/util/tdef.h index 428de5d171..16ae60c231 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -178,6 +178,7 @@ do { \ #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_CONSUMER_GROUP_LEN 192 +#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 68ba08b66e..9d1dd084ee 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -28,6 +28,9 @@ void mndCleanupConsumer(SMnode *pMnode); SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); +SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); +SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a2d6bbf4e6..a069987368 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -326,17 +326,156 @@ typedef struct SMqTopicConsumer { #endif typedef struct SMqConsumerEp { - int32_t vgId; + int32_t vgId; // -1 for unassigned SEpSet epset; - int64_t consumerId; + int64_t consumerId; // -1 for unassigned + int64_t lastConsumerHbTs; + int64_t lastVgHbTs; } SMqConsumerEp; -typedef struct SMqCgroupTopicPair { - char key[TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN]; - SArray* assigned; // SArray - SArray* unassignedConsumer; - SArray* unassignedVg; -} SMqCgroupTopicPair; +static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); + tlen += taosEncodeSEpSet(buf, &pConsumerEp->epset); + tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { + buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); + buf = taosDecodeSEpSet(buf, &pConsumerEp->epset); + buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); + return buf; +} + +//unit for rebalance +typedef struct SMqSubscribeObj { + char key[TSDB_SUBSCRIBE_KEY_LEN]; + int32_t epoch; + //TODO: replace with priority queue + SArray* availConsumer; // SArray (consumerId) + SArray* assigned; // SArray + SArray* unassignedConsumer; // SArray + SArray* unassignedVg; // SArray +} SMqSubscribeObj; + +static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { + SMqSubscribeObj* pSub = malloc(sizeof(SMqSubscribeObj)); + pSub->key[0] = 0; + pSub->epoch = 0; + if (pSub == NULL) { + return NULL; + } + pSub->availConsumer = taosArrayInit(0, sizeof(int64_t)); + if (pSub->availConsumer == NULL) { + free(pSub); + return NULL; + } + pSub->assigned = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->assigned == NULL) { + taosArrayDestroy(pSub->availConsumer); + free(pSub); + return NULL; + } + pSub->unassignedConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->assigned == NULL) { + taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->unassignedConsumer); + free(pSub); + return NULL; + } + pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->assigned == NULL) { + taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->unassignedConsumer); + taosArrayDestroy(pSub->unassignedVg); + free(pSub); + return NULL; + } + return NULL; +} + +static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pSub->key); + tlen += taosEncodeFixedI32(buf, pSub->epoch); + int32_t sz; + + sz = taosArrayGetSize(pSub->availConsumer); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + int64_t* pConsumerId = taosArrayGet(pSub->availConsumer, i); + tlen += taosEncodeFixedI64(buf, *pConsumerId); + } + + sz = taosArrayGetSize(pSub->assigned); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp* pCEp = taosArrayGet(pSub->assigned, i); + tlen += tEncodeSMqConsumerEp(buf, pCEp); + } + + sz = taosArrayGetSize(pSub->unassignedConsumer); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedConsumer, i); + tlen += tEncodeSMqConsumerEp(buf, pCEp); + } + + sz = taosArrayGetSize(pSub->unassignedVg); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedVg, i); + tlen += tEncodeSMqConsumerEp(buf, pCEp); + } + + return tlen; +} + +static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) { + buf = taosDecodeStringTo(buf, pSub->key); + buf = taosDecodeFixedI32(buf, &pSub->epoch); + + int32_t sz; + + buf = taosDecodeFixedI32(buf, &sz); + pSub->assigned = taosArrayInit(sz, sizeof(int64_t)); + if (pSub->assigned == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + int64_t consumerId; + buf = taosDecodeFixedI64(buf, &consumerId); + taosArrayPush(pSub->assigned, &consumerId); + } + + buf = taosDecodeFixedI32(buf, &sz); + pSub->unassignedConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->unassignedConsumer == NULL) { + taosArrayDestroy(pSub->assigned); + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp cEp; + buf = tDecodeSMqConsumerEp(buf, &cEp); + taosArrayPush(pSub->unassignedConsumer, &cEp); + } + + buf = taosDecodeFixedI32(buf, &sz); + pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->unassignedVg == NULL) { + taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->unassignedConsumer); + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp cEp; + buf = tDecodeSMqConsumerEp(buf, &cEp); + taosArrayPush(pSub->unassignedVg, &cEp); + } + + return buf; +} typedef struct SMqCGroup { char name[TSDB_CONSUMER_GROUP_LEN]; @@ -358,8 +497,8 @@ typedef struct SMqTopicObj { char *sql; char *logicalPlan; char *physicalPlan; - SHashObj *cgroups; // SHashObj - SHashObj *consumers; // SHashObj + //SHashObj *cgroups; // SHashObj + //SHashObj *consumers; // SHashObj } SMqTopicObj; // TODO: add cache and change name to id @@ -367,18 +506,93 @@ typedef struct SMqConsumerTopic { char name[TSDB_TOPIC_FNAME_LEN]; int32_t epoch; //TODO: replace with something with ep - SList *vgroups; // SList + //SList *vgroups; // SList + //vg assigned to the consumer on the topic SArray *pVgInfo; // SArray } SMqConsumerTopic; +static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { + SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic)); + if (pCTopic == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + strcpy(pCTopic->name, pTopic->name); + pCTopic->epoch = 0; + pCTopic->pVgInfo = taosArrayInit(0, sizeof(int32_t)); + + int32_t unassignedVgSz = taosArrayGetSize(pSub->unassignedVg); + if (unassignedVgSz > 0) { + SMqConsumerEp* pCEp = taosArrayPop(pSub->unassignedVg); + pCEp->consumerId = consumerId; + taosArrayPush(pCTopic->pVgInfo, &pCEp->vgId); + taosArrayPush(pSub->assigned, pCEp); + } + return pCTopic; +} + +static FORCE_INLINE int32_t tEncodeSMqConsumerTopic(void** buf, SMqConsumerTopic* pConsumerTopic) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pConsumerTopic->name); + tlen += taosEncodeFixedI32(buf, pConsumerTopic->epoch); + int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + int32_t* pVgInfo = taosArrayGet(pConsumerTopic->pVgInfo, i); + tlen += taosEncodeFixedI32(buf, *pVgInfo); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqConsumerTopic(void* buf, SMqConsumerTopic* pConsumerTopic) { + buf = taosDecodeStringTo(buf, pConsumerTopic->name); + buf = taosDecodeFixedI32(buf, &pConsumerTopic->epoch); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pConsumerTopic->pVgInfo = taosArrayInit(sz, sizeof(SMqConsumerTopic)); + for (int32_t i = 0; i < sz; i++) { + int32_t vgInfo; + buf = taosDecodeFixedI32(buf, &vgInfo); + taosArrayPush(pConsumerTopic->pVgInfo, &vgInfo); + } + return buf; +} + typedef struct SMqConsumerObj { int64_t consumerId; SRWLatch lock; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray *topics; // SArray - SHashObj *topicHash; //SHashObj + //SHashObj *topicHash; //SHashObj } SMqConsumerObj; +static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeString(buf, pConsumer->cgroup); + int32_t sz = taosArrayGetSize(pConsumer->topics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerTopic* pConsumerTopic = taosArrayGet(pConsumer->topics, i); + tlen += tEncodeSMqConsumerTopic(buf, pConsumerTopic); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) { + buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeStringTo(buf, pConsumer->cgroup); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->topics = taosArrayInit(sz, sizeof(SMqConsumerObj)); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerTopic cTopic; + buf = tDecodeSMqConsumerTopic(buf, &cTopic); + taosArrayPush(pConsumer->topics, &cTopic); + } + return buf; +} + typedef struct SMqSubConsumerObj { int64_t consumerUid; // if -1, unassigned SList *vgId; // SList diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h new file mode 100644 index 0000000000..b8e651e386 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_SUBSCRIBE_H_ +#define _TD_MND_SUBSCRIBE_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitSubscribe(SMnode *pMnode); +void mndCleanupSubscribe(SMnode *pMnode); + +SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, char *CGroup, char *topicName); +void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); + +SSdbRaw *mndSubscribeActionEncode(SMqSubscribeObj *pSub); +SSdbRow *mndSubscribeActionDecode(SSdbRaw *pRaw); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_SUBSCRIBE_H_*/ diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index d27bf53a90..5cdd8e77bd 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -30,24 +30,14 @@ #define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_RESERVE_SIZE 64 -static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); -static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); -static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg); static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); - int32_t mndInitConsumer(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CONSUMER, .keyType = SDB_KEY_BINARY, @@ -57,26 +47,29 @@ int32_t mndInitConsumer(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndConsumerActionUpdate, .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/ - /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/ - mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); return sdbSetTable(pMnode->pSdb, table); } void mndCleanupConsumer(SMnode *pMnode) {} -static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { - return 0; -} - -static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { - int32_t size = sizeof(SMqConsumerObj) + MND_CONSUMER_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); +SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); + SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, tlen); if (pRaw == NULL) goto CM_ENCODE_OVER; + void* buf = malloc(tlen); + if (buf == NULL) goto CM_ENCODE_OVER; + + void* abuf = buf; + tEncodeSMqConsumerObj(&abuf, pConsumer); + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, CM_ENCODE_OVER); + +#if 0 int32_t topicNum = taosArrayGetSize(pConsumer->topics); SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER); int32_t len = strlen(pConsumer->cgroup); @@ -101,10 +94,13 @@ static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { /*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/ } } +#endif SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER); + terrno = TSDB_CODE_SUCCESS; + CM_ENCODE_OVER: if (terrno != 0) { mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); @@ -116,7 +112,7 @@ CM_ENCODE_OVER: return pRaw; } -static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { +SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; @@ -127,18 +123,27 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { goto CONSUME_DECODE_OVER; } - int32_t size = sizeof(SMqConsumerObj); - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj)); if (pRow == NULL) goto CONSUME_DECODE_OVER; SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); if (pConsumer == NULL) goto CONSUME_DECODE_OVER; int32_t dataPos = 0; - SDB_GET_INT64(pRaw, dataPos, &pConsumer->consumerId, CONSUME_DECODE_OVER); - int32_t len, topicNum; + int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CONSUME_DECODE_OVER); + void* buf = malloc(len); + if (buf == NULL) goto CONSUME_DECODE_OVER; + + SDB_GET_BINARY(pRaw, dataPos, buf, len, CONSUME_DECODE_OVER); + + tDecodeSMqConsumerObj(buf, pConsumer); + + SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CONSUME_DECODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +#if 0 SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER); for (int i = 0; i < topicNum; i++) { int32_t topicLen; @@ -154,6 +159,7 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t vgSize; SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER); } +#endif CONSUME_DECODE_OVER: if (terrno != 0) { @@ -162,8 +168,6 @@ CONSUME_DECODE_OVER: return NULL; } - /*SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE);*/ - return pRow; } @@ -201,214 +205,13 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { sdbRelease(pSdb, pConsumer); } -static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - char *msgStr = pMsg->rpcMsg.pCont; - SCMSubscribeReq subscribe; - tDeserializeSCMSubscribeReq(msgStr, &subscribe); - int64_t consumerId = subscribe.consumerId; - char *consumerGroup = subscribe.consumerGroup; - int32_t cgroupLen = strlen(consumerGroup); - - SArray *newSub = NULL; - int newTopicNum = subscribe.topicNum; - if (newTopicNum) { - newSub = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); - } - SMqConsumerTopic *pConsumerTopics = calloc(newTopicNum, sizeof(SMqConsumerTopic)); - if (pConsumerTopics == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - for (int i = 0; i < newTopicNum; i++) { - char *newTopicName = taosArrayGetP(newSub, i); - SMqConsumerTopic *pConsumerTopic = &pConsumerTopics[i]; - - strcpy(pConsumerTopic->name, newTopicName); - pConsumerTopic->vgroups = tdListNew(sizeof(int64_t)); - } - - taosArrayAddBatch(newSub, pConsumerTopics, newTopicNum); - free(pConsumerTopics); - taosArraySortString(newSub, taosArrayCompareString); - - SArray *oldSub = NULL; - int oldTopicNum = 0; - // create consumer if not exist - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pConsumer == NULL) { - // create consumer - pConsumer = malloc(sizeof(SMqConsumerObj)); - if (pConsumer == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pConsumer->consumerId = consumerId; - strcpy(pConsumer->cgroup, consumerGroup); - - } else { - oldSub = pConsumer->topics; - oldTopicNum = taosArrayGetSize(oldSub); - } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - //TODO: free memory - return -1; - } - - int i = 0, j = 0; - while (i < newTopicNum || j < oldTopicNum) { - SMqConsumerTopic *pOldTopic = NULL; - SMqConsumerTopic *pNewTopic = NULL; - if (i >= newTopicNum) { - // encode unset topic msg to all vnodes related to that topic - pOldTopic = taosArrayGet(oldSub, j); - j++; - } else if (j >= oldTopicNum) { - pNewTopic = taosArrayGet(newSub, i); - i++; - } else { - pNewTopic = taosArrayGet(newSub, i); - pOldTopic = taosArrayGet(oldSub, j); - - char *newName = pNewTopic->name; - char *oldName = pOldTopic->name; - int comp = compareLenPrefixedStr(newName, oldName); - if (comp == 0) { - // do nothing - pOldTopic = pNewTopic = NULL; - i++; - j++; - continue; - } else if (comp < 0) { - pOldTopic = NULL; - i++; - } else { - pNewTopic = NULL; - j++; - } - } - - if (pOldTopic != NULL) { - //cancel subscribe of that old topic - ASSERT(pNewTopic == NULL); - char *oldTopicName = pOldTopic->name; - SList *vgroups = pOldTopic->vgroups; - SListIter iter; - tdListInitIter(vgroups, &iter, TD_LIST_FORWARD); - SListNode *pn; - - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName); - ASSERT(pTopic != NULL); - SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); - while ((pn = tdListNext(&iter)) != NULL) { - int32_t vgId = *(int64_t *)pn->data; - // acquire and get epset - SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); - // TODO what time to release? - if (pVgObj == NULL) { - // TODO handle error - continue; - } - //build reset msg - void *pMqVgSetReq = mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, consumerId, consumerGroup); - // TODO:serialize - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = pMqVgSetReq; - action.contLen = 0; // TODO - action.msgType = TDMT_VND_MQ_SET_CONN; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMqVgSetReq); - mndTransDrop(pTrans); - // TODO free - return -1; - } - } - //delete data in mnode - taosHashRemove(pTopic->cgroups, consumerGroup, cgroupLen); - mndReleaseTopic(pMnode, pTopic); - - } else if (pNewTopic != NULL) { - // save subscribe info to mnode - ASSERT(pOldTopic == NULL); - - char *newTopicName = pNewTopic->name; - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); - ASSERT(pTopic != NULL); - - SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); - if (pGroup == NULL) { - // add new group - pGroup = malloc(sizeof(SMqCGroup)); - if (pGroup == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pGroup->consumerIds = tdListNew(sizeof(int64_t)); - if (pGroup->consumerIds == NULL) { - free(pGroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pGroup->status = 0; - // add into cgroups - taosHashPut(pTopic->cgroups, consumerGroup, cgroupLen, pGroup, sizeof(SMqCGroup)); - } - /*taosHashPut(pTopic->consumers, &pConsumer->consumerId, sizeof(int64_t), pConsumer, sizeof(SMqConsumerObj));*/ - - // put the consumer into list - // rebalance will be triggered by timer - tdListAppend(pGroup->consumerIds, &consumerId); - - SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic); - sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY); - // TODO: error handling - mndTransAppendRedolog(pTrans, pTopicRaw); - - mndReleaseTopic(pMnode, pTopic); - - } else { - ASSERT(0); - } - } - // destroy old sub - taosArrayDestroy(oldSub); - // put new sub into consumerobj - pConsumer->topics = newSub; - - // persist consumerObj - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - // TODO: error handling - mndTransAppendRedolog(pTrans, pConsumerRaw); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - mndReleaseConsumer(pMnode, pConsumer); - return -1; - } - - // TODO: free memory - mndTransDrop(pTrans); - mndReleaseConsumer(pMnode, pConsumer); - return 0; -} - -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } - +#if 0 static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoReq *pInfo = pMsg->rpcMsg.pCont; mDebug("consumer:%s, start to retrieve meta", pInfo->tableFname); -#if 0 SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -463,7 +266,6 @@ static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { pMsg->contLen = contLen; mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags); -#endif return 0; } @@ -546,3 +348,4 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } +#endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 3773750ed3..22fdfde2ac 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -273,6 +273,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { } static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { +#if 0 SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -332,6 +333,8 @@ static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { pRsp->body = buf; pRsp->bodyLen = tlen; return pRsp; +#endif + return NULL; } static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c new file mode 100644 index 0000000000..1aca2e9894 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -0,0 +1,597 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndConsumer.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndSubscribe.h" +#include "mndTopic.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tcompare.h" +#include "tname.h" + +#define MND_SUBSCRIBE_VER_NUMBER 1 +#define MND_SUBSCRIBE_RESERVE_SIZE 64 + +static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); +static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); +static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); +static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); +static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); + +static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); + +int32_t mndInitSubscribe(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SUBSCRIBE, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSubActionEncode, + .decodeFp = (SdbDecodeFp)mndSubActionDecode, + .insertFp = (SdbInsertFp)mndSubActionInsert, + .updateFp = (SdbUpdateFp)mndSubActionUpdate, + .deleteFp = (SdbDeleteFp)mndSubActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); + /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/ + /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/ + mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); + return sdbSetTable(pMnode->pSdb, table); +} + +static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { + SMqConsumerEp CEp; + CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; + int32_t sz; + SVgObj *pVgroup = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = sdbFetch(pSdb, SDB_VGROUP, NULL, (void **)&pVgroup); + while (pIter != NULL) { + if (pVgroup->dbUid == pTopic->dbUid) { + CEp.epset = mndGetVgroupEpset(pMnode, pVgroup); + CEp.vgId = pVgroup->vgId; + taosArrayPush(unassignedVg, &CEp); + } + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + } + return 0; +} + +static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, + SMqConsumerTopic *pConsumerTopic) { + int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); + for (int32_t i = 0; i < sz; i++) { + int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i); + SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + SMqSetCVgReq req = { + .vgId = vgId, + .consumerId = pConsumer->consumerId, + }; + strcpy(req.cGroup, pConsumer->cgroup); + strcpy(req.topicName, pConsumerTopic->name); + int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + void *reqStr = malloc(tlen); + if (reqStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + void *abuf = reqStr; + tEncodeSMqSetCVgReq(abuf, &req); + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + action.pCont = reqStr; + action.contLen = tlen; + action.msgType = TDMT_VND_MQ_SET_CONN; + + mndReleaseVgroup(pMnode, pVgObj); + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(reqStr); + return -1; + } + } + return 0; +} + +void mndCleanupSubscribe(SMnode *pMnode) {} + +static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { + int32_t tlen = tEncodeSubscribeObj(NULL, pSub); + int32_t size = tlen + MND_SUBSCRIBE_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); + if (pRaw == NULL) goto SUB_ENCODE_OVER; + + void *buf = malloc(tlen); + if (buf == NULL) { + goto SUB_ENCODE_OVER; + } + void *abuf = buf; + + tEncodeSubscribeObj(&buf, pSub); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, SUB_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, SUB_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, SUB_ENCODE_OVER); + +SUB_ENCODE_OVER: + if (terrno != 0) { + mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("subscribe:%s, encode to raw:%p, row:%p", pSub->key, pRaw, pSub); + return pRaw; +} + +static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; + + if (sver != MND_SUBSCRIBE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto SUB_DECODE_OVER; + } + + int32_t size = sizeof(SMqSubscribeObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto SUB_DECODE_OVER; + + SMqSubscribeObj *pSub = sdbGetRowObj(pRow); + if (pSub == NULL) goto SUB_DECODE_OVER; + + int32_t dataPos = 0; + int32_t tlen; + void *buf = malloc(tlen + 1); + if (buf == NULL) goto SUB_DECODE_OVER; + SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); + + if (tDecodeSubscribeObj(buf, pSub) == NULL) { + goto SUB_DECODE_OVER; + } + +SUB_DECODE_OVER: + if (terrno != 0) { + mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); + // TODO free subscribeobj + tfree(pRow); + return NULL; + } + + return pRow; +} + +static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { + mTrace("subscribe:%s, perform insert action", pSub->key); + return 0; +} + +static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { + mTrace("subscribe:%s, perform delete action", pSub->key); + return 0; +} + +static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) { + mTrace("subscribe:%s, perform update action", pOldSub->key); + return 0; +} + +static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { + return 0; +} + +static char *mndMakeSubscribeKey(char *cgroup, char *topicName) { + char *key = malloc(TSDB_SHOW_SUBQUERY_LEN); + if (key == NULL) { + return NULL; + } + int tlen = strlen(cgroup); + memcpy(key, cgroup, tlen); + key[tlen] = ':'; + strcpy(key + tlen + 1, topicName); + return key; +} + +SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, char *cgroup, char *topicName) { + SSdb *pSdb = pMnode->pSdb; + char *key = mndMakeSubscribeKey(cgroup, topicName); + SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); + free(key); + if (pSub == NULL) { + /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + } + return pSub; +} + +void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pSub); +} + +static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SCMSubscribeReq subscribe; + tDeserializeSCMSubscribeReq(msgStr, &subscribe); + int64_t consumerId = subscribe.consumerId; + char *consumerGroup = subscribe.consumerGroup; + int32_t cgroupLen = strlen(consumerGroup); + + SArray *newSub = subscribe.topicNames; + int newTopicNum = subscribe.topicNum; + + taosArraySortString(newSub, taosArrayCompareString); + + SArray *oldSub = NULL; + int oldTopicNum = 0; + // create consumer if not exist + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + if (pConsumer == NULL) { + // create consumer + pConsumer = malloc(sizeof(SMqConsumerObj)); + if (pConsumer == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pConsumer->consumerId = consumerId; + strcpy(pConsumer->cgroup, consumerGroup); + taosInitRWLatch(&pConsumer->lock); + } else { + oldSub = pConsumer->topics; + } + pConsumer->topics = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); + + if (oldSub != NULL) { + oldTopicNum = taosArrayGetSize(oldSub); + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + // TODO: free memory + return -1; + } + + int i = 0, j = 0; + while (i < newTopicNum || j < oldTopicNum) { + char* newTopicName = NULL; + char* oldTopicName = NULL; + if (i >= newTopicNum) { + // encode unset topic msg to all vnodes related to that topic + oldTopicName = ((SMqConsumerTopic*)taosArrayGet(oldSub, j))->name; + j++; + } else if (j >= oldTopicNum) { + newTopicName = taosArrayGet(newSub, i); + i++; + } else { + newTopicName = taosArrayGet(newSub, i); + oldTopicName = ((SMqConsumerTopic*)taosArrayGet(oldSub, j))->name; + + int comp = compareLenPrefixedStr(newTopicName, oldTopicName); + if (comp == 0) { + // do nothing + oldTopicName = newTopicName = NULL; + i++; + j++; + continue; + } else if (comp < 0) { + oldTopicName = NULL; + i++; + } else { + newTopicName = NULL; + j++; + } + } + + if (oldTopicName != NULL) { +#if 0 + // cancel subscribe of that old topic + ASSERT(pNewTopic == NULL); + char *oldTopicName = pOldTopic->name; + SList *vgroups = pOldTopic->vgroups; + SListIter iter; + tdListInitIter(vgroups, &iter, TD_LIST_FORWARD); + SListNode *pn; + + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName); + ASSERT(pTopic != NULL); + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, oldTopicName); + SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); + while ((pn = tdListNext(&iter)) != NULL) { + int32_t vgId = *(int64_t *)pn->data; + // acquire and get epset + SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + // TODO what time to release? + if (pVgObj == NULL) { + // TODO handle error + continue; + } + // build reset msg + void *pMqVgSetReq = mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, consumerId, consumerGroup); + // TODO:serialize + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + action.pCont = pMqVgSetReq; + action.contLen = 0; // TODO + action.msgType = TDMT_VND_MQ_SET_CONN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMqVgSetReq); + mndTransDrop(pTrans); + // TODO free + return -1; + } + } + // delete data in mnode + taosHashRemove(pTopic->cgroups, consumerGroup, cgroupLen); + mndReleaseSubscribe(pMnode, pSub); + mndReleaseTopic(pMnode, pTopic); +#endif + } else if (newTopicName != NULL) { + // save subscribe info to mnode + ASSERT(oldTopicName == NULL); + + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); + if (pTopic == NULL) { + /*terrno = */ + continue; + } + + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); + if (pSub == NULL) { + pSub = tNewSubscribeObj(); + if (pSub == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + // set unassigned vg + mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); + } + SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); + taosArrayPush(pConsumer->topics, pConsumerTopic); + + if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { + int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); + // send setmsg to vnode + if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic) < 0) { + // TODO + return -1; + } + } + taosArrayDestroy(pConsumerTopic->pVgInfo); + free(pConsumerTopic); +#if 0 + SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); + if (pGroup == NULL) { + // add new group + pGroup = malloc(sizeof(SMqCGroup)); + if (pGroup == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pGroup->consumerIds = tdListNew(sizeof(int64_t)); + if (pGroup->consumerIds == NULL) { + free(pGroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pGroup->status = 0; + // add into cgroups + taosHashPut(pTopic->cgroups, consumerGroup, cgroupLen, pGroup, sizeof(SMqCGroup)); + } + /*taosHashPut(pTopic->consumers, &pConsumer->consumerId, sizeof(int64_t), pConsumer, sizeof(SMqConsumerObj));*/ + + // put the consumer into list + // rebalance will be triggered by timer + tdListAppend(pGroup->consumerIds, &consumerId); + + SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic); + sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY); + // TODO: error handling + mndTransAppendRedolog(pTrans, pTopicRaw); + +#endif + mndReleaseTopic(pMnode, pTopic); + mndReleaseSubscribe(pMnode, pSub); + } + } + // part3. persist consumerObj + + // destroy old sub + if (oldSub) taosArrayDestroy(oldSub); + // put new sub into consumerobj + + // persist consumerObj + SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); + sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); + // TODO: error handling + mndTransAppendRedolog(pTrans, pConsumerRaw); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + mndReleaseConsumer(pMnode, pConsumer); + return -1; + } + + // TODO: free memory + if (newSub) taosArrayDestroy(newSub); + mndTransDrop(pTrans); + mndReleaseConsumer(pMnode, pConsumer); + return 0; +} + +static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + STableInfoReq *pInfo = pMsg->rpcMsg.pCont; + + mDebug("subscribe:%s, start to retrieve meta", pInfo->tableFname); + +#if 0 + SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("consumer:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + SConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pInfo->tableFname); + if (pConsumer == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_CONSUMER; + mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + taosRLockLatch(&pConsumer->lock); + int32_t totalCols = pConsumer->numOfColumns + pConsumer->numOfTags; + int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema); + + STableMetaRsp *pMeta = rpcMallocCont(contLen); + if (pMeta == NULL) { + taosRUnLockLatch(&pConsumer->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + memcpy(pMeta->consumerFname, pConsumer->name, TSDB_TABLE_FNAME_LEN); + pMeta->numOfTags = htonl(pConsumer->numOfTags); + pMeta->numOfColumns = htonl(pConsumer->numOfColumns); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->update = pDb->cfg.update; + pMeta->sversion = htonl(pConsumer->version); + pMeta->tuid = htonl(pConsumer->uid); + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pSrcSchema = &pConsumer->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + taosRUnLockLatch(&pConsumer->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseConsumer(pMnode, pConsumer); + + pMsg->pCont = pMeta; + pMsg->contLen = contLen; + + mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags); +#endif + return 0; +} + +static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumOfConsumers) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfConsumers = 0; + void *pIter = NULL; + while (1) { + SMqConsumerObj *pConsumer = NULL; + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) break; + + numOfConsumers++; + + sdbRelease(pSdb, pConsumer); + } + + *pNumOfConsumers = numOfConsumers; + return 0; +} + +static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfConsumers(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "tags"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_CONSUMER); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 16a9828e71..1d4cbf37ce 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -79,8 +79,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER); int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; - pTopic->physicalPlan = calloc(physicalPlanLen, sizeof(char)); - if (pTopic->physicalPlan == NULL) goto TOPIC_ENCODE_OVER; SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); @@ -92,12 +90,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { TOPIC_ENCODE_OVER: if (terrno != TSDB_CODE_SUCCESS) { mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr()); - /*if (pTopic->logicalPlan) {*/ - /*free(pTopic->logicalPlan);*/ - /*}*/ - /*if (pTopic->physicalPlan) {*/ - /*free(pTopic->physicalPlan);*/ - /*}*/ sdbFreeRaw(pRaw); return NULL; } @@ -138,7 +130,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->logicalPlan = calloc(len+1, sizeof(char)); + pTopic->logicalPlan = calloc(len + 1, sizeof(char)); if (pTopic->logicalPlan == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; @@ -146,7 +138,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->logicalPlan = calloc(len + 1, sizeof(char)); + pTopic->physicalPlan = calloc(len + 1, sizeof(char)); if (pTopic->physicalPlan == NULL) { free(pTopic->logicalPlan); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -154,7 +146,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { } SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER); - SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); terrno = TSDB_CODE_SUCCESS; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 8fdb6b1657..a9267b0ea3 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -38,10 +38,10 @@ const char *sdbTableName(ESdbType type) { return "auth"; case SDB_ACCT: return "acct"; + case SDB_SUBSCRIBE: + return "subscribe"; case SDB_CONSUMER: return "consumer"; - case SDB_CGROUP: - return "cgroup"; case SDB_TOPIC: return "topic"; case SDB_VGROUP: From 0330ab12ad1e6759fafc5f095103693096140a52 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 19 Jan 2022 17:03:29 +0800 Subject: [PATCH 040/103] [td-11818] Refactor. --- source/libs/executor/src/executorimpl.c | 65 +++++++++++++++++-------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4cb233f71e..27c24669cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4558,6 +4558,7 @@ void appendDownstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); +void createResultBlock(const SArray* pExprInfo, SExchangeInfo* pInfo, const SOperatorInfo* pOperator, size_t size); static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 @@ -4930,7 +4931,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody); } -void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { +void processRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo *pSendInfo = (SMsgSendInfo *) pMsg->ahandle; assert(pMsg->ahandle != NULL); @@ -4958,13 +4959,14 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; *newgroup = false; - if (pExchangeInfo->pRsp != NULL && pExchangeInfo->pRsp->completed == 1) { return NULL; } SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { // todo handle malloc error + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _error; } SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, 0); @@ -4983,6 +4985,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("QID:%"PRIx64" calloc %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _error; } pMsgSendInfo->param = pExchangeInfo; @@ -4993,7 +4997,6 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { int64_t transporterId = 0; int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pExchangeInfo->ready); if (pExchangeInfo->pRsp->numOfRows == 0) { @@ -5007,7 +5010,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pExchangeInfo->pRsp->numOfRows); if (tmp == NULL) { - // todo + goto _error; } size_t len = pExchangeInfo->pRsp->numOfRows * pColInfoData->info.bytes; @@ -5021,8 +5024,17 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { pRes->info.rows = pExchangeInfo->pRsp->numOfRows; return pExchangeInfo->pResult; + + _error: + tfree(pMsg); + tfree(pMsgSendInfo); + + terrno = pTaskInfo->code; + return NULL; } +static SSDataBlock* createResultDataBlock(const SArray* pExprInfo); + SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -5038,21 +5050,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* assert(taosArrayGetSize(pInfo->pSources) > 0); size_t size = taosArrayGetSize(pExprInfo); - pInfo->pResult = calloc(1, sizeof(SSDataBlock)); - pInfo->pResult->pDataBlock = taosArrayInit(pOperator->numOfOutput, sizeof(SColumnInfoData)); - - SArray* pResult = pInfo->pResult->pDataBlock; - for(int32_t i = 0; i < size; ++i) { - SColumnInfoData colInfoData = {0}; - SExprInfo* p = taosArrayGetP(pExprInfo, i); - - SSchema* pSchema = &p->base.resSchema; - colInfoData.info.type = pSchema->type; - colInfoData.info.colId = pSchema->colId; - colInfoData.info.bytes = pSchema->bytes; - - taosArrayPush(pResult, &colInfoData); - } + pInfo->pResult = createResultDataBlock(pExprInfo); pOperator->name = "ExchangeOperator"; pOperator->operatorType = OP_Exchange; @@ -5064,13 +5062,13 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pOperator->exec = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; - { + { // todo refactor SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; rpcInit.numOfThreads = 1; - rpcInit.cfp = processMsgFromServer; + rpcInit.cfp = processRspMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; @@ -5088,6 +5086,31 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* return pOperator; } +SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { + SSDataBlock* pResBlock = calloc(1, sizeof(SSDataBlock)); + if (pResBlock == NULL) { + return NULL; + } + + size_t numOfCols = taosArrayGetSize(pExprInfo); + pResBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + + SArray* pResult = pResBlock->pDataBlock; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfoData = {0}; + SExprInfo* p = taosArrayGetP(pExprInfo, i); + + SSchema* pSchema = &p->base.resSchema; + colInfoData.info.type = pSchema->type; + colInfoData.info.colId = pSchema->colId; + colInfoData.info.bytes = pSchema->bytes; + + taosArrayPush(pResult, &colInfoData); + } + + return pResBlock; +} + SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); From c3e222bfe2908289e492a74b38fd719594e7de7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 01:28:03 -0800 Subject: [PATCH 041/103] refact tfs module --- source/dnode/mgmt/impl/src/dndVnodes.c | 3 ++- source/dnode/vnode/src/tsdb/tsdbFS.c | 23 ++++++++++++----------- source/dnode/vnode/src/tsdb/tsdbMain.c | 3 ++- source/dnode/vnode/src/vnd/vnodeMain.c | 1 + source/libs/tfs/inc/tfsInt.h | 12 ++++++------ source/libs/tfs/src/tfs.c | 11 ++++++----- source/libs/tfs/src/tfsDisk.c | 2 +- source/libs/tfs/src/tfsTier.c | 1 + 8 files changed, 31 insertions(+), 25 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 01179b38ae..0d4b9c803d 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -381,7 +381,7 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnodeCfg cfg = {.pDnode = pDnode, .vgId = pCfg->vgId}; + SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId}; SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); @@ -587,6 +587,7 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } vnodeCfg.pDnode = pDnode; + vnodeCfg.pTfs = pDnode->pTfs; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d787eebcff..135b81f282 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -23,9 +23,9 @@ static const char *tsdbTxnFname[] = {"current.t", "current"}; static int tsdbComparFidFSet(const void *arg1, const void *arg2); static void tsdbResetFSStatus(SFSStatus *pStatus); -static int tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid); +static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus); static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo); -static void tsdbGetTxnFname(STfs *pTfs, int repoid, TSDB_TXN_FILE_T ftype, char fname[]); +static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]); static int tsdbOpenFSFromCurrent(STsdb *pRepo); static int tsdbScanAndTryFixFS(STsdb *pRepo); static int tsdbScanRootDir(STsdb *pRepo); @@ -311,7 +311,7 @@ int tsdbOpenFS(STsdb *pRepo) { ASSERT(pfs != NULL); - tsdbGetTxnFname(pRepo->pTfs, REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); tsdbGetRtnSnap(pRepo, &pRepo->rtn); if (access(current, F_OK) == 0) { @@ -375,7 +375,7 @@ int tsdbEndFSTxn(STsdb *pRepo) { SFSStatus *pStatus; // Write current file system snapshot - if (tsdbSaveFSStatus(pRepo->pTfs, pfs->nstatus, REPO_ID(pRepo)) < 0) { + if (tsdbSaveFSStatus(pRepo, pfs->nstatus) < 0) { tsdbEndFSTxnWithError(pfs); return -1; } @@ -405,7 +405,7 @@ int tsdbEndFSTxnWithError(STsdbFS *pfs) { int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); } -static int tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid) { +static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { SFSHeader fsheader; void * pBuf = NULL; void * ptr; @@ -413,8 +413,8 @@ static int tsdbSaveFSStatus(STfs *pTfs, SFSStatus *pStatus, int vid) { char tfname[TSDB_FILENAME_LEN] = "\0"; char cfname[TSDB_FILENAME_LEN] = "\0"; - tsdbGetTxnFname(pTfs, vid, TSDB_TXN_TEMP_FILE, tfname); - tsdbGetTxnFname(pTfs, vid, TSDB_TXN_CURR_FILE, cfname); + tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); + tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); if (fd < 0) { @@ -645,8 +645,9 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } } -static void tsdbGetTxnFname(STfs *pTfs, int repoid, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(pTfs), repoid, tsdbTxnFname[ftype]); +static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(pRepo->pTfs), pRepo->vgId, + tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -657,7 +658,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { char current[TSDB_FILENAME_LEN] = "\0"; void * ptr; - tsdbGetTxnFname(pRepo->pTfs, REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); // current file exists, try to recover fd = open(current, O_RDONLY | O_BINARY); @@ -1287,7 +1288,7 @@ static int tsdbRestoreCurrent(STsdb *pRepo) { return -1; } - if (tsdbSaveFSStatus(pRepo->pTfs, pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) { + if (tsdbSaveFSStatus(pRepo, pRepo->fs->cstatus) < 0) { tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index adc7726780..4da1e3e428 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -21,7 +21,8 @@ static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); -STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs) { +STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, + STfs *pTfs) { STsdb *pTsdb = NULL; // Set default TSDB Options diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 76ec0c0594..76b7ccf0d9 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -28,6 +28,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { if (pVnodeCfg != NULL) { cfg.vgId = pVnodeCfg->vgId; cfg.pDnode = pVnodeCfg->pDnode; + cfg.pTfs = pVnodeCfg->pTfs; } // Validate options diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index 5ca93db47b..c88a2a4ea8 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -58,12 +58,12 @@ typedef struct { } SDiskIter; typedef struct STfsDir { - SDiskIter *iter; - SDiskID did; - char dirname[TSDB_FILENAME_LEN]; - STfsFile tfile; - DIR *dir; - STfs *pTfs; + SDiskIter iter; + SDiskID did; + char dirname[TSDB_FILENAME_LEN]; + STfsFile tfile; + DIR *dir; + STfs *pTfs; } STfsDir; typedef struct STfs { diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 8ceae8ac96..be411744cf 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -152,7 +152,6 @@ void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname) } bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { - ASSERT(pFile1 != NULL || pFile2 != NULL); if (pFile1 == NULL || pFile2 == NULL || pFile1->pTfs != pFile2->pTfs) return false; if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; @@ -308,7 +307,7 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { } SDiskID diskId = {.id = 0, .level = 0}; - pDir->iter->pDisk = TFS_DISK_AT(pTfs, diskId); + pDir->iter.pDisk = TFS_DISK_AT(pTfs, diskId); pDir->pTfs = pTfs; tstrncpy(pDir->dirname, rname, TSDB_FILENAME_LEN); @@ -331,7 +330,7 @@ const STfsFile *tfsReaddir(STfsDir *pDir) { // Skip . and .. if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; - snprintf(bname, TMPNAME_LEN * 2, "%s/%s", pDir->dirname, dp->d_name); + snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname); return &pDir->tfile; } @@ -496,7 +495,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) { } while (true) { - pDisk = tfsNextDisk(pTfs, pDir->iter); + pDisk = tfsNextDisk(pTfs, &pDir->iter); if (pDisk == NULL) return 0; pDir->did.level = pDisk->level; @@ -514,7 +513,9 @@ static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) { if (pIter == NULL) return NULL; STfsDisk *pDisk = pIter->pDisk; - SDiskID did = {.level = pDisk->level, .id = pDisk->id + 1}; + if (pDisk == NULL) return NULL; + + SDiskID did = {.level = pDisk->level, .id = pDisk->id + 1}; if (did.id < TFS_TIER_AT(pTfs, did.level)->ndisk) { pIter->pDisk = TFS_DISK_AT(pTfs, did); diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 7046530aeb..52396db3be 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -46,7 +46,7 @@ STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { } int32_t tfsUpdateDiskSize(STfsDisk *pDisk) { - if (taosGetDiskSize(pDisk->path, &pDisk->size) != 0) { + if (taosGetDiskSize(pDisk->path, &pDisk->size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); fError("failed to get disk:%s size, level:%d id:%d since %s", pDisk->path, pDisk->level, pDisk->id, terrstr()); return -1; diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index a28cd74f1a..270fff9ff3 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -82,6 +82,7 @@ void tfsUpdateTierSize(STfsTier *pTier) { for (int32_t id = 0; id < pTier->ndisk; id++) { STfsDisk *pDisk = pTier->disks[id]; if (pDisk == NULL) continue; + if (tfsUpdateDiskSize(pDisk) < 0) continue; size.total += pDisk->size.total; size.used += pDisk->size.used; From 2f003140efcdd0f3eb2599af16458a2d18a1c38f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 19 Jan 2022 17:35:06 +0800 Subject: [PATCH 042/103] add dependency for scheduler --- include/common/tmsg.h | 658 ++++++++++----------- source/dnode/mnode/impl/CMakeLists.txt | 3 +- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 1 - source/libs/scheduler/CMakeLists.txt | 4 +- 5 files changed, 330 insertions(+), 337 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7d9fd53e69..15daa97dbf 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -140,214 +140,6 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) - -typedef struct { - int32_t keyLen; - int32_t valueLen; - void* key; - void* value; -} SKv; - -typedef struct { - int32_t connId; - int32_t hbType; -} SClientHbKey; - -typedef struct { - SClientHbKey connKey; - SHashObj* info; // hash -} SClientHbReq; - -typedef struct { - int64_t reqId; - SArray* reqs; // SArray -} SClientHbBatchReq; - -typedef struct { - SClientHbKey connKey; - int32_t status; - int32_t bodyLen; - void* body; -} SClientHbRsp; - -typedef struct { - int64_t reqId; - int64_t rspId; - SArray* rsps; // SArray -} SClientHbBatchRsp; - -static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { - return taosIntHash_64(key, keyLen); -} - -int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); - -int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); -void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); - -static FORCE_INLINE void tFreeClientHbReq(void *pReq) { - SClientHbReq* req = (SClientHbReq*)pReq; - if (req->info) taosHashCleanup(req->info); -} - -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); -void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); - -static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { - SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; - if (deep) { - taosArrayDestroyEx(req->reqs, tFreeClientHbReq); - } else { - taosArrayDestroy(req->reqs); - } - free(pReq); -} - -int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); -void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); - -static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKv->keyLen); - tlen += taosEncodeFixedI32(buf, pKv->valueLen); - tlen += taosEncodeBinary(buf, pKv->key, pKv->keyLen); - tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { - buf = taosDecodeFixedI32(buf, &pKv->keyLen); - buf = taosDecodeFixedI32(buf, &pKv->valueLen); - buf = taosDecodeBinary(buf, &pKv->key, pKv->keyLen); - buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); - return buf; -} - -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; -} - -typedef struct SMqHbVgInfo { - int32_t vgId; -} SMqHbVgInfo; - -static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pVgInfo->vgId); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) { - buf = taosDecodeFixedI32(buf, &pVgInfo->vgId); - return buf; -} - -typedef struct SMqHbTopicInfo { - int32_t epoch; - int64_t topicUid; - char name[TSDB_TOPIC_FNAME_LEN]; - SArray* pVgInfo; -} SMqHbTopicInfo; - -static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch); - tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid); - tlen += taosEncodeString(buf, pTopicInfo->name); - int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i); - tlen += taosEncodeSMqVgInfo(buf, pVgInfo); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) { - buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch); - buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid); - buf = taosDecodeStringTo(buf, pTopicInfo->name); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo)); - for (int32_t i = 0; i < sz; i++) { - SMqHbVgInfo vgInfo; - buf = taosDecodeSMqVgInfo(buf, &vgInfo); - taosArrayPush(pTopicInfo->pVgInfo, &vgInfo); - } - return buf; -} - -typedef struct SMqHbMsg { - int32_t status; // ask hb endpoint - int32_t epoch; - int64_t consumerId; - SArray* pTopics; // SArray -} SMqHbMsg; - -static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pMsg->status); - tlen += taosEncodeFixedI32(buf, pMsg->epoch); - tlen += taosEncodeFixedI64(buf, pMsg->consumerId); - int32_t sz = taosArrayGetSize(pMsg->pTopics); - tlen += taosEncodeFixedI32(buf, sz); - for (int i = 0; i < sz; i++) { - SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i); - tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { - buf = taosDecodeFixedI32(buf, &pMsg->status); - buf = taosDecodeFixedI32(buf, &pMsg->epoch); - buf = taosDecodeFixedI64(buf, &pMsg->consumerId); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo)); - for (int i = 0; i < sz; i++) { - SMqHbTopicInfo topicInfo; - buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo); - taosArrayPush(pMsg->pTopics, &topicInfo); - } - return buf; -} - -typedef struct SMqSetCVgReq { - int32_t vgId; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_CONSUMER_GROUP_LEN]; -} SMqSetCVgReq; - -static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeString(buf, pReq->topicName); - tlen += taosEncodeString(buf, pReq->cGroup); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeStringTo(buf, pReq->topicName); - buf = taosDecodeStringTo(buf, pReq->cGroup); - return buf; -} - typedef struct { int32_t vgId; char* dbName; @@ -384,6 +176,18 @@ typedef struct SSubmitBlk { char data[]; } SSubmitBlk; +typedef struct { + /* data */ +} SSubmitReq; + +typedef struct { + /* data */ +} SSubmitRsp; + +typedef struct { + /* data */ +} SSubmitReqReader; + // Submit message for this TSDB typedef struct { SMsgHead header; @@ -486,92 +290,6 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { } return buf; } - -typedef struct SMqHbRsp { - int8_t status; //idle or not - int8_t vnodeChanged; - int8_t epChanged; // should use new epset - int8_t reserved; - SEpSet epSet; -} SMqHbRsp; - -static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { - int tlen = 0; - tlen += taosEncodeFixedI8(buf, pRsp->status); - tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); - tlen += taosEncodeFixedI8(buf, pRsp->epChanged); - tlen += taosEncodeSEpSet(buf, &pRsp->epSet); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { - buf = taosDecodeFixedI8(buf, &pRsp->status); - buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged); - buf = taosDecodeFixedI8(buf, &pRsp->epChanged); - buf = taosDecodeSEpSet(buf, &pRsp->epSet); - return buf; -} - -typedef struct SMqHbOneTopicBatchRsp { - char topicName[TSDB_TOPIC_FNAME_LEN]; - SArray* rsps; // SArray -} SMqHbOneTopicBatchRsp; - -static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { - int tlen = 0; - tlen += taosEncodeString(buf, pBatchRsp->topicName); - int32_t sz = taosArrayGetSize(pBatchRsp->rsps); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i); - tlen += taosEncodeSMqHbRsp(buf, pRsp); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) { - int32_t sz; - buf = taosDecodeStringTo(buf, pBatchRsp->topicName); - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp)); - for (int32_t i = 0; i < sz; i++) { - SMqHbRsp rsp; - buf = taosDecodeSMqHbRsp(buf, &rsp); - buf = taosArrayPush(pBatchRsp->rsps, &rsp); - } - return buf; -} - -typedef struct SMqHbBatchRsp { - int64_t consumerId; - SArray* batchRsps; // SArray -} SMqHbBatchRsp; - -static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { - int tlen = 0; - tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId); - int32_t sz; - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*) taosArrayGet(pBatchRsp->batchRsps, i); - tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) { - buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp)); - for (int32_t i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp rsp; - buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp); - buf = taosArrayPush(pBatchRsp->batchRsps, &rsp); - } - return buf; -} - typedef struct { int32_t acctId; int64_t clusterId; @@ -1152,45 +870,7 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupReq; -// mq related -typedef struct { -} SMqConnectReq; - -typedef struct { -} SMqConnectRsp; - -typedef struct { -} SMqDisconnectReq; - -typedef struct { -} SMqDisconnectRsp; - -typedef struct { -} SMqAckReq; - -typedef struct { -} SMqAckRsp; - -typedef struct { -} SMqResetReq; - -typedef struct { -} SMqResetRsp; -// mq related end - -typedef struct { - /* data */ -} SSubmitReq; - -typedef struct { - /* data */ -} SSubmitRsp; - -typedef struct { - /* data */ -} SSubmitReqReader; - -typedef struct { +typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; @@ -1557,6 +1237,318 @@ typedef struct { #pragma pack(pop) +static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pMsg->contLen); + tlen += taosEncodeFixedI32(buf, pMsg->vgId); + return tlen; +} + +typedef struct SMqHbRsp { + int8_t status; //idle or not + int8_t vnodeChanged; + int8_t epChanged; // should use new epset + int8_t reserved; + SEpSet epSet; +} SMqHbRsp; + +static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { + int tlen = 0; + tlen += taosEncodeFixedI8(buf, pRsp->status); + tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); + tlen += taosEncodeFixedI8(buf, pRsp->epChanged); + tlen += taosEncodeSEpSet(buf, &pRsp->epSet); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { + buf = taosDecodeFixedI8(buf, &pRsp->status); + buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged); + buf = taosDecodeFixedI8(buf, &pRsp->epChanged); + buf = taosDecodeSEpSet(buf, &pRsp->epSet); + return buf; +} + +typedef struct SMqHbOneTopicBatchRsp { + char topicName[TSDB_TOPIC_FNAME_LEN]; + SArray* rsps; // SArray +} SMqHbOneTopicBatchRsp; + +static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { + int tlen = 0; + tlen += taosEncodeString(buf, pBatchRsp->topicName); + int32_t sz = taosArrayGetSize(pBatchRsp->rsps); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i); + tlen += taosEncodeSMqHbRsp(buf, pRsp); + } + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) { + int32_t sz; + buf = taosDecodeStringTo(buf, pBatchRsp->topicName); + buf = taosDecodeFixedI32(buf, &sz); + pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp)); + for (int32_t i = 0; i < sz; i++) { + SMqHbRsp rsp; + buf = taosDecodeSMqHbRsp(buf, &rsp); + buf = taosArrayPush(pBatchRsp->rsps, &rsp); + } + return buf; +} + +typedef struct SMqHbBatchRsp { + int64_t consumerId; + SArray* batchRsps; // SArray +} SMqHbBatchRsp; + +static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { + int tlen = 0; + tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId); + int32_t sz; + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*) taosArrayGet(pBatchRsp->batchRsps, i); + tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp); + } + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) { + buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp)); + for (int32_t i = 0; i < sz; i++) { + SMqHbOneTopicBatchRsp rsp; + buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp); + buf = taosArrayPush(pBatchRsp->batchRsps, &rsp); + } + return buf; +} + +typedef struct { + int32_t keyLen; + int32_t valueLen; + void* key; + void* value; +} SKv; + +typedef struct { + int32_t connId; + int32_t hbType; +} SClientHbKey; + +typedef struct { + SClientHbKey connKey; + SHashObj* info; // hash +} SClientHbReq; + +typedef struct { + int64_t reqId; + SArray* reqs; // SArray +} SClientHbBatchReq; + +typedef struct { + SClientHbKey connKey; + int32_t status; + int32_t bodyLen; + void* body; +} SClientHbRsp; + +typedef struct { + int64_t reqId; + int64_t rspId; + SArray* rsps; // SArray +} SClientHbBatchRsp; + +static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { + return taosIntHash_64(key, keyLen); +} + +int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); +void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); + +int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); +void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); + +static FORCE_INLINE void tFreeClientHbReq(void *pReq) { + SClientHbReq* req = (SClientHbReq*)pReq; + if (req->info) taosHashCleanup(req->info); +} + +int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); +void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); + +static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { + SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; + if (deep) { + taosArrayDestroyEx(req->reqs, tFreeClientHbReq); + } else { + taosArrayDestroy(req->reqs); + } + free(pReq); +} + +int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); +void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); + +static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKv->keyLen); + tlen += taosEncodeFixedI32(buf, pKv->valueLen); + tlen += taosEncodeBinary(buf, pKv->key, pKv->keyLen); + tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { + buf = taosDecodeFixedI32(buf, &pKv->keyLen); + buf = taosDecodeFixedI32(buf, &pKv->valueLen); + buf = taosDecodeBinary(buf, &pKv->key, pKv->keyLen); + buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); + return buf; +} + +static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKey->connId); + tlen += taosEncodeFixedI32(buf, pKey->hbType); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { + buf = taosDecodeFixedI32(buf, &pKey->connId); + buf = taosDecodeFixedI32(buf, &pKey->hbType); + return buf; +} + +typedef struct SMqHbVgInfo { + int32_t vgId; +} SMqHbVgInfo; + +static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pVgInfo->vgId); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) { + buf = taosDecodeFixedI32(buf, &pVgInfo->vgId); + return buf; +} + +typedef struct SMqHbTopicInfo { + int32_t epoch; + int64_t topicUid; + char name[TSDB_TOPIC_FNAME_LEN]; + SArray* pVgInfo; +} SMqHbTopicInfo; + +static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch); + tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid); + tlen += taosEncodeString(buf, pTopicInfo->name); + int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i); + tlen += taosEncodeSMqVgInfo(buf, pVgInfo); + } + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) { + buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch); + buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid); + buf = taosDecodeStringTo(buf, pTopicInfo->name); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo)); + for (int32_t i = 0; i < sz; i++) { + SMqHbVgInfo vgInfo; + buf = taosDecodeSMqVgInfo(buf, &vgInfo); + taosArrayPush(pTopicInfo->pVgInfo, &vgInfo); + } + return buf; +} + +typedef struct SMqHbMsg { + int32_t status; // ask hb endpoint + int32_t epoch; + int64_t consumerId; + SArray* pTopics; // SArray +} SMqHbMsg; + +static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pMsg->status); + tlen += taosEncodeFixedI32(buf, pMsg->epoch); + tlen += taosEncodeFixedI64(buf, pMsg->consumerId); + int32_t sz = taosArrayGetSize(pMsg->pTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int i = 0; i < sz; i++) { + SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i); + tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo); + } + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { + buf = taosDecodeFixedI32(buf, &pMsg->status); + buf = taosDecodeFixedI32(buf, &pMsg->epoch); + buf = taosDecodeFixedI64(buf, &pMsg->consumerId); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo)); + for (int i = 0; i < sz; i++) { + SMqHbTopicInfo topicInfo; + buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo); + taosArrayPush(pMsg->pTopics, &topicInfo); + } + return buf; +} + +typedef struct SMqSetCVgReq { + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + SArray* tasks; // SArray +} SMqSetCVgReq; + + +static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI32(buf, pReq->vgId); + tlen += taosEncodeFixedI64(buf, pReq->consumerId); + tlen += taosEncodeString(buf, pReq->topicName); + tlen += taosEncodeString(buf, pReq->cGroup); + tlen += taosEncodeString(buf, pReq->sql); + tlen += taosEncodeString(buf, pReq->logicalPlan); + tlen += taosEncodeString(buf, pReq->physicalPlan); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { + buf = taosDecodeFixedI32(buf, &pReq->vgId); + buf = taosDecodeFixedI64(buf, &pReq->consumerId); + buf = taosDecodeStringTo(buf, pReq->topicName); + buf = taosDecodeStringTo(buf, pReq->cGroup); + buf = taosDecodeString(buf, &pReq->sql); + buf = taosDecodeString(buf, &pReq->logicalPlan); + buf = taosDecodeString(buf, &pReq->physicalPlan); + pReq->tasks = NULL; + return buf; +} + + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index adbef3b55f..3aad002d40 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -7,6 +7,7 @@ target_include_directories( ) target_link_libraries( mnode + PRIVATE scheduler PRIVATE sdb PRIVATE wal PRIVATE transport @@ -16,4 +17,4 @@ target_link_libraries( if(${BUILD_TEST}) add_subdirectory(test) -endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a069987368..45759117b5 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -26,6 +26,7 @@ #include "tlog.h" #include "trpc.h" #include "ttimer.h" +#include "scheduler.h" #include "mnode.h" diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 1aca2e9894..52f33fa89c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 4e297f4e17..1b4aee3ccf 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,9 +9,9 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner qcom common catalog transport + PUBLIC os util planner qcom common catalog transport ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) From 68f45ebef71d404e010f4ba39a681d82c00661d7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 01:39:36 -0800 Subject: [PATCH 043/103] add test cases --- tests/script/sim/table/basic1.sim | 26 +++++++- tests/script/tmp/dnodes.sim | 105 ------------------------------ 2 files changed, 25 insertions(+), 106 deletions(-) delete mode 100644 tests/script/tmp/dnodes.sim diff --git a/tests/script/sim/table/basic1.sim b/tests/script/sim/table/basic1.sim index b5393a03dc..c33182dbf9 100644 --- a/tests/script/sim/table/basic1.sim +++ b/tests/script/sim/table/basic1.sim @@ -67,7 +67,31 @@ sql insert into c1 values(now+1s, 1) sql insert into c1 values(now+2s, 2) sql insert into c1 values(now+3s, 3) -return +print =============== query data +sql select * from c1 +if $rows != 3 then + return -1 +endi + +print $data00 $data01 +print $data10 $data11 +print $data20 $data11 + +if $data01 != 1 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data21 != 3 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + print =============== query data sql select * from c1 if $rows != 3 then diff --git a/tests/script/tmp/dnodes.sim b/tests/script/tmp/dnodes.sim deleted file mode 100644 index a3f9a0c173..0000000000 --- a/tests/script/tmp/dnodes.sim +++ /dev/null @@ -1,105 +0,0 @@ -############## config parameter ##################### -$node1 = 192.168.0.201 -$node2 = 192.168.0.202 -$node3 = 192.168.0.203 -$node4 = 192.168.0.204 - -$self = $node1 -$num = 25 - -#deploy = 0, start = 1, stop = 2 -$option = 0 -print =============== option:$option - - -############### stop dnodes ##################### -if $option == 0 then - system sh/stop_dnodes.sh -endi - -############### process firstEp ##################### - -$firstEp = $node1 . :7100 -$firstPort = 7100 -if $self == $node1 then - if $option == 1 then - system sh/exec.sh -n dnode1 -s start - endi - - if $option == 2 then - system sh/exec.sh -n dnode1 -s stop -x SIGINT - endi - - if $option == 0 then - system sh/deploy.sh -n dnode1 -i 1 - system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp - system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp - system sh/cfg.sh -n dnode1 -c fqdn -v $node1 - system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort - system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - - system sh/exec.sh -n dnode1 -s start - sql connect - - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node1 port $port - endw - - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node2 port $port - endw - - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node3 port $port - endw - - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node4 port $port - endw - endi -endi - -############### process nodes ##################### - -$i = 0 -while $i < $num - $index = $i + 80 - $port = $i * 100 - $port = $port + 8100 - $dnodename = dnode . $index - $i = $i + 1 - - if $option == 1 then - system sh/exec.sh -n $dnodename -s start - endi - - if $option == 2 then - system sh/exec.sh -n $dnodename -s stop -x SIGINT - endi - - if $option == 0 then - system sh/deploy.sh -n $dnodename -i 1 - system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp - system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp - system sh/cfg.sh -n $dnodename -c fqdn -v $self - system sh/cfg.sh -n $dnodename -c serverPort -v $port - - system sh/exec.sh -n $dnodename -s start - endi -endw From 3a6a714a945e741f3c05f07da45645b0f4376799 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 01:56:41 -0800 Subject: [PATCH 044/103] remove files --- source/libs/tfs/inc/tfsInt.h | 1 - src/cq/CMakeLists.txt | 15 - src/cq/src/cqMain.c | 532 ----------------------------------- src/cq/test/CMakeLists.txt | 6 - src/cq/test/cqtest.c | 108 ------- src/inc/query.h | 106 ------- src/inc/tcq.h | 74 ----- src/inc/tsdb.h | 426 ---------------------------- 8 files changed, 1268 deletions(-) delete mode 100644 src/cq/CMakeLists.txt delete mode 100644 src/cq/src/cqMain.c delete mode 100644 src/cq/test/CMakeLists.txt delete mode 100644 src/cq/test/cqtest.c delete mode 100644 src/inc/query.h delete mode 100644 src/inc/tcq.h delete mode 100644 src/inc/tsdb.h diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index c88a2a4ea8..cfc246f07b 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -22,7 +22,6 @@ #include "taoserror.h" #include "tcoding.h" #include "tfs.h" -#include "tglobal.h" #include "thash.h" #include "tlog.h" diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt deleted file mode 100644 index f01ccb8728..0000000000 --- a/src/cq/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) - -ADD_LIBRARY(tcq ${SRC}) -IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) -ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) -ENDIF () -ADD_SUBDIRECTORY(test) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c deleted file mode 100644 index d610ed605f..0000000000 --- a/src/cq/src/cqMain.c +++ /dev/null @@ -1,532 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - -#include -#include -#include -#include - -#include "../../../include/client/taos.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tcq.h" -#include "tdataformat.h" -#include "tglobal.h" -#include "tlog.h" -#include "tsclient.h" -#include "ttimer.h" -#include "twal.h" - -#define cFatal(...) { if (cqDebugFlag & DEBUG_FATAL) { taosPrintLog("CQ FATAL ", 255, __VA_ARGS__); }} -#define cError(...) { if (cqDebugFlag & DEBUG_ERROR) { taosPrintLog("CQ ERROR ", 255, __VA_ARGS__); }} -#define cWarn(...) { if (cqDebugFlag & DEBUG_WARN) { taosPrintLog("CQ WARN ", 255, __VA_ARGS__); }} -#define cInfo(...) { if (cqDebugFlag & DEBUG_INFO) { taosPrintLog("CQ ", 255, __VA_ARGS__); }} -#define cDebug(...) { if (cqDebugFlag & DEBUG_DEBUG) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }} -#define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }} - - -typedef struct SCqObj { - tmr_h tmrId; - int64_t rid; - uint64_t uid; - int32_t tid; // table ID - int32_t rowSize; // bytes of a row - char * dstTable; - char * sqlStr; // SQL string - STSchema * pSchema; // pointer to schema array - void * pStream; - struct SCqObj *prev; - struct SCqObj *next; - SCqContext * pContext; -} SCqObj; - -static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); -static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); - -int32_t cqObjRef = -1; -int32_t cqVnodeNum = 0; - -void cqRmFromList(SCqObj *pObj) { - //LOCK in caller - - SCqContext *pContext = pObj->pContext; - - if (pObj->prev) { - pObj->prev->next = pObj->next; - } else { - pContext->pHead = pObj->next; - } - - if (pObj->next) { - pObj->next->prev = pObj->prev; - } - -} - -static void freeSCqContext(void *handle) { - if (handle == NULL) { - return; - } - SCqContext *pContext = handle; - pthread_mutex_destroy(&pContext->mutex); - - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; - cDebug("vgId:%d, CQ is closed", pContext->vgId); - free(pContext); -} - - -void cqFree(void *handle) { - if (tsEnableStream == 0) { - return; - } - SCqObj *pObj = handle; - SCqContext *pContext = pObj->pContext; - int32_t delete = 0; - - pthread_mutex_lock(&pContext->mutex); - - // free the resources associated - if (pObj->pStream) { - taos_close_stream(pObj->pStream); - pObj->pStream = NULL; - } else { - taosTmrStop(pObj->tmrId); - pObj->tmrId = 0; - } - - cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); - tdFreeSchema(pObj->pSchema); - free(pObj->dstTable); - free(pObj->sqlStr); - free(pObj); - - pContext->cqObjNum--; - - if (pContext->cqObjNum <= 0 && pContext->delete) { - delete = 1; - } - - pthread_mutex_unlock(&pContext->mutex); - - if (delete) { - freeSCqContext(pContext); - } -} - - -void cqCreateRef() { - int32_t ref = atomic_load_32(&cqObjRef); - if (ref == -1) { - ref = taosOpenRef(4096, cqFree); - - if (atomic_val_compare_exchange_32(&cqObjRef, -1, ref) != -1) { - taosCloseRef(ref); - } - } -} - - -void *cqOpen(void *ahandle, const SCqCfg *pCfg) { - if (tsEnableStream == 0) { - return NULL; - } - SCqContext *pContext = calloc(sizeof(SCqContext), 1); - if (pContext == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - atomic_add_fetch_32(&cqVnodeNum, 1); - - cqCreateRef(); - - pContext->tmrCtrl = taosTmrInit(0, 0, 0, "CQ"); - - tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user)); - tstrncpy(pContext->pass, pCfg->pass, sizeof(pContext->pass)); - const char* db = pCfg->db; - for (const char* p = db; *p != 0; p++) { - if (*p == '.') { - db = p + 1; - break; - } - } - tstrncpy(pContext->db, db, sizeof(pContext->db)); - pContext->vgId = pCfg->vgId; - pContext->cqWrite = pCfg->cqWrite; - tscEmbedded = 1; - - pthread_mutex_init(&pContext->mutex, NULL); - - - cDebug("vgId:%d, CQ is opened", pContext->vgId); - - return pContext; -} - - -void cqClose(void *handle) { - if (tsEnableStream == 0) { - return; - } - SCqContext *pContext = handle; - if (handle == NULL) return; - - pContext->delete = 1; - int32_t hasCq = 0; - int32_t existLoop = 0; - - // stop all CQs - cqStop(pContext); - - int64_t rid = 0; - - while (1) { - pthread_mutex_lock(&pContext->mutex); - - SCqObj *pObj = pContext->pHead; - if (pObj) { - cqRmFromList(pObj); - - rid = pObj->rid; - - hasCq = 1; - - if (pContext->pHead == NULL) { - existLoop = 1; - } - } else { - pthread_mutex_unlock(&pContext->mutex); - break; - } - - pthread_mutex_unlock(&pContext->mutex); - - taosRemoveRef(cqObjRef, rid); - - if (existLoop) { - break; - } - } - - if (hasCq == 0) { - freeSCqContext(pContext); - } - - int32_t remainn = atomic_sub_fetch_32(&cqVnodeNum, 1); - if (remainn <= 0) { - int32_t ref = cqObjRef; - cqObjRef = -1; - taosCloseRef(ref); - } -} - -void cqStart(void *handle) { - if (tsEnableStream == 0) { - return; - } - SCqContext *pContext = handle; - if (pContext->dbConn || pContext->master) return; - - cDebug("vgId:%d, start all CQs", pContext->vgId); - pthread_mutex_lock(&pContext->mutex); - - pContext->master = 1; - - SCqObj *pObj = pContext->pHead; - while (pObj) { - cqCreateStream(pContext, pObj); - pObj = pObj->next; - } - - pthread_mutex_unlock(&pContext->mutex); -} - -void cqStop(void *handle) { - if (tsEnableStream == 0) { - return; - } - - SCqContext *pContext = handle; - cDebug("vgId:%d, stop all CQs", pContext->vgId); - if (pContext->dbConn == NULL || pContext->master == 0) return; - - pthread_mutex_lock(&pContext->mutex); - - pContext->master = 0; - SCqObj *pObj = pContext->pHead; - while (pObj) { - if (pObj->pStream) { - taos_close_stream(pObj->pStream); - pObj->pStream = NULL; - cInfo("vgId:%d, id:%d CQ:%s is closed", pContext->vgId, pObj->tid, pObj->sqlStr); - } else { - taosTmrStop(pObj->tmrId); - pObj->tmrId = 0; - } - pObj = pObj->next; - } - - if (pContext->dbConn) taos_close(pContext->dbConn); - pContext->dbConn = NULL; - - pthread_mutex_unlock(&pContext->mutex); -} - -void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema, int start) { - if (tsEnableStream == 0) { - return NULL; - } - SCqContext *pContext = handle; - int64_t rid = 0; - - pthread_mutex_lock(&pContext->mutex); - - SCqObj *pObj = pContext->pHead; - while (pObj) { - if (pObj->uid == uid) { - rid = pObj->rid; - pthread_mutex_unlock(&pContext->mutex); - return (void *)rid; - } - - pObj = pObj->next; - } - - pthread_mutex_unlock(&pContext->mutex); - - pObj = calloc(sizeof(SCqObj), 1); - if (pObj == NULL) return NULL; - - pObj->uid = uid; - pObj->tid = sid; - if (dstTable != NULL) { - pObj->dstTable = strdup(dstTable); - } - pObj->sqlStr = strdup(sqlStr); - - pObj->pSchema = tdDupSchema(pSchema); - pObj->rowSize = schemaTLen(pSchema); - - cInfo("vgId:%d, id:%d CQ:%s is created", pContext->vgId, pObj->tid, pObj->sqlStr); - - pthread_mutex_lock(&pContext->mutex); - - pObj->next = pContext->pHead; - if (pContext->pHead) pContext->pHead->prev = pObj; - pContext->pHead = pObj; - - pContext->cqObjNum++; - - pObj->rid = taosAddRef(cqObjRef, pObj); - - if(start && pContext->master) { - cqCreateStream(pContext, pObj); - } else { - pObj->pContext = pContext; - } - - rid = pObj->rid; - - pthread_mutex_unlock(&pContext->mutex); - - - return (void *)rid; -} - -void cqDrop(void *handle) { - if (tsEnableStream == 0) { - return; - } - - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)handle); - if (pObj == NULL) { - return; - } - - SCqContext *pContext = pObj->pContext; - - pthread_mutex_lock(&pContext->mutex); - - cqRmFromList(pObj); - - // free the resources associated - if (pObj->pStream) { - taos_close_stream(pObj->pStream); - pObj->pStream = NULL; - } else { - taosTmrStop(pObj->tmrId); - pObj->tmrId = 0; - } - - pthread_mutex_unlock(&pContext->mutex); - - taosRemoveRef(cqObjRef, (int64_t)handle); - taosReleaseRef(cqObjRef, (int64_t)handle); -} - -static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); - if (pObj == NULL) { - return; - } - - SCqContext* pContext = pObj->pContext; - SSqlObj* pSql = (SSqlObj*)result; - if (code == TSDB_CODE_SUCCESS) { - if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) { - taos_close(pSql->pTscObj); - } - } - - pthread_mutex_lock(&pContext->mutex); - cqCreateStream(pContext, pObj); - pthread_mutex_unlock(&pContext->mutex); - - taosReleaseRef(cqObjRef, (int64_t)param); -} - -static void cqProcessCreateTimer(void *param, void *tmrId) { - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); - if (pObj == NULL) { - return; - } - - SCqContext* pContext = pObj->pContext; - - if (pContext->dbConn == NULL) { - cDebug("vgId:%d, try connect to TDengine", pContext->vgId); - taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL); - } else { - pthread_mutex_lock(&pContext->mutex); - cqCreateStream(pContext, pObj); - pthread_mutex_unlock(&pContext->mutex); - } - - taosReleaseRef(cqObjRef, (int64_t)param); -} - -// inner implement in tscStream.c -TAOS_STREAM *taos_open_stream_withname(TAOS *taos, const char* desName, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), - int64_t stime, void *param, void (*callback)(void *), void* cqhandle); - -static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { - pObj->pContext = pContext; - - if (pContext->dbConn == NULL) { - cDebug("vgId:%d, create dbConn after 1000 ms", pContext->vgId); - pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, (void *)pObj->rid, pContext->tmrCtrl); - return; - } - - pObj->tmrId = 0; - - if (pObj->pStream == NULL) { - pObj->pStream = taos_open_stream_withname(pContext->dbConn, pObj->dstTable, pObj->sqlStr, cqProcessStreamRes, \ - INT64_MIN, (void *)pObj->rid, NULL, pContext); - - // TODO the pObj->pStream may be released if error happens - if (pObj->pStream) { - pContext->num++; - cDebug("vgId:%d, id:%d CQ:%s is opened", pContext->vgId, pObj->tid, pObj->sqlStr); - } else { - cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr); - } - } -} - -static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { - SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); - if (pObj == NULL) { - return; - } - - if (tres == NULL && row == NULL) { - taos_close_stream(pObj->pStream); - - pObj->pStream = NULL; - - taosReleaseRef(cqObjRef, (int64_t)param); - - return; - } - - SCqContext *pContext = pObj->pContext; - STSchema *pSchema = pObj->pSchema; - if (pObj->pStream == NULL) { - taosReleaseRef(cqObjRef, (int64_t)param); - return; - } - - cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - - int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_DATA_HEAD_SIZE + pObj->rowSize; - char *buffer = calloc(size, 1); - - SWalHead *pHead = (SWalHead *)buffer; - SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead)); - SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg)); - - SMemRow trow = (SMemRow)pBlk->data; - SDataRow dataRow = (SDataRow)memRowDataBody(trow); - memRowSetType(trow, SMEM_ROW_DATA); - tdInitDataRow(dataRow, pSchema); - - for (int32_t i = 0; i < pSchema->numOfCols; i++) { - STColumn *c = pSchema->columns + i; - void *val = row[i]; - if (val == NULL) { - val = (void *)getNullValue(c->type); - } else if (c->type == TSDB_DATA_TYPE_BINARY) { - val = ((char*)val) - sizeof(VarDataLenT); - } else if (c->type == TSDB_DATA_TYPE_NCHAR) { - char buf[TSDB_MAX_NCHAR_LEN]; - int32_t len = taos_fetch_lengths(tres)[i]; - taosMbsToUcs4(val, len, buf, sizeof(buf), &len); - memcpy((char *)val + sizeof(VarDataLenT), buf, len); - varDataLen(val) = len; - } - tdAppendColVal(dataRow, val, c->type, c->offset); - } - pBlk->dataLen = htonl(memRowDataTLen(trow)); - pBlk->schemaLen = 0; - - pBlk->uid = htobe64(pObj->uid); - pBlk->tid = htonl(pObj->tid); - pBlk->numOfRows = htons(1); - pBlk->sversion = htonl(pSchema->version); - pBlk->padding = 0; - - pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowDataTLen(trow); - - pMsg->header.vgId = htonl(pContext->vgId); - pMsg->header.contLen = htonl(pHead->len); - pMsg->length = pMsg->header.contLen; - pMsg->numOfBlocks = htonl(1); - - pHead->msgType = TDMT_VND_SUBMIT; - pHead->version = 0; - - // write into vnode write queue - pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL); - free(buffer); - - taosReleaseRef(cqObjRef, (int64_t)param); -} - diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt deleted file mode 100644 index d713dd7401..0000000000 --- a/src/cq/test/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -LIST(APPEND CQTEST_SRC ./cqtest.c) -ADD_EXECUTABLE(cqtest ${CQTEST_SRC}) -TARGET_LINK_LIBRARIES(cqtest tcq taos_static) diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c deleted file mode 100644 index ee52152ad8..0000000000 --- a/src/cq/test/cqtest.c +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tglobal.h" -#include "tlog.h" -#include "tcq.h" - -int64_t ver = 0; -void *pCq = NULL; - -int writeToQueue(int32_t vgId, void *data, int type, void *pMsg) { - return 0; -} - -int main(int argc, char *argv[]) { - int num = 3; - - for (int i=1; i - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_QUERY_H -#define TDENGINE_QUERY_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* qinfo_t; - -/** - * create the qinfo object according to QueryTableMsg - * @param tsdb - * @param pQueryTableMsg - * @param qinfo - * @return - */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); - - -/** - * the main query execution function, including query on both table and multitables, - * which are decided according to the tag or table name query conditions - * - * @param qinfo - * @return - */ -bool qTableQuery(qinfo_t qinfo, uint64_t *qId); - -/** - * Retrieve the produced results information, if current query is not paused or completed, - * this function will be blocked to wait for the query execution completed or paused, - * in which case enough results have been produced already. - * - * @param qinfo - * @return - */ -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); - -/** - * - * Retrieve the actual results to fill the response message payload. - * Note that this function must be executed after qRetrieveQueryResultInfo is invoked. - * - * @param qinfo qinfo object - * @param pRsp response message - * @param contLen payload length - * @return - */ -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); - -/** - * - * @param qinfo - * @return - */ -void* qGetResultRetrieveMsg(qinfo_t qinfo); - -/** - * kill current ongoing query and free query handle automatically - * @param qinfo qhandle - * @return - */ -int32_t qKillQuery(qinfo_t qinfo); - -//kill by qid -int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount); - -bool qSolveCommitNoBlock(void* pRepo, void* pMgmt); - -int32_t qQueryCompleted(qinfo_t qinfo); - -/** - * destroy query info structure - * @param qHandle - */ -void qDestroyQueryInfo(qinfo_t qHandle); - -void* qOpenQueryMgmt(int32_t vgId); -void qQueryMgmtNotifyClosed(void* pExecutor); -void qQueryMgmtReOpen(void *pExecutor); -void qCleanupQueryMgmt(void* pExecutor); -void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); -void** qAcquireQInfo(void* pMgmt, uint64_t key); -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle); -bool checkQIdEqual(void *qHandle, uint64_t qId); -int64_t genQueryId(void); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QUERY_H diff --git a/src/inc/tcq.h b/src/inc/tcq.h deleted file mode 100644 index 71efe33011..0000000000 --- a/src/inc/tcq.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_CQ_H_ -#define _TD_CQ_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tdataformat.h" - -typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); - -typedef struct { - int32_t vgId; - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] - FCqWrite cqWrite; -} SCqCfg; - -// SCqContext -typedef struct { - int32_t vgId; - int32_t master; - int32_t num; // number of continuous streams - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_NAME_LEN]; - FCqWrite cqWrite; - struct SCqObj *pHead; - void *dbConn; - void *tmrCtrl; - pthread_mutex_t mutex; - int32_t delete; - int32_t cqObjNum; -} SCqContext; - -// the following API shall be called by vnode -void *cqOpen(void *ahandle, const SCqCfg *pCfg); -void cqClose(void *handle); - -// if vnode is master, vnode call this API to start CQ -void cqStart(void *handle); - -// if vnode is slave/unsynced, vnode shall call this API to stop CQ -void cqStop(void *handle); - -// cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema, int start); - -// cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate -void cqDrop(void *handle); - -extern int32_t cqDebugFlag; - - -#ifdef __cplusplus -} -#endif - -#endif // _TD_CQ_H_ diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h deleted file mode 100644 index 130628e799..0000000000 --- a/src/inc/tsdb.h +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_TSDB_H_ -#define _TD_TSDB_H_ - -#include -#include -#include - -#include "taosdef.h" -#include "tmsg.h" -#include "tarray.h" -#include "tdataformat.h" -#include "tname.h" -#include "hash.h" -#include "tlockfree.h" -#include "tlist.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define TSDB_VERSION_MAJOR 1 -#define TSDB_VERSION_MINOR 0 - -#define TSDB_INVALID_SUPER_TABLE_ID -1 - -#define TSDB_STATUS_COMMIT_START 1 -#define TSDB_STATUS_COMMIT_OVER 2 -#define TSDB_STATUS_COMMIT_NOBLOCK 3 //commit no block, need to be solved - -// TSDB STATE DEFINITION -#define TSDB_STATE_OK 0x0 -#define TSDB_STATE_BAD_META 0x1 -#define TSDB_STATE_BAD_DATA 0x2 - -// --------- TSDB APPLICATION HANDLE DEFINITION -typedef struct { - void *appH; - void *cqH; - int (*notifyStatus)(void *, int status, int eno); - int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema, int start); - void (*cqDropFunc)(void *handle); -} STsdbAppH; - -// --------- TSDB REPOSITORY CONFIGURATION DEFINITION -typedef struct { - int32_t tsdbId; - int32_t cacheBlockSize; - int32_t totalBlocks; - int32_t daysPerFile; // day per file sharding policy - int32_t keep; // day of data to keep - int32_t keep1; - int32_t keep2; - int32_t minRowsPerFileBlock; // minimum rows per file block - int32_t maxRowsPerFileBlock; // maximum rows per file block - int8_t precision; - int8_t compression; - int8_t update; - int8_t cacheLastRow; // 0:no cache, 1: cache last row, 2: cache last NULL column 3: 1&2 -} STsdbCfg; - -#define CACHE_NO_LAST(c) ((c)->cacheLastRow == 0) -#define CACHE_LAST_ROW(c) (((c)->cacheLastRow & 1) > 0) -#define CACHE_LAST_NULL_COLUMN(c) (((c)->cacheLastRow & 2) > 0) - -// --------- TSDB REPOSITORY USAGE STATISTICS -typedef struct { - int64_t totalStorage; // total bytes occupie - int64_t compStorage; - int64_t pointsWritten; // total data points written -} STsdbStat; - -typedef struct STsdbRepo STsdbRepo; - -STsdbCfg *tsdbGetCfg(const STsdbRepo *repo); - -// --------- TSDB REPOSITORY DEFINITION -int32_t tsdbCreateRepo(int repoid); -int32_t tsdbDropRepo(int repoid); -STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); -int tsdbCloseRepo(STsdbRepo *repo, int toCommit); -int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg); -int tsdbGetState(STsdbRepo *repo); -int8_t tsdbGetCompactState(STsdbRepo *repo); -// --------- TSDB TABLE DEFINITION -typedef struct { - uint64_t uid; // the unique table ID - int32_t tid; // the table ID in the repository. -} STableId; - -// --------- TSDB TABLE configuration -typedef struct { - ETableType type; - char * name; - STableId tableId; - int32_t sversion; - char * sname; // super table name - uint64_t superUid; - STSchema * schema; - STSchema * tagSchema; - SKVRow tagValues; - char * sql; -} STableCfg; - -void tsdbClearTableCfg(STableCfg *config); - -void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes); -char *tsdbGetTableName(void *pTable); - -#define TSDB_TABLEID(_table) ((STableId*) (_table)) -#define TSDB_PREV_ROW 0x1 -#define TSDB_NEXT_ROW 0x2 - -STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); - -int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg); -int tsdbDropTable(STsdbRepo *pRepo, STableId tableId); -int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg); - -uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); - -// the TSDB repository info -typedef struct STsdbRepoInfo { - STsdbCfg tsdbCfg; - uint64_t version; // version of the repository - int64_t tsdbTotalDataSize; // the original inserted data size - int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository - // TODO: Other informations to add -} STsdbRepoInfo; -STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo); - -// the meter information report structure -typedef struct { - STableCfg tableCfg; - uint64_t version; - int64_t tableTotalDataSize; // In bytes - int64_t tableTotalDiskSize; // In bytes -} STableInfo; - -// -- FOR INSERT DATA -/** - * Insert data to a table in a repository - * @param pRepo the TSDB repository handle - * @param pData the data to insert (will give a more specific description) - * - * @return the number of points inserted, -1 for failure and the error number is set - */ -int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); - -// -- FOR QUERY TIME SERIES DATA - -typedef void *TsdbQueryHandleT; // Use void to hide implementation details - -#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 -#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 -#define BLOCK_LOAD_TABLE_RR_ORDER 3 - -// query condition to build multi-table data block iterator -typedef struct STsdbQueryCond { - STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block - int32_t numOfCols; - SColumnInfo *colList; - bool loadExternalRows; // load external rows or not - int32_t type; // data block load type: -} STsdbQueryCond; - -typedef struct STableData STableData; -typedef struct { - T_REF_DECLARE() - SRWLatch latch; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - int32_t maxTables; - STableData **tData; - SList * actList; - SList * extraBuffList; - SList * bufBlockList; - int64_t pointsAdd; // TODO - int64_t storageAdd; // TODO -} SMemTable; - -typedef struct { - SMemTable* mem; - SMemTable* imem; - SMemTable mtable; - SMemTable* omem; -} SMemSnapshot; - -typedef struct SMemRef { - int32_t ref; - SMemSnapshot snapshot; -} SMemRef; - -typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t numOfCols; - int64_t uid; - int32_t tid; -} SDataBlockInfo; - -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - -typedef struct { - void *pTable; - TSKEY lastKey; -} STableKeyInfo; - -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - -#define TSDB_BLOCK_DIST_STEP_ROWS 16 -typedef struct { - uint16_t rowSize; - uint16_t numOfFiles; - uint32_t numOfTables; - uint64_t totalSize; - uint64_t totalRows; - int32_t maxRows; - int32_t minRows; - int32_t firstSeekTimeUs; - uint32_t numOfRowsInMemTable; - uint32_t numOfSmallBlocks; - SArray *dataBlockInfos; -} STableBlockDist; - -/** - * Get the data block iterator, starting from position according to the query condition - * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the - * group by condition - * @param qinfo query info handle from query processor - * @return - */ -TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, - SMemRef *pRef); - -/** - * Get the last row of the given query time window for all the tables in STableGroupInfo object. - * Note that only one data block with only row will be returned while invoking retrieve data block function for - * all tables in this group. - * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableInfo table list. - * @return - */ -TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, - SMemRef *pRef); - - -TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); - -bool isTsdbCacheLastRow(TsdbQueryHandleT* pTsdbReadHandle); - - -/** - * get the queried table object list - * @param pHandle - * @return - */ -SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); - -/** - * get the group list according to table id from client - * @param tsdb - * @param pCond - * @param groupList - * @param qinfo - * @return - */ -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, - uint64_t qId, SMemRef *pRef); - - -/** - * get num of rows in mem table - * - * @param pHandle - * @return row size - */ - -int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); - -/** - * move to next block if exists - * - * @param pTsdbReadHandle - * @return - */ -bool tsdbNextDataBlock(TsdbQueryHandleT pTsdbReadHandle); - -/** - * Get current data block information - * - * @param pTsdbReadHandle - * @param pBlockInfo - * @return - */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); - -/** - * - * Get the pre-calculated information w.r.t. current data block. - * - * In case of data block in cache, the pBlockStatis will always be NULL. - * If a block is not completed loaded from disk, the pBlockStatis will be NULL. - - * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 - * @return - */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); - -/** - * - * The query condition with primary timestamp is passed to iterator during its constructor function, - * the returned data block must be satisfied with the time window condition in any cases, - * which means the SData data block is not actually the completed disk data blocks. - * - * @param pTsdbReadHandle query handle - * @param pColumnIdList required data columns id list - * @return - */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pTsdbReadHandle, SArray *pColumnIdList); - -/** - * Get the qualified table id for a super table according to the tag query expression. - * @param stableid. super table sid - * @param pTagCond. tag query condition - */ -int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, - int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, - SColIndex *pColIndex, int32_t numOfCols); - -/** - * destroy the created table group list, which is generated by tag query - * @param pGroupList - */ -void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); - -/** - * create the table group result including only one table, used to handle the normal table query - * - * @param tsdb tsdbHandle - * @param uid table uid - * @param pGroupInfo the generated result - * @return - */ -int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); - -/** - * - * @param tsdb - * @param pTableIdList - * @param pGroupInfo - * @return - */ -int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); - -/** - * clean up the query handle - * @param queryHandle - */ -void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); - -void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond); - -void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList); - -int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo); - -/** - * get the statistics of repo usage - * @param repo. point to the tsdbrepo - * @param totalPoints. total data point written - * @param totalStorage. total bytes took by the tsdb - * @param compStorage. total bytes took by the tsdb after compressed - */ -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage); - -int tsdbInitCommitQueue(); -void tsdbDestroyCommitQueue(); -int tsdbSyncCommit(STsdbRepo *repo); -void tsdbIncCommitRef(int vgId); -void tsdbDecCommitRef(int vgId); - -// For TSDB file sync -int tsdbSyncSend(void *pRepo, SOCKET socketFd); -int tsdbSyncRecv(void *pRepo, SOCKET socketFd); - -// For TSDB Compact -int tsdbCompact(STsdbRepo *pRepo); - -// For TSDB Health Monitor - -// no problem return true -bool tsdbNoProblem(STsdbRepo* pRepo); - -#ifdef __cplusplus -} -#endif - -#endif // _TD_TSDB_H_ From fcbc65cb349af6d763ab8e96366dacea2cc10d05 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:01:23 -0800 Subject: [PATCH 045/103] rename files --- {tests => 2.0/tests}/comparisonTest/cassandra/application.conf | 0 .../tests}/comparisonTest/cassandra/cassandratest/pom.xml | 0 .../src/main/java/com/cassandra/test/CassandraTest.java | 0 .../src/main/java/com/cassandra/test/WriteThread.java | 0 {tests => 2.0/tests}/comparisonTest/cassandra/q1.txt | 0 {tests => 2.0/tests}/comparisonTest/cassandra/q2.txt | 0 {tests => 2.0/tests}/comparisonTest/cassandra/q3.txt | 0 {tests => 2.0/tests}/comparisonTest/cassandra/q4.txt | 0 .../dataGenerator/com/taosdata/generator/DataGenerator.java | 0 {tests => 2.0/tests}/comparisonTest/influxdb/main.go | 0 {tests => 2.0/tests}/comparisonTest/influxdb/q1.txt | 0 {tests => 2.0/tests}/comparisonTest/influxdb/q2.txt | 0 {tests => 2.0/tests}/comparisonTest/influxdb/q3.txt | 0 {tests => 2.0/tests}/comparisonTest/influxdb/q4.txt | 0 {tests => 2.0/tests}/comparisonTest/opentsdb/opentsdbtest/pom.xml | 0 .../src/main/java/com/opentsdb/test/OpentsdbTest.java | 0 .../opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java | 0 .../opentsdb/opentsdbtest/src/main/resources/logback.xml | 0 {tests => 2.0/tests}/comparisonTest/tdengine/CMakeLists.txt | 0 {tests => 2.0/tests}/comparisonTest/tdengine/makefile | 0 {tests => 2.0/tests}/comparisonTest/tdengine/q1.txt | 0 {tests => 2.0/tests}/comparisonTest/tdengine/q2.txt | 0 {tests => 2.0/tests}/comparisonTest/tdengine/q3.txt | 0 {tests => 2.0/tests}/comparisonTest/tdengine/q4.txt | 0 {tests => 2.0/tests}/comparisonTest/tdengine/q5.txt | 0 {tests => 2.0/tests}/comparisonTest/tdengine/tdengineTest.c | 0 .../tests}/connectorTest/C#Test/nanosupport/TDengineDriver.cs | 0 {tests => 2.0/tests}/connectorTest/C#Test/nanosupport/nanotest.cs | 0 .../tests}/connectorTest/nodejsTest/nanosupport/nanosecondTest.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/cinterface.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/connection.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/constants.js | 0 {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/cursor.js | 0 {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/error.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/globalfunc.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/taosobjects.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/taosquery.js | 0 .../tests}/connectorTest/nodejsTest/nodetaos/taosresult.js | 0 {tests => 2.0/tests}/connectorTest/nodejsTest/readme.md | 0 {tests => 2.0/tests}/connectorTest/nodejsTest/tdengine.js | 0 {tests => 2.0/tests}/connectorTest/nodejsTest/test/performance.js | 0 {tests => 2.0/tests}/connectorTest/nodejsTest/test/test.js | 0 .../tests}/connectorTest/nodejsTest/test/testMicroseconds.js | 0 .../tests}/connectorTest/nodejsTest/test/testNanoseconds.js | 0 .../tests}/connectorTest/nodejsTest/test/testSubscribe.js | 0 .../tests}/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py | 0 {tests => 2.0/tests}/connectorTest/odbcTest/nanosupport/odbc.go | 0 {tests => 2.0/tests}/connectorTest/odbcTest/nanosupport/odbc.py | 0 {tests => 2.0/tests}/fuzz/sql-fuzzer.c | 0 {tests => 2.0/tests}/gotest/batchtest.bat | 0 {tests => 2.0/tests}/gotest/batchtest.sh | 0 {tests => 2.0/tests}/gotest/case001/case001.bat | 0 {tests => 2.0/tests}/gotest/case001/case001.go | 0 {tests => 2.0/tests}/gotest/case001/case001.sh | 0 {tests => 2.0/tests}/gotest/case002/case002.bat | 0 {tests => 2.0/tests}/gotest/case002/case002.go | 0 {tests => 2.0/tests}/gotest/case002/case002.sh | 0 {tests => 2.0/tests}/gotest/nanosupport/connector/executor.go | 0 {tests => 2.0/tests}/gotest/nanosupport/nanoCase.bat | 0 {tests => 2.0/tests}/gotest/nanosupport/nanoCase.sh | 0 {tests => 2.0/tests}/gotest/nanosupport/nanosupport.go | 0 {tests => 2.0/tests}/gotest/test.sh | 0 {tests => 2.0/tests}/http/restful/http_create_db.c | 0 {tests => 2.0/tests}/http/restful/http_create_tb.c | 0 {tests => 2.0/tests}/http/restful/http_drop_db.c | 0 {tests => 2.0/tests}/http/restful/http_insert_tb.c | 0 {tests => 2.0/tests}/http/restful/http_query_tb.c | 0 {tests => 2.0/tests}/http/restful/http_use_db.c | 0 {tests => 2.0/tests}/nettest/FQDNnettest.sh | 0 {tests => 2.0/tests}/nettest/TCPUDP.sh | 0 {tests => 2.0/tests}/nettest/fqdn.sh | 0 {tests => 2.0/tests}/nettest/tcpudp.sh | 0 {tests => 2.0/tests}/perftest-scripts/cassandraTestQ1Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/cassandraTestQ2Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/cassandraTestQ3Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/cassandraTestQ4Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/cassandraTestWriteLoop.sh | 0 {tests => 2.0/tests}/perftest-scripts/coverage_test.sh | 0 {tests => 2.0/tests}/perftest-scripts/full_test.sh | 0 {tests => 2.0/tests}/perftest-scripts/influxdbTestQ1Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/influxdbTestQ2Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/influxdbTestQ3Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/influxdbTestQ4Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/influxdbTestWriteLoop.sh | 0 {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ1Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ2Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ3Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ4Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/opentsdbTestWriteLoop.sh | 0 .../tests}/perftest-scripts/perftest-consistent-inserting-data.sh | 0 {tests => 2.0/tests}/perftest-scripts/perftest-csv2png.gnuplot | 0 {tests => 2.0/tests}/perftest-scripts/perftest-daily.sh | 0 {tests => 2.0/tests}/perftest-scripts/perftest-query.sh | 0 .../tests}/perftest-scripts/perftest-taosdemo-compare.sh | 0 {tests => 2.0/tests}/perftest-scripts/perftest-taosdemo.sh | 0 .../tests}/perftest-scripts/perftest-tsdb-compare-13d.sh | 0 {tests => 2.0/tests}/perftest-scripts/perftest-tsdb-compare-1d.sh | 0 .../perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh | 0 .../perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh | 0 {tests => 2.0/tests}/perftest-scripts/run-csv.sh | 0 {tests => 2.0/tests}/perftest-scripts/runInfluxdb-13d-csv.sh | 0 {tests => 2.0/tests}/perftest-scripts/runInfluxdb.sh | 0 {tests => 2.0/tests}/perftest-scripts/runTDengine.sh | 0 {tests => 2.0/tests}/perftest-scripts/runreal-13d-csv.sh | 0 {tests => 2.0/tests}/perftest-scripts/runreal-1d-csv.sh | 0 {tests => 2.0/tests}/perftest-scripts/taosdemo-csv2png.gnuplot | 0 .../tests}/perftest-scripts/taosdemo-rps-csv2png.gnuplot | 0 {tests => 2.0/tests}/perftest-scripts/tdengineTestQ1Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdengineTestQ2Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdengineTestQ3Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdengineTestQ4Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdengineTestQ5Loop.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdengineTestWriteLoop.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdinternal_coverage_test.sh | 0 {tests => 2.0/tests}/perftest-scripts/tdinternal_test.sh | 0 {tests => 2.0/tests}/robust/cluster.sh | 0 {tests => 2.0/tests}/robust/makefile | 0 {tests => 2.0/tests}/robust/monitor.sh | 0 {tests => 2.0/tests}/robust/robust.c | 0 {tests => 2.0/tests}/stress/.gitignore | 0 {tests => 2.0/tests}/stress/README.md | 0 {tests => 2.0/tests}/stress/go.mod | 0 {tests => 2.0/tests}/stress/main.go | 0 123 files changed, 0 insertions(+), 0 deletions(-) rename {tests => 2.0/tests}/comparisonTest/cassandra/application.conf (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/cassandratest/pom.xml (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/q1.txt (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/q2.txt (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/q3.txt (100%) rename {tests => 2.0/tests}/comparisonTest/cassandra/q4.txt (100%) rename {tests => 2.0/tests}/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java (100%) rename {tests => 2.0/tests}/comparisonTest/influxdb/main.go (100%) rename {tests => 2.0/tests}/comparisonTest/influxdb/q1.txt (100%) rename {tests => 2.0/tests}/comparisonTest/influxdb/q2.txt (100%) rename {tests => 2.0/tests}/comparisonTest/influxdb/q3.txt (100%) rename {tests => 2.0/tests}/comparisonTest/influxdb/q4.txt (100%) rename {tests => 2.0/tests}/comparisonTest/opentsdb/opentsdbtest/pom.xml (100%) rename {tests => 2.0/tests}/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java (100%) rename {tests => 2.0/tests}/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java (100%) rename {tests => 2.0/tests}/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/CMakeLists.txt (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/makefile (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/q1.txt (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/q2.txt (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/q3.txt (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/q4.txt (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/q5.txt (100%) rename {tests => 2.0/tests}/comparisonTest/tdengine/tdengineTest.c (100%) rename {tests => 2.0/tests}/connectorTest/C#Test/nanosupport/TDengineDriver.cs (100%) rename {tests => 2.0/tests}/connectorTest/C#Test/nanosupport/nanotest.cs (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nanosupport/nanosecondTest.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/cinterface.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/connection.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/constants.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/cursor.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/error.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/globalfunc.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/taosobjects.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/taosquery.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/nodetaos/taosresult.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/readme.md (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/tdengine.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/test/performance.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/test/test.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/test/testMicroseconds.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/test/testNanoseconds.js (100%) rename {tests => 2.0/tests}/connectorTest/nodejsTest/test/testSubscribe.js (100%) rename {tests => 2.0/tests}/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py (100%) rename {tests => 2.0/tests}/connectorTest/odbcTest/nanosupport/odbc.go (100%) rename {tests => 2.0/tests}/connectorTest/odbcTest/nanosupport/odbc.py (100%) rename {tests => 2.0/tests}/fuzz/sql-fuzzer.c (100%) rename {tests => 2.0/tests}/gotest/batchtest.bat (100%) rename {tests => 2.0/tests}/gotest/batchtest.sh (100%) rename {tests => 2.0/tests}/gotest/case001/case001.bat (100%) rename {tests => 2.0/tests}/gotest/case001/case001.go (100%) rename {tests => 2.0/tests}/gotest/case001/case001.sh (100%) rename {tests => 2.0/tests}/gotest/case002/case002.bat (100%) rename {tests => 2.0/tests}/gotest/case002/case002.go (100%) rename {tests => 2.0/tests}/gotest/case002/case002.sh (100%) rename {tests => 2.0/tests}/gotest/nanosupport/connector/executor.go (100%) rename {tests => 2.0/tests}/gotest/nanosupport/nanoCase.bat (100%) rename {tests => 2.0/tests}/gotest/nanosupport/nanoCase.sh (100%) rename {tests => 2.0/tests}/gotest/nanosupport/nanosupport.go (100%) rename {tests => 2.0/tests}/gotest/test.sh (100%) rename {tests => 2.0/tests}/http/restful/http_create_db.c (100%) rename {tests => 2.0/tests}/http/restful/http_create_tb.c (100%) rename {tests => 2.0/tests}/http/restful/http_drop_db.c (100%) rename {tests => 2.0/tests}/http/restful/http_insert_tb.c (100%) rename {tests => 2.0/tests}/http/restful/http_query_tb.c (100%) rename {tests => 2.0/tests}/http/restful/http_use_db.c (100%) rename {tests => 2.0/tests}/nettest/FQDNnettest.sh (100%) rename {tests => 2.0/tests}/nettest/TCPUDP.sh (100%) rename {tests => 2.0/tests}/nettest/fqdn.sh (100%) rename {tests => 2.0/tests}/nettest/tcpudp.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/cassandraTestQ1Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/cassandraTestQ2Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/cassandraTestQ3Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/cassandraTestQ4Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/cassandraTestWriteLoop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/coverage_test.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/full_test.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/influxdbTestQ1Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/influxdbTestQ2Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/influxdbTestQ3Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/influxdbTestQ4Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/influxdbTestWriteLoop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ1Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ2Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ3Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/opentsdbTestQ4Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/opentsdbTestWriteLoop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-consistent-inserting-data.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-csv2png.gnuplot (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-daily.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-query.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-taosdemo-compare.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-taosdemo.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-tsdb-compare-13d.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-tsdb-compare-1d.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/run-csv.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/runInfluxdb-13d-csv.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/runInfluxdb.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/runTDengine.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/runreal-13d-csv.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/runreal-1d-csv.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/taosdemo-csv2png.gnuplot (100%) rename {tests => 2.0/tests}/perftest-scripts/taosdemo-rps-csv2png.gnuplot (100%) rename {tests => 2.0/tests}/perftest-scripts/tdengineTestQ1Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdengineTestQ2Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdengineTestQ3Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdengineTestQ4Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdengineTestQ5Loop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdengineTestWriteLoop.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdinternal_coverage_test.sh (100%) rename {tests => 2.0/tests}/perftest-scripts/tdinternal_test.sh (100%) rename {tests => 2.0/tests}/robust/cluster.sh (100%) rename {tests => 2.0/tests}/robust/makefile (100%) rename {tests => 2.0/tests}/robust/monitor.sh (100%) rename {tests => 2.0/tests}/robust/robust.c (100%) rename {tests => 2.0/tests}/stress/.gitignore (100%) rename {tests => 2.0/tests}/stress/README.md (100%) rename {tests => 2.0/tests}/stress/go.mod (100%) rename {tests => 2.0/tests}/stress/main.go (100%) diff --git a/tests/comparisonTest/cassandra/application.conf b/2.0/tests/comparisonTest/cassandra/application.conf similarity index 100% rename from tests/comparisonTest/cassandra/application.conf rename to 2.0/tests/comparisonTest/cassandra/application.conf diff --git a/tests/comparisonTest/cassandra/cassandratest/pom.xml b/2.0/tests/comparisonTest/cassandra/cassandratest/pom.xml similarity index 100% rename from tests/comparisonTest/cassandra/cassandratest/pom.xml rename to 2.0/tests/comparisonTest/cassandra/cassandratest/pom.xml diff --git a/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java b/2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java similarity index 100% rename from tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java rename to 2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/CassandraTest.java diff --git a/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java b/2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java similarity index 100% rename from tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java rename to 2.0/tests/comparisonTest/cassandra/cassandratest/src/main/java/com/cassandra/test/WriteThread.java diff --git a/tests/comparisonTest/cassandra/q1.txt b/2.0/tests/comparisonTest/cassandra/q1.txt similarity index 100% rename from tests/comparisonTest/cassandra/q1.txt rename to 2.0/tests/comparisonTest/cassandra/q1.txt diff --git a/tests/comparisonTest/cassandra/q2.txt b/2.0/tests/comparisonTest/cassandra/q2.txt similarity index 100% rename from tests/comparisonTest/cassandra/q2.txt rename to 2.0/tests/comparisonTest/cassandra/q2.txt diff --git a/tests/comparisonTest/cassandra/q3.txt b/2.0/tests/comparisonTest/cassandra/q3.txt similarity index 100% rename from tests/comparisonTest/cassandra/q3.txt rename to 2.0/tests/comparisonTest/cassandra/q3.txt diff --git a/tests/comparisonTest/cassandra/q4.txt b/2.0/tests/comparisonTest/cassandra/q4.txt similarity index 100% rename from tests/comparisonTest/cassandra/q4.txt rename to 2.0/tests/comparisonTest/cassandra/q4.txt diff --git a/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java b/2.0/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java similarity index 100% rename from tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java rename to 2.0/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java diff --git a/tests/comparisonTest/influxdb/main.go b/2.0/tests/comparisonTest/influxdb/main.go similarity index 100% rename from tests/comparisonTest/influxdb/main.go rename to 2.0/tests/comparisonTest/influxdb/main.go diff --git a/tests/comparisonTest/influxdb/q1.txt b/2.0/tests/comparisonTest/influxdb/q1.txt similarity index 100% rename from tests/comparisonTest/influxdb/q1.txt rename to 2.0/tests/comparisonTest/influxdb/q1.txt diff --git a/tests/comparisonTest/influxdb/q2.txt b/2.0/tests/comparisonTest/influxdb/q2.txt similarity index 100% rename from tests/comparisonTest/influxdb/q2.txt rename to 2.0/tests/comparisonTest/influxdb/q2.txt diff --git a/tests/comparisonTest/influxdb/q3.txt b/2.0/tests/comparisonTest/influxdb/q3.txt similarity index 100% rename from tests/comparisonTest/influxdb/q3.txt rename to 2.0/tests/comparisonTest/influxdb/q3.txt diff --git a/tests/comparisonTest/influxdb/q4.txt b/2.0/tests/comparisonTest/influxdb/q4.txt similarity index 100% rename from tests/comparisonTest/influxdb/q4.txt rename to 2.0/tests/comparisonTest/influxdb/q4.txt diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/pom.xml rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/WriteThread.java diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml b/2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml similarity index 100% rename from tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml rename to 2.0/tests/comparisonTest/opentsdb/opentsdbtest/src/main/resources/logback.xml diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/2.0/tests/comparisonTest/tdengine/CMakeLists.txt similarity index 100% rename from tests/comparisonTest/tdengine/CMakeLists.txt rename to 2.0/tests/comparisonTest/tdengine/CMakeLists.txt diff --git a/tests/comparisonTest/tdengine/makefile b/2.0/tests/comparisonTest/tdengine/makefile similarity index 100% rename from tests/comparisonTest/tdengine/makefile rename to 2.0/tests/comparisonTest/tdengine/makefile diff --git a/tests/comparisonTest/tdengine/q1.txt b/2.0/tests/comparisonTest/tdengine/q1.txt similarity index 100% rename from tests/comparisonTest/tdengine/q1.txt rename to 2.0/tests/comparisonTest/tdengine/q1.txt diff --git a/tests/comparisonTest/tdengine/q2.txt b/2.0/tests/comparisonTest/tdengine/q2.txt similarity index 100% rename from tests/comparisonTest/tdengine/q2.txt rename to 2.0/tests/comparisonTest/tdengine/q2.txt diff --git a/tests/comparisonTest/tdengine/q3.txt b/2.0/tests/comparisonTest/tdengine/q3.txt similarity index 100% rename from tests/comparisonTest/tdengine/q3.txt rename to 2.0/tests/comparisonTest/tdengine/q3.txt diff --git a/tests/comparisonTest/tdengine/q4.txt b/2.0/tests/comparisonTest/tdengine/q4.txt similarity index 100% rename from tests/comparisonTest/tdengine/q4.txt rename to 2.0/tests/comparisonTest/tdengine/q4.txt diff --git a/tests/comparisonTest/tdengine/q5.txt b/2.0/tests/comparisonTest/tdengine/q5.txt similarity index 100% rename from tests/comparisonTest/tdengine/q5.txt rename to 2.0/tests/comparisonTest/tdengine/q5.txt diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/2.0/tests/comparisonTest/tdengine/tdengineTest.c similarity index 100% rename from tests/comparisonTest/tdengine/tdengineTest.c rename to 2.0/tests/comparisonTest/tdengine/tdengineTest.c diff --git a/tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs b/2.0/tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs similarity index 100% rename from tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs rename to 2.0/tests/connectorTest/C#Test/nanosupport/TDengineDriver.cs diff --git a/tests/connectorTest/C#Test/nanosupport/nanotest.cs b/2.0/tests/connectorTest/C#Test/nanosupport/nanotest.cs similarity index 100% rename from tests/connectorTest/C#Test/nanosupport/nanotest.cs rename to 2.0/tests/connectorTest/C#Test/nanosupport/nanotest.cs diff --git a/tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js b/2.0/tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js similarity index 100% rename from tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js rename to 2.0/tests/connectorTest/nodejsTest/nanosupport/nanosecondTest.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/cinterface.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/cinterface.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/cinterface.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/cinterface.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/connection.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/connection.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/connection.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/connection.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/constants.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/constants.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/constants.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/constants.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/cursor.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/cursor.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/cursor.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/cursor.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/error.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/error.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/error.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/error.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/globalfunc.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/globalfunc.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/globalfunc.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/globalfunc.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/taosobjects.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/taosobjects.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/taosobjects.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/taosobjects.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/taosquery.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/taosquery.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/taosquery.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/taosquery.js diff --git a/tests/connectorTest/nodejsTest/nodetaos/taosresult.js b/2.0/tests/connectorTest/nodejsTest/nodetaos/taosresult.js similarity index 100% rename from tests/connectorTest/nodejsTest/nodetaos/taosresult.js rename to 2.0/tests/connectorTest/nodejsTest/nodetaos/taosresult.js diff --git a/tests/connectorTest/nodejsTest/readme.md b/2.0/tests/connectorTest/nodejsTest/readme.md similarity index 100% rename from tests/connectorTest/nodejsTest/readme.md rename to 2.0/tests/connectorTest/nodejsTest/readme.md diff --git a/tests/connectorTest/nodejsTest/tdengine.js b/2.0/tests/connectorTest/nodejsTest/tdengine.js similarity index 100% rename from tests/connectorTest/nodejsTest/tdengine.js rename to 2.0/tests/connectorTest/nodejsTest/tdengine.js diff --git a/tests/connectorTest/nodejsTest/test/performance.js b/2.0/tests/connectorTest/nodejsTest/test/performance.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/performance.js rename to 2.0/tests/connectorTest/nodejsTest/test/performance.js diff --git a/tests/connectorTest/nodejsTest/test/test.js b/2.0/tests/connectorTest/nodejsTest/test/test.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/test.js rename to 2.0/tests/connectorTest/nodejsTest/test/test.js diff --git a/tests/connectorTest/nodejsTest/test/testMicroseconds.js b/2.0/tests/connectorTest/nodejsTest/test/testMicroseconds.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/testMicroseconds.js rename to 2.0/tests/connectorTest/nodejsTest/test/testMicroseconds.js diff --git a/tests/connectorTest/nodejsTest/test/testNanoseconds.js b/2.0/tests/connectorTest/nodejsTest/test/testNanoseconds.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/testNanoseconds.js rename to 2.0/tests/connectorTest/nodejsTest/test/testNanoseconds.js diff --git a/tests/connectorTest/nodejsTest/test/testSubscribe.js b/2.0/tests/connectorTest/nodejsTest/test/testSubscribe.js similarity index 100% rename from tests/connectorTest/nodejsTest/test/testSubscribe.js rename to 2.0/tests/connectorTest/nodejsTest/test/testSubscribe.js diff --git a/tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py b/2.0/tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py similarity index 100% rename from tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py rename to 2.0/tests/connectorTest/odbcTest/nanosupport/nanoTest_odbc.py diff --git a/tests/connectorTest/odbcTest/nanosupport/odbc.go b/2.0/tests/connectorTest/odbcTest/nanosupport/odbc.go similarity index 100% rename from tests/connectorTest/odbcTest/nanosupport/odbc.go rename to 2.0/tests/connectorTest/odbcTest/nanosupport/odbc.go diff --git a/tests/connectorTest/odbcTest/nanosupport/odbc.py b/2.0/tests/connectorTest/odbcTest/nanosupport/odbc.py similarity index 100% rename from tests/connectorTest/odbcTest/nanosupport/odbc.py rename to 2.0/tests/connectorTest/odbcTest/nanosupport/odbc.py diff --git a/tests/fuzz/sql-fuzzer.c b/2.0/tests/fuzz/sql-fuzzer.c similarity index 100% rename from tests/fuzz/sql-fuzzer.c rename to 2.0/tests/fuzz/sql-fuzzer.c diff --git a/tests/gotest/batchtest.bat b/2.0/tests/gotest/batchtest.bat similarity index 100% rename from tests/gotest/batchtest.bat rename to 2.0/tests/gotest/batchtest.bat diff --git a/tests/gotest/batchtest.sh b/2.0/tests/gotest/batchtest.sh similarity index 100% rename from tests/gotest/batchtest.sh rename to 2.0/tests/gotest/batchtest.sh diff --git a/tests/gotest/case001/case001.bat b/2.0/tests/gotest/case001/case001.bat similarity index 100% rename from tests/gotest/case001/case001.bat rename to 2.0/tests/gotest/case001/case001.bat diff --git a/tests/gotest/case001/case001.go b/2.0/tests/gotest/case001/case001.go similarity index 100% rename from tests/gotest/case001/case001.go rename to 2.0/tests/gotest/case001/case001.go diff --git a/tests/gotest/case001/case001.sh b/2.0/tests/gotest/case001/case001.sh similarity index 100% rename from tests/gotest/case001/case001.sh rename to 2.0/tests/gotest/case001/case001.sh diff --git a/tests/gotest/case002/case002.bat b/2.0/tests/gotest/case002/case002.bat similarity index 100% rename from tests/gotest/case002/case002.bat rename to 2.0/tests/gotest/case002/case002.bat diff --git a/tests/gotest/case002/case002.go b/2.0/tests/gotest/case002/case002.go similarity index 100% rename from tests/gotest/case002/case002.go rename to 2.0/tests/gotest/case002/case002.go diff --git a/tests/gotest/case002/case002.sh b/2.0/tests/gotest/case002/case002.sh similarity index 100% rename from tests/gotest/case002/case002.sh rename to 2.0/tests/gotest/case002/case002.sh diff --git a/tests/gotest/nanosupport/connector/executor.go b/2.0/tests/gotest/nanosupport/connector/executor.go similarity index 100% rename from tests/gotest/nanosupport/connector/executor.go rename to 2.0/tests/gotest/nanosupport/connector/executor.go diff --git a/tests/gotest/nanosupport/nanoCase.bat b/2.0/tests/gotest/nanosupport/nanoCase.bat similarity index 100% rename from tests/gotest/nanosupport/nanoCase.bat rename to 2.0/tests/gotest/nanosupport/nanoCase.bat diff --git a/tests/gotest/nanosupport/nanoCase.sh b/2.0/tests/gotest/nanosupport/nanoCase.sh similarity index 100% rename from tests/gotest/nanosupport/nanoCase.sh rename to 2.0/tests/gotest/nanosupport/nanoCase.sh diff --git a/tests/gotest/nanosupport/nanosupport.go b/2.0/tests/gotest/nanosupport/nanosupport.go similarity index 100% rename from tests/gotest/nanosupport/nanosupport.go rename to 2.0/tests/gotest/nanosupport/nanosupport.go diff --git a/tests/gotest/test.sh b/2.0/tests/gotest/test.sh similarity index 100% rename from tests/gotest/test.sh rename to 2.0/tests/gotest/test.sh diff --git a/tests/http/restful/http_create_db.c b/2.0/tests/http/restful/http_create_db.c similarity index 100% rename from tests/http/restful/http_create_db.c rename to 2.0/tests/http/restful/http_create_db.c diff --git a/tests/http/restful/http_create_tb.c b/2.0/tests/http/restful/http_create_tb.c similarity index 100% rename from tests/http/restful/http_create_tb.c rename to 2.0/tests/http/restful/http_create_tb.c diff --git a/tests/http/restful/http_drop_db.c b/2.0/tests/http/restful/http_drop_db.c similarity index 100% rename from tests/http/restful/http_drop_db.c rename to 2.0/tests/http/restful/http_drop_db.c diff --git a/tests/http/restful/http_insert_tb.c b/2.0/tests/http/restful/http_insert_tb.c similarity index 100% rename from tests/http/restful/http_insert_tb.c rename to 2.0/tests/http/restful/http_insert_tb.c diff --git a/tests/http/restful/http_query_tb.c b/2.0/tests/http/restful/http_query_tb.c similarity index 100% rename from tests/http/restful/http_query_tb.c rename to 2.0/tests/http/restful/http_query_tb.c diff --git a/tests/http/restful/http_use_db.c b/2.0/tests/http/restful/http_use_db.c similarity index 100% rename from tests/http/restful/http_use_db.c rename to 2.0/tests/http/restful/http_use_db.c diff --git a/tests/nettest/FQDNnettest.sh b/2.0/tests/nettest/FQDNnettest.sh similarity index 100% rename from tests/nettest/FQDNnettest.sh rename to 2.0/tests/nettest/FQDNnettest.sh diff --git a/tests/nettest/TCPUDP.sh b/2.0/tests/nettest/TCPUDP.sh similarity index 100% rename from tests/nettest/TCPUDP.sh rename to 2.0/tests/nettest/TCPUDP.sh diff --git a/tests/nettest/fqdn.sh b/2.0/tests/nettest/fqdn.sh similarity index 100% rename from tests/nettest/fqdn.sh rename to 2.0/tests/nettest/fqdn.sh diff --git a/tests/nettest/tcpudp.sh b/2.0/tests/nettest/tcpudp.sh similarity index 100% rename from tests/nettest/tcpudp.sh rename to 2.0/tests/nettest/tcpudp.sh diff --git a/tests/perftest-scripts/cassandraTestQ1Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ1Loop.sh diff --git a/tests/perftest-scripts/cassandraTestQ2Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ2Loop.sh diff --git a/tests/perftest-scripts/cassandraTestQ3Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ3Loop.sh diff --git a/tests/perftest-scripts/cassandraTestQ4Loop.sh b/2.0/tests/perftest-scripts/cassandraTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/cassandraTestQ4Loop.sh diff --git a/tests/perftest-scripts/cassandraTestWriteLoop.sh b/2.0/tests/perftest-scripts/cassandraTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/cassandraTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/cassandraTestWriteLoop.sh diff --git a/tests/perftest-scripts/coverage_test.sh b/2.0/tests/perftest-scripts/coverage_test.sh similarity index 100% rename from tests/perftest-scripts/coverage_test.sh rename to 2.0/tests/perftest-scripts/coverage_test.sh diff --git a/tests/perftest-scripts/full_test.sh b/2.0/tests/perftest-scripts/full_test.sh similarity index 100% rename from tests/perftest-scripts/full_test.sh rename to 2.0/tests/perftest-scripts/full_test.sh diff --git a/tests/perftest-scripts/influxdbTestQ1Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ1Loop.sh diff --git a/tests/perftest-scripts/influxdbTestQ2Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ2Loop.sh diff --git a/tests/perftest-scripts/influxdbTestQ3Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ3Loop.sh diff --git a/tests/perftest-scripts/influxdbTestQ4Loop.sh b/2.0/tests/perftest-scripts/influxdbTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/influxdbTestQ4Loop.sh diff --git a/tests/perftest-scripts/influxdbTestWriteLoop.sh b/2.0/tests/perftest-scripts/influxdbTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/influxdbTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/influxdbTestWriteLoop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ1Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ1Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ2Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ2Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ3Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ3Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ4Loop.sh b/2.0/tests/perftest-scripts/opentsdbTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestQ4Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestWriteLoop.sh b/2.0/tests/perftest-scripts/opentsdbTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/opentsdbTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/opentsdbTestWriteLoop.sh diff --git a/tests/perftest-scripts/perftest-consistent-inserting-data.sh b/2.0/tests/perftest-scripts/perftest-consistent-inserting-data.sh similarity index 100% rename from tests/perftest-scripts/perftest-consistent-inserting-data.sh rename to 2.0/tests/perftest-scripts/perftest-consistent-inserting-data.sh diff --git a/tests/perftest-scripts/perftest-csv2png.gnuplot b/2.0/tests/perftest-scripts/perftest-csv2png.gnuplot similarity index 100% rename from tests/perftest-scripts/perftest-csv2png.gnuplot rename to 2.0/tests/perftest-scripts/perftest-csv2png.gnuplot diff --git a/tests/perftest-scripts/perftest-daily.sh b/2.0/tests/perftest-scripts/perftest-daily.sh similarity index 100% rename from tests/perftest-scripts/perftest-daily.sh rename to 2.0/tests/perftest-scripts/perftest-daily.sh diff --git a/tests/perftest-scripts/perftest-query.sh b/2.0/tests/perftest-scripts/perftest-query.sh similarity index 100% rename from tests/perftest-scripts/perftest-query.sh rename to 2.0/tests/perftest-scripts/perftest-query.sh diff --git a/tests/perftest-scripts/perftest-taosdemo-compare.sh b/2.0/tests/perftest-scripts/perftest-taosdemo-compare.sh similarity index 100% rename from tests/perftest-scripts/perftest-taosdemo-compare.sh rename to 2.0/tests/perftest-scripts/perftest-taosdemo-compare.sh diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/2.0/tests/perftest-scripts/perftest-taosdemo.sh similarity index 100% rename from tests/perftest-scripts/perftest-taosdemo.sh rename to 2.0/tests/perftest-scripts/perftest-taosdemo.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-13d.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-13d.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-13d.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-1d.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-1d.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-1d.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh b/2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh similarity index 100% rename from tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh rename to 2.0/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh diff --git a/tests/perftest-scripts/run-csv.sh b/2.0/tests/perftest-scripts/run-csv.sh similarity index 100% rename from tests/perftest-scripts/run-csv.sh rename to 2.0/tests/perftest-scripts/run-csv.sh diff --git a/tests/perftest-scripts/runInfluxdb-13d-csv.sh b/2.0/tests/perftest-scripts/runInfluxdb-13d-csv.sh similarity index 100% rename from tests/perftest-scripts/runInfluxdb-13d-csv.sh rename to 2.0/tests/perftest-scripts/runInfluxdb-13d-csv.sh diff --git a/tests/perftest-scripts/runInfluxdb.sh b/2.0/tests/perftest-scripts/runInfluxdb.sh similarity index 100% rename from tests/perftest-scripts/runInfluxdb.sh rename to 2.0/tests/perftest-scripts/runInfluxdb.sh diff --git a/tests/perftest-scripts/runTDengine.sh b/2.0/tests/perftest-scripts/runTDengine.sh similarity index 100% rename from tests/perftest-scripts/runTDengine.sh rename to 2.0/tests/perftest-scripts/runTDengine.sh diff --git a/tests/perftest-scripts/runreal-13d-csv.sh b/2.0/tests/perftest-scripts/runreal-13d-csv.sh similarity index 100% rename from tests/perftest-scripts/runreal-13d-csv.sh rename to 2.0/tests/perftest-scripts/runreal-13d-csv.sh diff --git a/tests/perftest-scripts/runreal-1d-csv.sh b/2.0/tests/perftest-scripts/runreal-1d-csv.sh similarity index 100% rename from tests/perftest-scripts/runreal-1d-csv.sh rename to 2.0/tests/perftest-scripts/runreal-1d-csv.sh diff --git a/tests/perftest-scripts/taosdemo-csv2png.gnuplot b/2.0/tests/perftest-scripts/taosdemo-csv2png.gnuplot similarity index 100% rename from tests/perftest-scripts/taosdemo-csv2png.gnuplot rename to 2.0/tests/perftest-scripts/taosdemo-csv2png.gnuplot diff --git a/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot b/2.0/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot similarity index 100% rename from tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot rename to 2.0/tests/perftest-scripts/taosdemo-rps-csv2png.gnuplot diff --git a/tests/perftest-scripts/tdengineTestQ1Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ1Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ1Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ1Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ2Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ2Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ2Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ2Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ3Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ3Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ3Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ3Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ4Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ4Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ4Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ4Loop.sh diff --git a/tests/perftest-scripts/tdengineTestQ5Loop.sh b/2.0/tests/perftest-scripts/tdengineTestQ5Loop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestQ5Loop.sh rename to 2.0/tests/perftest-scripts/tdengineTestQ5Loop.sh diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/2.0/tests/perftest-scripts/tdengineTestWriteLoop.sh similarity index 100% rename from tests/perftest-scripts/tdengineTestWriteLoop.sh rename to 2.0/tests/perftest-scripts/tdengineTestWriteLoop.sh diff --git a/tests/perftest-scripts/tdinternal_coverage_test.sh b/2.0/tests/perftest-scripts/tdinternal_coverage_test.sh similarity index 100% rename from tests/perftest-scripts/tdinternal_coverage_test.sh rename to 2.0/tests/perftest-scripts/tdinternal_coverage_test.sh diff --git a/tests/perftest-scripts/tdinternal_test.sh b/2.0/tests/perftest-scripts/tdinternal_test.sh similarity index 100% rename from tests/perftest-scripts/tdinternal_test.sh rename to 2.0/tests/perftest-scripts/tdinternal_test.sh diff --git a/tests/robust/cluster.sh b/2.0/tests/robust/cluster.sh similarity index 100% rename from tests/robust/cluster.sh rename to 2.0/tests/robust/cluster.sh diff --git a/tests/robust/makefile b/2.0/tests/robust/makefile similarity index 100% rename from tests/robust/makefile rename to 2.0/tests/robust/makefile diff --git a/tests/robust/monitor.sh b/2.0/tests/robust/monitor.sh similarity index 100% rename from tests/robust/monitor.sh rename to 2.0/tests/robust/monitor.sh diff --git a/tests/robust/robust.c b/2.0/tests/robust/robust.c similarity index 100% rename from tests/robust/robust.c rename to 2.0/tests/robust/robust.c diff --git a/tests/stress/.gitignore b/2.0/tests/stress/.gitignore similarity index 100% rename from tests/stress/.gitignore rename to 2.0/tests/stress/.gitignore diff --git a/tests/stress/README.md b/2.0/tests/stress/README.md similarity index 100% rename from tests/stress/README.md rename to 2.0/tests/stress/README.md diff --git a/tests/stress/go.mod b/2.0/tests/stress/go.mod similarity index 100% rename from tests/stress/go.mod rename to 2.0/tests/stress/go.mod diff --git a/tests/stress/main.go b/2.0/tests/stress/main.go similarity index 100% rename from tests/stress/main.go rename to 2.0/tests/stress/main.go From 04f2d917320abaff48dfd6bd6555dd3b052e379e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:12:07 -0800 Subject: [PATCH 046/103] rename files --- {src => 2.0/src}/kit/taosdemo/CMakeLists.txt | 0 {src => 2.0/src}/kit/taosdemo/async-sub.json | 0 {src => 2.0/src}/kit/taosdemo/insert-interlace.json | 0 {src => 2.0/src}/kit/taosdemo/insert.json | 0 {src => 2.0/src}/kit/taosdemo/query.json | 0 {src => 2.0/src}/kit/taosdemo/subscribe.json | 0 {src => 2.0/src}/kit/taosdemo/taosdemo.c | 0 {src => 2.0/src}/kit/taosdump/CMakeLists.txt | 0 {src => 2.0/src}/kit/taosdump/taosdump.c | 0 {src => 2.0/src}/kit/taosdump/taosdump.sh | 0 {src => 2.0/src}/kit/taospack/CMakeLists.txt | 0 {src => 2.0/src}/kit/taospack/taospack.c | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename {src => 2.0/src}/kit/taosdemo/CMakeLists.txt (100%) rename {src => 2.0/src}/kit/taosdemo/async-sub.json (100%) rename {src => 2.0/src}/kit/taosdemo/insert-interlace.json (100%) rename {src => 2.0/src}/kit/taosdemo/insert.json (100%) rename {src => 2.0/src}/kit/taosdemo/query.json (100%) rename {src => 2.0/src}/kit/taosdemo/subscribe.json (100%) rename {src => 2.0/src}/kit/taosdemo/taosdemo.c (100%) rename {src => 2.0/src}/kit/taosdump/CMakeLists.txt (100%) rename {src => 2.0/src}/kit/taosdump/taosdump.c (100%) rename {src => 2.0/src}/kit/taosdump/taosdump.sh (100%) rename {src => 2.0/src}/kit/taospack/CMakeLists.txt (100%) rename {src => 2.0/src}/kit/taospack/taospack.c (100%) diff --git a/src/kit/taosdemo/CMakeLists.txt b/2.0/src/kit/taosdemo/CMakeLists.txt similarity index 100% rename from src/kit/taosdemo/CMakeLists.txt rename to 2.0/src/kit/taosdemo/CMakeLists.txt diff --git a/src/kit/taosdemo/async-sub.json b/2.0/src/kit/taosdemo/async-sub.json similarity index 100% rename from src/kit/taosdemo/async-sub.json rename to 2.0/src/kit/taosdemo/async-sub.json diff --git a/src/kit/taosdemo/insert-interlace.json b/2.0/src/kit/taosdemo/insert-interlace.json similarity index 100% rename from src/kit/taosdemo/insert-interlace.json rename to 2.0/src/kit/taosdemo/insert-interlace.json diff --git a/src/kit/taosdemo/insert.json b/2.0/src/kit/taosdemo/insert.json similarity index 100% rename from src/kit/taosdemo/insert.json rename to 2.0/src/kit/taosdemo/insert.json diff --git a/src/kit/taosdemo/query.json b/2.0/src/kit/taosdemo/query.json similarity index 100% rename from src/kit/taosdemo/query.json rename to 2.0/src/kit/taosdemo/query.json diff --git a/src/kit/taosdemo/subscribe.json b/2.0/src/kit/taosdemo/subscribe.json similarity index 100% rename from src/kit/taosdemo/subscribe.json rename to 2.0/src/kit/taosdemo/subscribe.json diff --git a/src/kit/taosdemo/taosdemo.c b/2.0/src/kit/taosdemo/taosdemo.c similarity index 100% rename from src/kit/taosdemo/taosdemo.c rename to 2.0/src/kit/taosdemo/taosdemo.c diff --git a/src/kit/taosdump/CMakeLists.txt b/2.0/src/kit/taosdump/CMakeLists.txt similarity index 100% rename from src/kit/taosdump/CMakeLists.txt rename to 2.0/src/kit/taosdump/CMakeLists.txt diff --git a/src/kit/taosdump/taosdump.c b/2.0/src/kit/taosdump/taosdump.c similarity index 100% rename from src/kit/taosdump/taosdump.c rename to 2.0/src/kit/taosdump/taosdump.c diff --git a/src/kit/taosdump/taosdump.sh b/2.0/src/kit/taosdump/taosdump.sh similarity index 100% rename from src/kit/taosdump/taosdump.sh rename to 2.0/src/kit/taosdump/taosdump.sh diff --git a/src/kit/taospack/CMakeLists.txt b/2.0/src/kit/taospack/CMakeLists.txt similarity index 100% rename from src/kit/taospack/CMakeLists.txt rename to 2.0/src/kit/taospack/CMakeLists.txt diff --git a/src/kit/taospack/taospack.c b/2.0/src/kit/taospack/taospack.c similarity index 100% rename from src/kit/taospack/taospack.c rename to 2.0/src/kit/taospack/taospack.c From 6bb9ccfe9cde6f8253d0521755dafbc6bc455aa2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:12:26 -0800 Subject: [PATCH 047/103] rename files --- {src => 2.0/src}/query/CMakeLists.txt | 0 {src => 2.0/src}/query/inc/qAggMain.h | 0 {src => 2.0/src}/query/inc/qExecutor.h | 0 {src => 2.0/src}/query/inc/qExtbuffer.h | 0 {src => 2.0/src}/query/inc/qFill.h | 0 {src => 2.0/src}/query/inc/qHistogram.h | 0 {src => 2.0/src}/query/inc/qPercentile.h | 0 {src => 2.0/src}/query/inc/qPlan.h | 0 {src => 2.0/src}/query/inc/qResultbuf.h | 0 {src => 2.0/src}/query/inc/qSqlparser.h | 0 {src => 2.0/src}/query/inc/qTableMeta.h | 0 {src => 2.0/src}/query/inc/qTsbuf.h | 0 {src => 2.0/src}/query/inc/qUdf.h | 0 {src => 2.0/src}/query/inc/queryLog.h | 0 {src => 2.0/src}/query/inc/sql.y | 0 {src => 2.0/src}/query/src/qAggMain.c | 0 {src => 2.0/src}/query/src/qExecutor.c | 0 {src => 2.0/src}/query/src/qExtbuffer.c | 0 {src => 2.0/src}/query/src/qFill.c | 0 {src => 2.0/src}/query/src/qFilter.c | 0 {src => 2.0/src}/query/src/qFilterfunc.c | 0 {src => 2.0/src}/query/src/qHistogram.c | 0 {src => 2.0/src}/query/src/qPercentile.c | 0 {src => 2.0/src}/query/src/qPlan.c | 0 {src => 2.0/src}/query/src/qResultbuf.c | 0 {src => 2.0/src}/query/src/qSqlParser.c | 0 {src => 2.0/src}/query/src/qTableMeta.c | 0 {src => 2.0/src}/query/src/qTsbuf.c | 0 {src => 2.0/src}/query/src/queryMain.c | 0 {src => 2.0/src}/query/src/sql.c | 0 {src => 2.0/src}/query/tests/CMakeLists.txt | 0 {src => 2.0/src}/query/tests/astTest.cpp | 0 {src => 2.0/src}/query/tests/cSortTest.cpp | 0 {src => 2.0/src}/query/tests/histogramTest.cpp | 0 {src => 2.0/src}/query/tests/patternMatchTest.cpp | 0 {src => 2.0/src}/query/tests/percentileTest.cpp | 0 {src => 2.0/src}/query/tests/rangeMergeTest.cpp | 0 {src => 2.0/src}/query/tests/resultBufferTest.cpp | 0 {src => 2.0/src}/query/tests/tsBufTest.cpp | 0 {src => 2.0/src}/query/tests/unitTest.cpp | 0 40 files changed, 0 insertions(+), 0 deletions(-) rename {src => 2.0/src}/query/CMakeLists.txt (100%) rename {src => 2.0/src}/query/inc/qAggMain.h (100%) rename {src => 2.0/src}/query/inc/qExecutor.h (100%) rename {src => 2.0/src}/query/inc/qExtbuffer.h (100%) rename {src => 2.0/src}/query/inc/qFill.h (100%) rename {src => 2.0/src}/query/inc/qHistogram.h (100%) rename {src => 2.0/src}/query/inc/qPercentile.h (100%) rename {src => 2.0/src}/query/inc/qPlan.h (100%) rename {src => 2.0/src}/query/inc/qResultbuf.h (100%) rename {src => 2.0/src}/query/inc/qSqlparser.h (100%) rename {src => 2.0/src}/query/inc/qTableMeta.h (100%) rename {src => 2.0/src}/query/inc/qTsbuf.h (100%) rename {src => 2.0/src}/query/inc/qUdf.h (100%) rename {src => 2.0/src}/query/inc/queryLog.h (100%) rename {src => 2.0/src}/query/inc/sql.y (100%) rename {src => 2.0/src}/query/src/qAggMain.c (100%) rename {src => 2.0/src}/query/src/qExecutor.c (100%) rename {src => 2.0/src}/query/src/qExtbuffer.c (100%) rename {src => 2.0/src}/query/src/qFill.c (100%) rename {src => 2.0/src}/query/src/qFilter.c (100%) rename {src => 2.0/src}/query/src/qFilterfunc.c (100%) rename {src => 2.0/src}/query/src/qHistogram.c (100%) rename {src => 2.0/src}/query/src/qPercentile.c (100%) rename {src => 2.0/src}/query/src/qPlan.c (100%) rename {src => 2.0/src}/query/src/qResultbuf.c (100%) rename {src => 2.0/src}/query/src/qSqlParser.c (100%) rename {src => 2.0/src}/query/src/qTableMeta.c (100%) rename {src => 2.0/src}/query/src/qTsbuf.c (100%) rename {src => 2.0/src}/query/src/queryMain.c (100%) rename {src => 2.0/src}/query/src/sql.c (100%) rename {src => 2.0/src}/query/tests/CMakeLists.txt (100%) rename {src => 2.0/src}/query/tests/astTest.cpp (100%) rename {src => 2.0/src}/query/tests/cSortTest.cpp (100%) rename {src => 2.0/src}/query/tests/histogramTest.cpp (100%) rename {src => 2.0/src}/query/tests/patternMatchTest.cpp (100%) rename {src => 2.0/src}/query/tests/percentileTest.cpp (100%) rename {src => 2.0/src}/query/tests/rangeMergeTest.cpp (100%) rename {src => 2.0/src}/query/tests/resultBufferTest.cpp (100%) rename {src => 2.0/src}/query/tests/tsBufTest.cpp (100%) rename {src => 2.0/src}/query/tests/unitTest.cpp (100%) diff --git a/src/query/CMakeLists.txt b/2.0/src/query/CMakeLists.txt similarity index 100% rename from src/query/CMakeLists.txt rename to 2.0/src/query/CMakeLists.txt diff --git a/src/query/inc/qAggMain.h b/2.0/src/query/inc/qAggMain.h similarity index 100% rename from src/query/inc/qAggMain.h rename to 2.0/src/query/inc/qAggMain.h diff --git a/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h similarity index 100% rename from src/query/inc/qExecutor.h rename to 2.0/src/query/inc/qExecutor.h diff --git a/src/query/inc/qExtbuffer.h b/2.0/src/query/inc/qExtbuffer.h similarity index 100% rename from src/query/inc/qExtbuffer.h rename to 2.0/src/query/inc/qExtbuffer.h diff --git a/src/query/inc/qFill.h b/2.0/src/query/inc/qFill.h similarity index 100% rename from src/query/inc/qFill.h rename to 2.0/src/query/inc/qFill.h diff --git a/src/query/inc/qHistogram.h b/2.0/src/query/inc/qHistogram.h similarity index 100% rename from src/query/inc/qHistogram.h rename to 2.0/src/query/inc/qHistogram.h diff --git a/src/query/inc/qPercentile.h b/2.0/src/query/inc/qPercentile.h similarity index 100% rename from src/query/inc/qPercentile.h rename to 2.0/src/query/inc/qPercentile.h diff --git a/src/query/inc/qPlan.h b/2.0/src/query/inc/qPlan.h similarity index 100% rename from src/query/inc/qPlan.h rename to 2.0/src/query/inc/qPlan.h diff --git a/src/query/inc/qResultbuf.h b/2.0/src/query/inc/qResultbuf.h similarity index 100% rename from src/query/inc/qResultbuf.h rename to 2.0/src/query/inc/qResultbuf.h diff --git a/src/query/inc/qSqlparser.h b/2.0/src/query/inc/qSqlparser.h similarity index 100% rename from src/query/inc/qSqlparser.h rename to 2.0/src/query/inc/qSqlparser.h diff --git a/src/query/inc/qTableMeta.h b/2.0/src/query/inc/qTableMeta.h similarity index 100% rename from src/query/inc/qTableMeta.h rename to 2.0/src/query/inc/qTableMeta.h diff --git a/src/query/inc/qTsbuf.h b/2.0/src/query/inc/qTsbuf.h similarity index 100% rename from src/query/inc/qTsbuf.h rename to 2.0/src/query/inc/qTsbuf.h diff --git a/src/query/inc/qUdf.h b/2.0/src/query/inc/qUdf.h similarity index 100% rename from src/query/inc/qUdf.h rename to 2.0/src/query/inc/qUdf.h diff --git a/src/query/inc/queryLog.h b/2.0/src/query/inc/queryLog.h similarity index 100% rename from src/query/inc/queryLog.h rename to 2.0/src/query/inc/queryLog.h diff --git a/src/query/inc/sql.y b/2.0/src/query/inc/sql.y similarity index 100% rename from src/query/inc/sql.y rename to 2.0/src/query/inc/sql.y diff --git a/src/query/src/qAggMain.c b/2.0/src/query/src/qAggMain.c similarity index 100% rename from src/query/src/qAggMain.c rename to 2.0/src/query/src/qAggMain.c diff --git a/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c similarity index 100% rename from src/query/src/qExecutor.c rename to 2.0/src/query/src/qExecutor.c diff --git a/src/query/src/qExtbuffer.c b/2.0/src/query/src/qExtbuffer.c similarity index 100% rename from src/query/src/qExtbuffer.c rename to 2.0/src/query/src/qExtbuffer.c diff --git a/src/query/src/qFill.c b/2.0/src/query/src/qFill.c similarity index 100% rename from src/query/src/qFill.c rename to 2.0/src/query/src/qFill.c diff --git a/src/query/src/qFilter.c b/2.0/src/query/src/qFilter.c similarity index 100% rename from src/query/src/qFilter.c rename to 2.0/src/query/src/qFilter.c diff --git a/src/query/src/qFilterfunc.c b/2.0/src/query/src/qFilterfunc.c similarity index 100% rename from src/query/src/qFilterfunc.c rename to 2.0/src/query/src/qFilterfunc.c diff --git a/src/query/src/qHistogram.c b/2.0/src/query/src/qHistogram.c similarity index 100% rename from src/query/src/qHistogram.c rename to 2.0/src/query/src/qHistogram.c diff --git a/src/query/src/qPercentile.c b/2.0/src/query/src/qPercentile.c similarity index 100% rename from src/query/src/qPercentile.c rename to 2.0/src/query/src/qPercentile.c diff --git a/src/query/src/qPlan.c b/2.0/src/query/src/qPlan.c similarity index 100% rename from src/query/src/qPlan.c rename to 2.0/src/query/src/qPlan.c diff --git a/src/query/src/qResultbuf.c b/2.0/src/query/src/qResultbuf.c similarity index 100% rename from src/query/src/qResultbuf.c rename to 2.0/src/query/src/qResultbuf.c diff --git a/src/query/src/qSqlParser.c b/2.0/src/query/src/qSqlParser.c similarity index 100% rename from src/query/src/qSqlParser.c rename to 2.0/src/query/src/qSqlParser.c diff --git a/src/query/src/qTableMeta.c b/2.0/src/query/src/qTableMeta.c similarity index 100% rename from src/query/src/qTableMeta.c rename to 2.0/src/query/src/qTableMeta.c diff --git a/src/query/src/qTsbuf.c b/2.0/src/query/src/qTsbuf.c similarity index 100% rename from src/query/src/qTsbuf.c rename to 2.0/src/query/src/qTsbuf.c diff --git a/src/query/src/queryMain.c b/2.0/src/query/src/queryMain.c similarity index 100% rename from src/query/src/queryMain.c rename to 2.0/src/query/src/queryMain.c diff --git a/src/query/src/sql.c b/2.0/src/query/src/sql.c similarity index 100% rename from src/query/src/sql.c rename to 2.0/src/query/src/sql.c diff --git a/src/query/tests/CMakeLists.txt b/2.0/src/query/tests/CMakeLists.txt similarity index 100% rename from src/query/tests/CMakeLists.txt rename to 2.0/src/query/tests/CMakeLists.txt diff --git a/src/query/tests/astTest.cpp b/2.0/src/query/tests/astTest.cpp similarity index 100% rename from src/query/tests/astTest.cpp rename to 2.0/src/query/tests/astTest.cpp diff --git a/src/query/tests/cSortTest.cpp b/2.0/src/query/tests/cSortTest.cpp similarity index 100% rename from src/query/tests/cSortTest.cpp rename to 2.0/src/query/tests/cSortTest.cpp diff --git a/src/query/tests/histogramTest.cpp b/2.0/src/query/tests/histogramTest.cpp similarity index 100% rename from src/query/tests/histogramTest.cpp rename to 2.0/src/query/tests/histogramTest.cpp diff --git a/src/query/tests/patternMatchTest.cpp b/2.0/src/query/tests/patternMatchTest.cpp similarity index 100% rename from src/query/tests/patternMatchTest.cpp rename to 2.0/src/query/tests/patternMatchTest.cpp diff --git a/src/query/tests/percentileTest.cpp b/2.0/src/query/tests/percentileTest.cpp similarity index 100% rename from src/query/tests/percentileTest.cpp rename to 2.0/src/query/tests/percentileTest.cpp diff --git a/src/query/tests/rangeMergeTest.cpp b/2.0/src/query/tests/rangeMergeTest.cpp similarity index 100% rename from src/query/tests/rangeMergeTest.cpp rename to 2.0/src/query/tests/rangeMergeTest.cpp diff --git a/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp similarity index 100% rename from src/query/tests/resultBufferTest.cpp rename to 2.0/src/query/tests/resultBufferTest.cpp diff --git a/src/query/tests/tsBufTest.cpp b/2.0/src/query/tests/tsBufTest.cpp similarity index 100% rename from src/query/tests/tsBufTest.cpp rename to 2.0/src/query/tests/tsBufTest.cpp diff --git a/src/query/tests/unitTest.cpp b/2.0/src/query/tests/unitTest.cpp similarity index 100% rename from src/query/tests/unitTest.cpp rename to 2.0/src/query/tests/unitTest.cpp From c1f0aa57d13fa126861255b01cc2a986eca2fddf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:12:50 -0800 Subject: [PATCH 048/103] rename files --- {src => 2.0/src}/client/CMakeLists.txt | 0 {src => 2.0/src}/client/inc/tscGlobalmerge.h | 0 {src => 2.0/src}/client/inc/tscLog.h | 0 {src => 2.0/src}/client/inc/tscParseLine.h | 0 {src => 2.0/src}/client/inc/tscProfile.h | 0 {src => 2.0/src}/client/inc/tscSubquery.h | 0 {src => 2.0/src}/client/inc/tscUtil.h | 0 {src => 2.0/src}/client/inc/tsclient.h | 0 .../client/jni/com_taosdata_jdbc_TSDBJNIConnector.h | 0 {src => 2.0/src}/client/src/TSDBJNIConnector.c | 0 {src => 2.0/src}/client/src/taos.def | 0 {src => 2.0/src}/client/src/taos.rc.in | 0 {src => 2.0/src}/client/src/tscAsync.c | 0 {src => 2.0/src}/client/src/tscGlobalmerge.c | 0 {src => 2.0/src}/client/src/tscLocal.c | 0 {src => 2.0/src}/client/src/tscParseInsert.c | 0 {src => 2.0/src}/client/src/tscParseLineProtocol.c | 0 {src => 2.0/src}/client/src/tscParseOpenTSDB.c | 0 {src => 2.0/src}/client/src/tscPrepare.c | 0 {src => 2.0/src}/client/src/tscProfile.c | 0 {src => 2.0/src}/client/src/tscSQLParser.c | 0 {src => 2.0/src}/client/src/tscServer.c | 0 {src => 2.0/src}/client/src/tscSql.c | 0 {src => 2.0/src}/client/src/tscStream.c | 0 {src => 2.0/src}/client/src/tscSub.c | 0 {src => 2.0/src}/client/src/tscSubquery.c | 0 {src => 2.0/src}/client/src/tscSystem.c | 0 {src => 2.0/src}/client/src/tscUtil.c | 0 {src => 2.0/src}/client/tests/CMakeLists.txt | 0 {src => 2.0/src}/client/tests/cliTest.cpp | 0 {src => 2.0/src}/client/tests/timeParseTest.cpp | 0 src/CMakeLists.txt | 13 ------------- 32 files changed, 13 deletions(-) rename {src => 2.0/src}/client/CMakeLists.txt (100%) rename {src => 2.0/src}/client/inc/tscGlobalmerge.h (100%) rename {src => 2.0/src}/client/inc/tscLog.h (100%) rename {src => 2.0/src}/client/inc/tscParseLine.h (100%) rename {src => 2.0/src}/client/inc/tscProfile.h (100%) rename {src => 2.0/src}/client/inc/tscSubquery.h (100%) rename {src => 2.0/src}/client/inc/tscUtil.h (100%) rename {src => 2.0/src}/client/inc/tsclient.h (100%) rename {src => 2.0/src}/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h (100%) rename {src => 2.0/src}/client/src/TSDBJNIConnector.c (100%) rename {src => 2.0/src}/client/src/taos.def (100%) rename {src => 2.0/src}/client/src/taos.rc.in (100%) rename {src => 2.0/src}/client/src/tscAsync.c (100%) rename {src => 2.0/src}/client/src/tscGlobalmerge.c (100%) rename {src => 2.0/src}/client/src/tscLocal.c (100%) rename {src => 2.0/src}/client/src/tscParseInsert.c (100%) rename {src => 2.0/src}/client/src/tscParseLineProtocol.c (100%) rename {src => 2.0/src}/client/src/tscParseOpenTSDB.c (100%) rename {src => 2.0/src}/client/src/tscPrepare.c (100%) rename {src => 2.0/src}/client/src/tscProfile.c (100%) rename {src => 2.0/src}/client/src/tscSQLParser.c (100%) rename {src => 2.0/src}/client/src/tscServer.c (100%) rename {src => 2.0/src}/client/src/tscSql.c (100%) rename {src => 2.0/src}/client/src/tscStream.c (100%) rename {src => 2.0/src}/client/src/tscSub.c (100%) rename {src => 2.0/src}/client/src/tscSubquery.c (100%) rename {src => 2.0/src}/client/src/tscSystem.c (100%) rename {src => 2.0/src}/client/src/tscUtil.c (100%) rename {src => 2.0/src}/client/tests/CMakeLists.txt (100%) rename {src => 2.0/src}/client/tests/cliTest.cpp (100%) rename {src => 2.0/src}/client/tests/timeParseTest.cpp (100%) delete mode 100644 src/CMakeLists.txt diff --git a/src/client/CMakeLists.txt b/2.0/src/client/CMakeLists.txt similarity index 100% rename from src/client/CMakeLists.txt rename to 2.0/src/client/CMakeLists.txt diff --git a/src/client/inc/tscGlobalmerge.h b/2.0/src/client/inc/tscGlobalmerge.h similarity index 100% rename from src/client/inc/tscGlobalmerge.h rename to 2.0/src/client/inc/tscGlobalmerge.h diff --git a/src/client/inc/tscLog.h b/2.0/src/client/inc/tscLog.h similarity index 100% rename from src/client/inc/tscLog.h rename to 2.0/src/client/inc/tscLog.h diff --git a/src/client/inc/tscParseLine.h b/2.0/src/client/inc/tscParseLine.h similarity index 100% rename from src/client/inc/tscParseLine.h rename to 2.0/src/client/inc/tscParseLine.h diff --git a/src/client/inc/tscProfile.h b/2.0/src/client/inc/tscProfile.h similarity index 100% rename from src/client/inc/tscProfile.h rename to 2.0/src/client/inc/tscProfile.h diff --git a/src/client/inc/tscSubquery.h b/2.0/src/client/inc/tscSubquery.h similarity index 100% rename from src/client/inc/tscSubquery.h rename to 2.0/src/client/inc/tscSubquery.h diff --git a/src/client/inc/tscUtil.h b/2.0/src/client/inc/tscUtil.h similarity index 100% rename from src/client/inc/tscUtil.h rename to 2.0/src/client/inc/tscUtil.h diff --git a/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h similarity index 100% rename from src/client/inc/tsclient.h rename to 2.0/src/client/inc/tsclient.h diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h similarity index 100% rename from src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h rename to 2.0/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h diff --git a/src/client/src/TSDBJNIConnector.c b/2.0/src/client/src/TSDBJNIConnector.c similarity index 100% rename from src/client/src/TSDBJNIConnector.c rename to 2.0/src/client/src/TSDBJNIConnector.c diff --git a/src/client/src/taos.def b/2.0/src/client/src/taos.def similarity index 100% rename from src/client/src/taos.def rename to 2.0/src/client/src/taos.def diff --git a/src/client/src/taos.rc.in b/2.0/src/client/src/taos.rc.in similarity index 100% rename from src/client/src/taos.rc.in rename to 2.0/src/client/src/taos.rc.in diff --git a/src/client/src/tscAsync.c b/2.0/src/client/src/tscAsync.c similarity index 100% rename from src/client/src/tscAsync.c rename to 2.0/src/client/src/tscAsync.c diff --git a/src/client/src/tscGlobalmerge.c b/2.0/src/client/src/tscGlobalmerge.c similarity index 100% rename from src/client/src/tscGlobalmerge.c rename to 2.0/src/client/src/tscGlobalmerge.c diff --git a/src/client/src/tscLocal.c b/2.0/src/client/src/tscLocal.c similarity index 100% rename from src/client/src/tscLocal.c rename to 2.0/src/client/src/tscLocal.c diff --git a/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c similarity index 100% rename from src/client/src/tscParseInsert.c rename to 2.0/src/client/src/tscParseInsert.c diff --git a/src/client/src/tscParseLineProtocol.c b/2.0/src/client/src/tscParseLineProtocol.c similarity index 100% rename from src/client/src/tscParseLineProtocol.c rename to 2.0/src/client/src/tscParseLineProtocol.c diff --git a/src/client/src/tscParseOpenTSDB.c b/2.0/src/client/src/tscParseOpenTSDB.c similarity index 100% rename from src/client/src/tscParseOpenTSDB.c rename to 2.0/src/client/src/tscParseOpenTSDB.c diff --git a/src/client/src/tscPrepare.c b/2.0/src/client/src/tscPrepare.c similarity index 100% rename from src/client/src/tscPrepare.c rename to 2.0/src/client/src/tscPrepare.c diff --git a/src/client/src/tscProfile.c b/2.0/src/client/src/tscProfile.c similarity index 100% rename from src/client/src/tscProfile.c rename to 2.0/src/client/src/tscProfile.c diff --git a/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c similarity index 100% rename from src/client/src/tscSQLParser.c rename to 2.0/src/client/src/tscSQLParser.c diff --git a/src/client/src/tscServer.c b/2.0/src/client/src/tscServer.c similarity index 100% rename from src/client/src/tscServer.c rename to 2.0/src/client/src/tscServer.c diff --git a/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c similarity index 100% rename from src/client/src/tscSql.c rename to 2.0/src/client/src/tscSql.c diff --git a/src/client/src/tscStream.c b/2.0/src/client/src/tscStream.c similarity index 100% rename from src/client/src/tscStream.c rename to 2.0/src/client/src/tscStream.c diff --git a/src/client/src/tscSub.c b/2.0/src/client/src/tscSub.c similarity index 100% rename from src/client/src/tscSub.c rename to 2.0/src/client/src/tscSub.c diff --git a/src/client/src/tscSubquery.c b/2.0/src/client/src/tscSubquery.c similarity index 100% rename from src/client/src/tscSubquery.c rename to 2.0/src/client/src/tscSubquery.c diff --git a/src/client/src/tscSystem.c b/2.0/src/client/src/tscSystem.c similarity index 100% rename from src/client/src/tscSystem.c rename to 2.0/src/client/src/tscSystem.c diff --git a/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c similarity index 100% rename from src/client/src/tscUtil.c rename to 2.0/src/client/src/tscUtil.c diff --git a/src/client/tests/CMakeLists.txt b/2.0/src/client/tests/CMakeLists.txt similarity index 100% rename from src/client/tests/CMakeLists.txt rename to 2.0/src/client/tests/CMakeLists.txt diff --git a/src/client/tests/cliTest.cpp b/2.0/src/client/tests/cliTest.cpp similarity index 100% rename from src/client/tests/cliTest.cpp rename to 2.0/src/client/tests/cliTest.cpp diff --git a/src/client/tests/timeParseTest.cpp b/2.0/src/client/tests/timeParseTest.cpp similarity index 100% rename from src/client/tests/timeParseTest.cpp rename to 2.0/src/client/tests/timeParseTest.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index b35cf60dab..0000000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Base components -add_subdirectory(os) -add_subdirectory(util) -add_subdirectory(common) - -# Library components - -# Service components -# add_subdirectory(mnode) -# add_subdirectory(vnode) -# add_subdirectory(qnode) -add_subdirectory(dnode) - From 3bd11e7d969ffbbd58960938842e5b4c077de89c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:17:19 -0800 Subject: [PATCH 049/103] rename files --- {src => 2.0/src}/connector/C#/TDengineDriver.cs | 0 src/connector/go | 1 - src/connector/grafanaplugin | 1 - src/connector/hivemq-tdengine-extension | 1 - 4 files changed, 3 deletions(-) rename {src => 2.0/src}/connector/C#/TDengineDriver.cs (100%) delete mode 160000 src/connector/go delete mode 160000 src/connector/grafanaplugin delete mode 160000 src/connector/hivemq-tdengine-extension diff --git a/src/connector/C#/TDengineDriver.cs b/2.0/src/connector/C#/TDengineDriver.cs similarity index 100% rename from src/connector/C#/TDengineDriver.cs rename to 2.0/src/connector/C#/TDengineDriver.cs diff --git a/src/connector/go b/src/connector/go deleted file mode 160000 index b8f76da4a7..0000000000 --- a/src/connector/go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin deleted file mode 160000 index 4a4d79099b..0000000000 --- a/src/connector/grafanaplugin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension deleted file mode 160000 index ce52010141..0000000000 --- a/src/connector/hivemq-tdengine-extension +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ce5201014136503d34fecbd56494b67b4961056c From 75566ab84afbc40cae6bd3342cf41b7e5df86975 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:17:57 -0800 Subject: [PATCH 050/103] rename files --- {src => 2.0/src}/connector/odbc/.gitignore | 0 {src => 2.0/src}/connector/odbc/CMakeLists.txt | 0 {src => 2.0/src}/connector/odbc/README.cn.md | 0 {src => 2.0/src}/connector/odbc/README.md | 0 {src => 2.0/src}/connector/odbc/examples/CMakeLists.txt | 0 {src => 2.0/src}/connector/odbc/examples/c/CMakeLists.txt | 0 {src => 2.0/src}/connector/odbc/examples/c/main.c | 0 {src => 2.0/src}/connector/odbc/examples/c/main.cpp | 0 {src => 2.0/src}/connector/odbc/examples/go/odbc.go | 0 {src => 2.0/src}/connector/odbc/examples/js/.gitignore | 0 {src => 2.0/src}/connector/odbc/examples/js/odbc.js | 0 {src => 2.0/src}/connector/odbc/examples/js/package.json | 0 {src => 2.0/src}/connector/odbc/examples/lua/odbc.lua | 0 {src => 2.0/src}/connector/odbc/examples/py/odbc.py | 0 {src => 2.0/src}/connector/odbc/examples/rust/.gitignore | 0 {src => 2.0/src}/connector/odbc/examples/rust/main/Cargo.toml | 0 {src => 2.0/src}/connector/odbc/examples/rust/main/src/main.rs | 0 {src => 2.0/src}/connector/odbc/samples/create_data.stmts | 0 {src => 2.0/src}/connector/odbc/samples/query_data.stmts | 0 {src => 2.0/src}/connector/odbc/samples/select.stmts | 0 {src => 2.0/src}/connector/odbc/samples/simples.stmts | 0 {src => 2.0/src}/connector/odbc/src/CMakeLists.txt | 0 {src => 2.0/src}/connector/odbc/src/base.c | 0 {src => 2.0/src}/connector/odbc/src/base.h | 0 {src => 2.0/src}/connector/odbc/src/base/CMakeLists.txt | 0 {src => 2.0/src}/connector/odbc/src/base/conn.c | 0 {src => 2.0/src}/connector/odbc/src/base/conn.h | 0 {src => 2.0/src}/connector/odbc/src/base/env.c | 0 {src => 2.0/src}/connector/odbc/src/base/env.h | 0 {src => 2.0/src}/connector/odbc/src/base/err.c | 0 {src => 2.0/src}/connector/odbc/src/base/err.h | 0 {src => 2.0/src}/connector/odbc/src/base/field.c | 0 {src => 2.0/src}/connector/odbc/src/base/field.h | 0 {src => 2.0/src}/connector/odbc/src/base/null_conn.c | 0 {src => 2.0/src}/connector/odbc/src/base/null_conn.h | 0 {src => 2.0/src}/connector/odbc/src/base/param.c | 0 {src => 2.0/src}/connector/odbc/src/base/param.h | 0 {src => 2.0/src}/connector/odbc/src/base/rs.c | 0 {src => 2.0/src}/connector/odbc/src/base/rs.h | 0 {src => 2.0/src}/connector/odbc/src/base/stmt.c | 0 {src => 2.0/src}/connector/odbc/src/base/stmt.h | 0 {src => 2.0/src}/connector/odbc/src/base/tsdb_impl.c | 0 {src => 2.0/src}/connector/odbc/src/base/tsdb_impl.h | 0 {src => 2.0/src}/connector/odbc/src/col.h | 0 {src => 2.0/src}/connector/odbc/src/install.cmd | 0 {src => 2.0/src}/connector/odbc/src/install.sh | 0 {src => 2.0/src}/connector/odbc/src/todbc.def | 0 {src => 2.0/src}/connector/odbc/src/todbc.rc.in | 0 {src => 2.0/src}/connector/odbc/src/todbc.rsp | 0 {src => 2.0/src}/connector/odbc/src/todbc_buf.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_buf.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_flex.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_hash.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_hash.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_iconv.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_iconv.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_list.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_list.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_log.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_log.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_scanner.l | 0 {src => 2.0/src}/connector/odbc/src/todbc_string.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_string.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_tls.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_tls.h | 0 {src => 2.0/src}/connector/odbc/src/todbc_util.c | 0 {src => 2.0/src}/connector/odbc/src/todbc_util.h | 0 {src => 2.0/src}/connector/odbc/tools/CMakeLists.txt | 0 {src => 2.0/src}/connector/odbc/tools/main.c | 0 {src => 2.0/src}/connector/odbc/tools/tconv.c | 0 70 files changed, 0 insertions(+), 0 deletions(-) rename {src => 2.0/src}/connector/odbc/.gitignore (100%) rename {src => 2.0/src}/connector/odbc/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/odbc/README.cn.md (100%) rename {src => 2.0/src}/connector/odbc/README.md (100%) rename {src => 2.0/src}/connector/odbc/examples/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/odbc/examples/c/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/odbc/examples/c/main.c (100%) rename {src => 2.0/src}/connector/odbc/examples/c/main.cpp (100%) rename {src => 2.0/src}/connector/odbc/examples/go/odbc.go (100%) rename {src => 2.0/src}/connector/odbc/examples/js/.gitignore (100%) rename {src => 2.0/src}/connector/odbc/examples/js/odbc.js (100%) rename {src => 2.0/src}/connector/odbc/examples/js/package.json (100%) rename {src => 2.0/src}/connector/odbc/examples/lua/odbc.lua (100%) rename {src => 2.0/src}/connector/odbc/examples/py/odbc.py (100%) rename {src => 2.0/src}/connector/odbc/examples/rust/.gitignore (100%) rename {src => 2.0/src}/connector/odbc/examples/rust/main/Cargo.toml (100%) rename {src => 2.0/src}/connector/odbc/examples/rust/main/src/main.rs (100%) rename {src => 2.0/src}/connector/odbc/samples/create_data.stmts (100%) rename {src => 2.0/src}/connector/odbc/samples/query_data.stmts (100%) rename {src => 2.0/src}/connector/odbc/samples/select.stmts (100%) rename {src => 2.0/src}/connector/odbc/samples/simples.stmts (100%) rename {src => 2.0/src}/connector/odbc/src/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/odbc/src/base.c (100%) rename {src => 2.0/src}/connector/odbc/src/base.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/odbc/src/base/conn.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/conn.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/env.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/env.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/err.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/err.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/field.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/field.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/null_conn.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/null_conn.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/param.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/param.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/rs.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/rs.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/stmt.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/stmt.h (100%) rename {src => 2.0/src}/connector/odbc/src/base/tsdb_impl.c (100%) rename {src => 2.0/src}/connector/odbc/src/base/tsdb_impl.h (100%) rename {src => 2.0/src}/connector/odbc/src/col.h (100%) rename {src => 2.0/src}/connector/odbc/src/install.cmd (100%) rename {src => 2.0/src}/connector/odbc/src/install.sh (100%) rename {src => 2.0/src}/connector/odbc/src/todbc.def (100%) rename {src => 2.0/src}/connector/odbc/src/todbc.rc.in (100%) rename {src => 2.0/src}/connector/odbc/src/todbc.rsp (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_buf.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_buf.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_flex.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_hash.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_hash.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_iconv.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_iconv.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_list.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_list.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_log.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_log.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_scanner.l (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_string.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_string.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_tls.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_tls.h (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_util.c (100%) rename {src => 2.0/src}/connector/odbc/src/todbc_util.h (100%) rename {src => 2.0/src}/connector/odbc/tools/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/odbc/tools/main.c (100%) rename {src => 2.0/src}/connector/odbc/tools/tconv.c (100%) diff --git a/src/connector/odbc/.gitignore b/2.0/src/connector/odbc/.gitignore similarity index 100% rename from src/connector/odbc/.gitignore rename to 2.0/src/connector/odbc/.gitignore diff --git a/src/connector/odbc/CMakeLists.txt b/2.0/src/connector/odbc/CMakeLists.txt similarity index 100% rename from src/connector/odbc/CMakeLists.txt rename to 2.0/src/connector/odbc/CMakeLists.txt diff --git a/src/connector/odbc/README.cn.md b/2.0/src/connector/odbc/README.cn.md similarity index 100% rename from src/connector/odbc/README.cn.md rename to 2.0/src/connector/odbc/README.cn.md diff --git a/src/connector/odbc/README.md b/2.0/src/connector/odbc/README.md similarity index 100% rename from src/connector/odbc/README.md rename to 2.0/src/connector/odbc/README.md diff --git a/src/connector/odbc/examples/CMakeLists.txt b/2.0/src/connector/odbc/examples/CMakeLists.txt similarity index 100% rename from src/connector/odbc/examples/CMakeLists.txt rename to 2.0/src/connector/odbc/examples/CMakeLists.txt diff --git a/src/connector/odbc/examples/c/CMakeLists.txt b/2.0/src/connector/odbc/examples/c/CMakeLists.txt similarity index 100% rename from src/connector/odbc/examples/c/CMakeLists.txt rename to 2.0/src/connector/odbc/examples/c/CMakeLists.txt diff --git a/src/connector/odbc/examples/c/main.c b/2.0/src/connector/odbc/examples/c/main.c similarity index 100% rename from src/connector/odbc/examples/c/main.c rename to 2.0/src/connector/odbc/examples/c/main.c diff --git a/src/connector/odbc/examples/c/main.cpp b/2.0/src/connector/odbc/examples/c/main.cpp similarity index 100% rename from src/connector/odbc/examples/c/main.cpp rename to 2.0/src/connector/odbc/examples/c/main.cpp diff --git a/src/connector/odbc/examples/go/odbc.go b/2.0/src/connector/odbc/examples/go/odbc.go similarity index 100% rename from src/connector/odbc/examples/go/odbc.go rename to 2.0/src/connector/odbc/examples/go/odbc.go diff --git a/src/connector/odbc/examples/js/.gitignore b/2.0/src/connector/odbc/examples/js/.gitignore similarity index 100% rename from src/connector/odbc/examples/js/.gitignore rename to 2.0/src/connector/odbc/examples/js/.gitignore diff --git a/src/connector/odbc/examples/js/odbc.js b/2.0/src/connector/odbc/examples/js/odbc.js similarity index 100% rename from src/connector/odbc/examples/js/odbc.js rename to 2.0/src/connector/odbc/examples/js/odbc.js diff --git a/src/connector/odbc/examples/js/package.json b/2.0/src/connector/odbc/examples/js/package.json similarity index 100% rename from src/connector/odbc/examples/js/package.json rename to 2.0/src/connector/odbc/examples/js/package.json diff --git a/src/connector/odbc/examples/lua/odbc.lua b/2.0/src/connector/odbc/examples/lua/odbc.lua similarity index 100% rename from src/connector/odbc/examples/lua/odbc.lua rename to 2.0/src/connector/odbc/examples/lua/odbc.lua diff --git a/src/connector/odbc/examples/py/odbc.py b/2.0/src/connector/odbc/examples/py/odbc.py similarity index 100% rename from src/connector/odbc/examples/py/odbc.py rename to 2.0/src/connector/odbc/examples/py/odbc.py diff --git a/src/connector/odbc/examples/rust/.gitignore b/2.0/src/connector/odbc/examples/rust/.gitignore similarity index 100% rename from src/connector/odbc/examples/rust/.gitignore rename to 2.0/src/connector/odbc/examples/rust/.gitignore diff --git a/src/connector/odbc/examples/rust/main/Cargo.toml b/2.0/src/connector/odbc/examples/rust/main/Cargo.toml similarity index 100% rename from src/connector/odbc/examples/rust/main/Cargo.toml rename to 2.0/src/connector/odbc/examples/rust/main/Cargo.toml diff --git a/src/connector/odbc/examples/rust/main/src/main.rs b/2.0/src/connector/odbc/examples/rust/main/src/main.rs similarity index 100% rename from src/connector/odbc/examples/rust/main/src/main.rs rename to 2.0/src/connector/odbc/examples/rust/main/src/main.rs diff --git a/src/connector/odbc/samples/create_data.stmts b/2.0/src/connector/odbc/samples/create_data.stmts similarity index 100% rename from src/connector/odbc/samples/create_data.stmts rename to 2.0/src/connector/odbc/samples/create_data.stmts diff --git a/src/connector/odbc/samples/query_data.stmts b/2.0/src/connector/odbc/samples/query_data.stmts similarity index 100% rename from src/connector/odbc/samples/query_data.stmts rename to 2.0/src/connector/odbc/samples/query_data.stmts diff --git a/src/connector/odbc/samples/select.stmts b/2.0/src/connector/odbc/samples/select.stmts similarity index 100% rename from src/connector/odbc/samples/select.stmts rename to 2.0/src/connector/odbc/samples/select.stmts diff --git a/src/connector/odbc/samples/simples.stmts b/2.0/src/connector/odbc/samples/simples.stmts similarity index 100% rename from src/connector/odbc/samples/simples.stmts rename to 2.0/src/connector/odbc/samples/simples.stmts diff --git a/src/connector/odbc/src/CMakeLists.txt b/2.0/src/connector/odbc/src/CMakeLists.txt similarity index 100% rename from src/connector/odbc/src/CMakeLists.txt rename to 2.0/src/connector/odbc/src/CMakeLists.txt diff --git a/src/connector/odbc/src/base.c b/2.0/src/connector/odbc/src/base.c similarity index 100% rename from src/connector/odbc/src/base.c rename to 2.0/src/connector/odbc/src/base.c diff --git a/src/connector/odbc/src/base.h b/2.0/src/connector/odbc/src/base.h similarity index 100% rename from src/connector/odbc/src/base.h rename to 2.0/src/connector/odbc/src/base.h diff --git a/src/connector/odbc/src/base/CMakeLists.txt b/2.0/src/connector/odbc/src/base/CMakeLists.txt similarity index 100% rename from src/connector/odbc/src/base/CMakeLists.txt rename to 2.0/src/connector/odbc/src/base/CMakeLists.txt diff --git a/src/connector/odbc/src/base/conn.c b/2.0/src/connector/odbc/src/base/conn.c similarity index 100% rename from src/connector/odbc/src/base/conn.c rename to 2.0/src/connector/odbc/src/base/conn.c diff --git a/src/connector/odbc/src/base/conn.h b/2.0/src/connector/odbc/src/base/conn.h similarity index 100% rename from src/connector/odbc/src/base/conn.h rename to 2.0/src/connector/odbc/src/base/conn.h diff --git a/src/connector/odbc/src/base/env.c b/2.0/src/connector/odbc/src/base/env.c similarity index 100% rename from src/connector/odbc/src/base/env.c rename to 2.0/src/connector/odbc/src/base/env.c diff --git a/src/connector/odbc/src/base/env.h b/2.0/src/connector/odbc/src/base/env.h similarity index 100% rename from src/connector/odbc/src/base/env.h rename to 2.0/src/connector/odbc/src/base/env.h diff --git a/src/connector/odbc/src/base/err.c b/2.0/src/connector/odbc/src/base/err.c similarity index 100% rename from src/connector/odbc/src/base/err.c rename to 2.0/src/connector/odbc/src/base/err.c diff --git a/src/connector/odbc/src/base/err.h b/2.0/src/connector/odbc/src/base/err.h similarity index 100% rename from src/connector/odbc/src/base/err.h rename to 2.0/src/connector/odbc/src/base/err.h diff --git a/src/connector/odbc/src/base/field.c b/2.0/src/connector/odbc/src/base/field.c similarity index 100% rename from src/connector/odbc/src/base/field.c rename to 2.0/src/connector/odbc/src/base/field.c diff --git a/src/connector/odbc/src/base/field.h b/2.0/src/connector/odbc/src/base/field.h similarity index 100% rename from src/connector/odbc/src/base/field.h rename to 2.0/src/connector/odbc/src/base/field.h diff --git a/src/connector/odbc/src/base/null_conn.c b/2.0/src/connector/odbc/src/base/null_conn.c similarity index 100% rename from src/connector/odbc/src/base/null_conn.c rename to 2.0/src/connector/odbc/src/base/null_conn.c diff --git a/src/connector/odbc/src/base/null_conn.h b/2.0/src/connector/odbc/src/base/null_conn.h similarity index 100% rename from src/connector/odbc/src/base/null_conn.h rename to 2.0/src/connector/odbc/src/base/null_conn.h diff --git a/src/connector/odbc/src/base/param.c b/2.0/src/connector/odbc/src/base/param.c similarity index 100% rename from src/connector/odbc/src/base/param.c rename to 2.0/src/connector/odbc/src/base/param.c diff --git a/src/connector/odbc/src/base/param.h b/2.0/src/connector/odbc/src/base/param.h similarity index 100% rename from src/connector/odbc/src/base/param.h rename to 2.0/src/connector/odbc/src/base/param.h diff --git a/src/connector/odbc/src/base/rs.c b/2.0/src/connector/odbc/src/base/rs.c similarity index 100% rename from src/connector/odbc/src/base/rs.c rename to 2.0/src/connector/odbc/src/base/rs.c diff --git a/src/connector/odbc/src/base/rs.h b/2.0/src/connector/odbc/src/base/rs.h similarity index 100% rename from src/connector/odbc/src/base/rs.h rename to 2.0/src/connector/odbc/src/base/rs.h diff --git a/src/connector/odbc/src/base/stmt.c b/2.0/src/connector/odbc/src/base/stmt.c similarity index 100% rename from src/connector/odbc/src/base/stmt.c rename to 2.0/src/connector/odbc/src/base/stmt.c diff --git a/src/connector/odbc/src/base/stmt.h b/2.0/src/connector/odbc/src/base/stmt.h similarity index 100% rename from src/connector/odbc/src/base/stmt.h rename to 2.0/src/connector/odbc/src/base/stmt.h diff --git a/src/connector/odbc/src/base/tsdb_impl.c b/2.0/src/connector/odbc/src/base/tsdb_impl.c similarity index 100% rename from src/connector/odbc/src/base/tsdb_impl.c rename to 2.0/src/connector/odbc/src/base/tsdb_impl.c diff --git a/src/connector/odbc/src/base/tsdb_impl.h b/2.0/src/connector/odbc/src/base/tsdb_impl.h similarity index 100% rename from src/connector/odbc/src/base/tsdb_impl.h rename to 2.0/src/connector/odbc/src/base/tsdb_impl.h diff --git a/src/connector/odbc/src/col.h b/2.0/src/connector/odbc/src/col.h similarity index 100% rename from src/connector/odbc/src/col.h rename to 2.0/src/connector/odbc/src/col.h diff --git a/src/connector/odbc/src/install.cmd b/2.0/src/connector/odbc/src/install.cmd similarity index 100% rename from src/connector/odbc/src/install.cmd rename to 2.0/src/connector/odbc/src/install.cmd diff --git a/src/connector/odbc/src/install.sh b/2.0/src/connector/odbc/src/install.sh similarity index 100% rename from src/connector/odbc/src/install.sh rename to 2.0/src/connector/odbc/src/install.sh diff --git a/src/connector/odbc/src/todbc.def b/2.0/src/connector/odbc/src/todbc.def similarity index 100% rename from src/connector/odbc/src/todbc.def rename to 2.0/src/connector/odbc/src/todbc.def diff --git a/src/connector/odbc/src/todbc.rc.in b/2.0/src/connector/odbc/src/todbc.rc.in similarity index 100% rename from src/connector/odbc/src/todbc.rc.in rename to 2.0/src/connector/odbc/src/todbc.rc.in diff --git a/src/connector/odbc/src/todbc.rsp b/2.0/src/connector/odbc/src/todbc.rsp similarity index 100% rename from src/connector/odbc/src/todbc.rsp rename to 2.0/src/connector/odbc/src/todbc.rsp diff --git a/src/connector/odbc/src/todbc_buf.c b/2.0/src/connector/odbc/src/todbc_buf.c similarity index 100% rename from src/connector/odbc/src/todbc_buf.c rename to 2.0/src/connector/odbc/src/todbc_buf.c diff --git a/src/connector/odbc/src/todbc_buf.h b/2.0/src/connector/odbc/src/todbc_buf.h similarity index 100% rename from src/connector/odbc/src/todbc_buf.h rename to 2.0/src/connector/odbc/src/todbc_buf.h diff --git a/src/connector/odbc/src/todbc_flex.h b/2.0/src/connector/odbc/src/todbc_flex.h similarity index 100% rename from src/connector/odbc/src/todbc_flex.h rename to 2.0/src/connector/odbc/src/todbc_flex.h diff --git a/src/connector/odbc/src/todbc_hash.c b/2.0/src/connector/odbc/src/todbc_hash.c similarity index 100% rename from src/connector/odbc/src/todbc_hash.c rename to 2.0/src/connector/odbc/src/todbc_hash.c diff --git a/src/connector/odbc/src/todbc_hash.h b/2.0/src/connector/odbc/src/todbc_hash.h similarity index 100% rename from src/connector/odbc/src/todbc_hash.h rename to 2.0/src/connector/odbc/src/todbc_hash.h diff --git a/src/connector/odbc/src/todbc_iconv.c b/2.0/src/connector/odbc/src/todbc_iconv.c similarity index 100% rename from src/connector/odbc/src/todbc_iconv.c rename to 2.0/src/connector/odbc/src/todbc_iconv.c diff --git a/src/connector/odbc/src/todbc_iconv.h b/2.0/src/connector/odbc/src/todbc_iconv.h similarity index 100% rename from src/connector/odbc/src/todbc_iconv.h rename to 2.0/src/connector/odbc/src/todbc_iconv.h diff --git a/src/connector/odbc/src/todbc_list.c b/2.0/src/connector/odbc/src/todbc_list.c similarity index 100% rename from src/connector/odbc/src/todbc_list.c rename to 2.0/src/connector/odbc/src/todbc_list.c diff --git a/src/connector/odbc/src/todbc_list.h b/2.0/src/connector/odbc/src/todbc_list.h similarity index 100% rename from src/connector/odbc/src/todbc_list.h rename to 2.0/src/connector/odbc/src/todbc_list.h diff --git a/src/connector/odbc/src/todbc_log.c b/2.0/src/connector/odbc/src/todbc_log.c similarity index 100% rename from src/connector/odbc/src/todbc_log.c rename to 2.0/src/connector/odbc/src/todbc_log.c diff --git a/src/connector/odbc/src/todbc_log.h b/2.0/src/connector/odbc/src/todbc_log.h similarity index 100% rename from src/connector/odbc/src/todbc_log.h rename to 2.0/src/connector/odbc/src/todbc_log.h diff --git a/src/connector/odbc/src/todbc_scanner.l b/2.0/src/connector/odbc/src/todbc_scanner.l similarity index 100% rename from src/connector/odbc/src/todbc_scanner.l rename to 2.0/src/connector/odbc/src/todbc_scanner.l diff --git a/src/connector/odbc/src/todbc_string.c b/2.0/src/connector/odbc/src/todbc_string.c similarity index 100% rename from src/connector/odbc/src/todbc_string.c rename to 2.0/src/connector/odbc/src/todbc_string.c diff --git a/src/connector/odbc/src/todbc_string.h b/2.0/src/connector/odbc/src/todbc_string.h similarity index 100% rename from src/connector/odbc/src/todbc_string.h rename to 2.0/src/connector/odbc/src/todbc_string.h diff --git a/src/connector/odbc/src/todbc_tls.c b/2.0/src/connector/odbc/src/todbc_tls.c similarity index 100% rename from src/connector/odbc/src/todbc_tls.c rename to 2.0/src/connector/odbc/src/todbc_tls.c diff --git a/src/connector/odbc/src/todbc_tls.h b/2.0/src/connector/odbc/src/todbc_tls.h similarity index 100% rename from src/connector/odbc/src/todbc_tls.h rename to 2.0/src/connector/odbc/src/todbc_tls.h diff --git a/src/connector/odbc/src/todbc_util.c b/2.0/src/connector/odbc/src/todbc_util.c similarity index 100% rename from src/connector/odbc/src/todbc_util.c rename to 2.0/src/connector/odbc/src/todbc_util.c diff --git a/src/connector/odbc/src/todbc_util.h b/2.0/src/connector/odbc/src/todbc_util.h similarity index 100% rename from src/connector/odbc/src/todbc_util.h rename to 2.0/src/connector/odbc/src/todbc_util.h diff --git a/src/connector/odbc/tools/CMakeLists.txt b/2.0/src/connector/odbc/tools/CMakeLists.txt similarity index 100% rename from src/connector/odbc/tools/CMakeLists.txt rename to 2.0/src/connector/odbc/tools/CMakeLists.txt diff --git a/src/connector/odbc/tools/main.c b/2.0/src/connector/odbc/tools/main.c similarity index 100% rename from src/connector/odbc/tools/main.c rename to 2.0/src/connector/odbc/tools/main.c diff --git a/src/connector/odbc/tools/tconv.c b/2.0/src/connector/odbc/tools/tconv.c similarity index 100% rename from src/connector/odbc/tools/tconv.c rename to 2.0/src/connector/odbc/tools/tconv.c From 3b7180749b9414e31fd007df4ff05183a1be7110 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:18:34 -0800 Subject: [PATCH 051/103] rename files --- {src => 2.0/src}/connector/nodejs/nodetaos/cinterface.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/connection.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/constants.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/cursor.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/error.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/globalfunc.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/taosobjects.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/taosquery.js | 0 {src => 2.0/src}/connector/nodejs/nodetaos/taosresult.js | 0 {src => 2.0/src}/connector/nodejs/package.json | 0 {src => 2.0/src}/connector/nodejs/readme.md | 0 {src => 2.0/src}/connector/nodejs/tdengine.js | 0 {src => 2.0/src}/connector/nodejs/test/performance.js | 0 {src => 2.0/src}/connector/nodejs/test/test.js | 0 {src => 2.0/src}/connector/nodejs/test/testMicroseconds.js | 0 {src => 2.0/src}/connector/nodejs/test/testNanoseconds.js | 0 {src => 2.0/src}/connector/nodejs/test/testSubscribe.js | 0 {src => 2.0/src}/connector/nodejs/test/testnchar.js | 0 {src => 2.0/src}/connector/python/.gitignore | 0 {src => 2.0/src}/connector/python/LICENSE | 0 {src => 2.0/src}/connector/python/README.md | 0 {src => 2.0/src}/connector/python/examples/bind-multi.py | 0 {src => 2.0/src}/connector/python/examples/bind-row.py | 0 {src => 2.0/src}/connector/python/examples/demo.py | 0 {src => 2.0/src}/connector/python/examples/insert-lines.py | 0 {src => 2.0/src}/connector/python/examples/pep-249.py | 0 {src => 2.0/src}/connector/python/examples/query-async.py | 0 {src => 2.0/src}/connector/python/examples/query-objectively.py | 0 {src => 2.0/src}/connector/python/examples/subscribe-async.py | 0 {src => 2.0/src}/connector/python/examples/subscribe-sync.py | 0 {src => 2.0/src}/connector/python/pyproject.toml | 0 {src => 2.0/src}/connector/python/setup.py | 0 {src => 2.0/src}/connector/python/taos/__init__.py | 0 {src => 2.0/src}/connector/python/taos/bind.py | 0 {src => 2.0/src}/connector/python/taos/cinterface.py | 0 {src => 2.0/src}/connector/python/taos/connection.py | 0 {src => 2.0/src}/connector/python/taos/constants.py | 0 {src => 2.0/src}/connector/python/taos/cursor.py | 0 {src => 2.0/src}/connector/python/taos/error.py | 0 {src => 2.0/src}/connector/python/taos/field.py | 0 {src => 2.0/src}/connector/python/taos/precision.py | 0 {src => 2.0/src}/connector/python/taos/result.py | 0 {src => 2.0/src}/connector/python/taos/statement.py | 0 {src => 2.0/src}/connector/python/taos/stream.py | 0 {src => 2.0/src}/connector/python/taos/subscription.py | 0 {src => 2.0/src}/connector/python/taos/timestamp.py | 0 {src => 2.0/src}/connector/python/tests/test-td6231.py | 0 {src => 2.0/src}/connector/python/tests/test_ctaos.py | 0 {src => 2.0/src}/connector/python/tests/test_info.py | 0 {src => 2.0/src}/connector/python/tests/test_lines.py | 0 {src => 2.0/src}/connector/python/tests/test_query.py | 0 {src => 2.0/src}/connector/python/tests/test_query_a.py | 0 {src => 2.0/src}/connector/python/tests/test_stmt.py | 0 {src => 2.0/src}/connector/python/tests/test_stream.py | 0 {src => 2.0/src}/connector/python/tests/test_subscribe.py | 0 55 files changed, 0 insertions(+), 0 deletions(-) rename {src => 2.0/src}/connector/nodejs/nodetaos/cinterface.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/connection.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/constants.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/cursor.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/error.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/globalfunc.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/taosobjects.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/taosquery.js (100%) rename {src => 2.0/src}/connector/nodejs/nodetaos/taosresult.js (100%) rename {src => 2.0/src}/connector/nodejs/package.json (100%) rename {src => 2.0/src}/connector/nodejs/readme.md (100%) rename {src => 2.0/src}/connector/nodejs/tdengine.js (100%) rename {src => 2.0/src}/connector/nodejs/test/performance.js (100%) rename {src => 2.0/src}/connector/nodejs/test/test.js (100%) rename {src => 2.0/src}/connector/nodejs/test/testMicroseconds.js (100%) rename {src => 2.0/src}/connector/nodejs/test/testNanoseconds.js (100%) rename {src => 2.0/src}/connector/nodejs/test/testSubscribe.js (100%) rename {src => 2.0/src}/connector/nodejs/test/testnchar.js (100%) rename {src => 2.0/src}/connector/python/.gitignore (100%) rename {src => 2.0/src}/connector/python/LICENSE (100%) rename {src => 2.0/src}/connector/python/README.md (100%) rename {src => 2.0/src}/connector/python/examples/bind-multi.py (100%) rename {src => 2.0/src}/connector/python/examples/bind-row.py (100%) rename {src => 2.0/src}/connector/python/examples/demo.py (100%) rename {src => 2.0/src}/connector/python/examples/insert-lines.py (100%) rename {src => 2.0/src}/connector/python/examples/pep-249.py (100%) rename {src => 2.0/src}/connector/python/examples/query-async.py (100%) rename {src => 2.0/src}/connector/python/examples/query-objectively.py (100%) rename {src => 2.0/src}/connector/python/examples/subscribe-async.py (100%) rename {src => 2.0/src}/connector/python/examples/subscribe-sync.py (100%) rename {src => 2.0/src}/connector/python/pyproject.toml (100%) rename {src => 2.0/src}/connector/python/setup.py (100%) rename {src => 2.0/src}/connector/python/taos/__init__.py (100%) rename {src => 2.0/src}/connector/python/taos/bind.py (100%) rename {src => 2.0/src}/connector/python/taos/cinterface.py (100%) rename {src => 2.0/src}/connector/python/taos/connection.py (100%) rename {src => 2.0/src}/connector/python/taos/constants.py (100%) rename {src => 2.0/src}/connector/python/taos/cursor.py (100%) rename {src => 2.0/src}/connector/python/taos/error.py (100%) rename {src => 2.0/src}/connector/python/taos/field.py (100%) rename {src => 2.0/src}/connector/python/taos/precision.py (100%) rename {src => 2.0/src}/connector/python/taos/result.py (100%) rename {src => 2.0/src}/connector/python/taos/statement.py (100%) rename {src => 2.0/src}/connector/python/taos/stream.py (100%) rename {src => 2.0/src}/connector/python/taos/subscription.py (100%) rename {src => 2.0/src}/connector/python/taos/timestamp.py (100%) rename {src => 2.0/src}/connector/python/tests/test-td6231.py (100%) rename {src => 2.0/src}/connector/python/tests/test_ctaos.py (100%) rename {src => 2.0/src}/connector/python/tests/test_info.py (100%) rename {src => 2.0/src}/connector/python/tests/test_lines.py (100%) rename {src => 2.0/src}/connector/python/tests/test_query.py (100%) rename {src => 2.0/src}/connector/python/tests/test_query_a.py (100%) rename {src => 2.0/src}/connector/python/tests/test_stmt.py (100%) rename {src => 2.0/src}/connector/python/tests/test_stream.py (100%) rename {src => 2.0/src}/connector/python/tests/test_subscribe.py (100%) diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/2.0/src/connector/nodejs/nodetaos/cinterface.js similarity index 100% rename from src/connector/nodejs/nodetaos/cinterface.js rename to 2.0/src/connector/nodejs/nodetaos/cinterface.js diff --git a/src/connector/nodejs/nodetaos/connection.js b/2.0/src/connector/nodejs/nodetaos/connection.js similarity index 100% rename from src/connector/nodejs/nodetaos/connection.js rename to 2.0/src/connector/nodejs/nodetaos/connection.js diff --git a/src/connector/nodejs/nodetaos/constants.js b/2.0/src/connector/nodejs/nodetaos/constants.js similarity index 100% rename from src/connector/nodejs/nodetaos/constants.js rename to 2.0/src/connector/nodejs/nodetaos/constants.js diff --git a/src/connector/nodejs/nodetaos/cursor.js b/2.0/src/connector/nodejs/nodetaos/cursor.js similarity index 100% rename from src/connector/nodejs/nodetaos/cursor.js rename to 2.0/src/connector/nodejs/nodetaos/cursor.js diff --git a/src/connector/nodejs/nodetaos/error.js b/2.0/src/connector/nodejs/nodetaos/error.js similarity index 100% rename from src/connector/nodejs/nodetaos/error.js rename to 2.0/src/connector/nodejs/nodetaos/error.js diff --git a/src/connector/nodejs/nodetaos/globalfunc.js b/2.0/src/connector/nodejs/nodetaos/globalfunc.js similarity index 100% rename from src/connector/nodejs/nodetaos/globalfunc.js rename to 2.0/src/connector/nodejs/nodetaos/globalfunc.js diff --git a/src/connector/nodejs/nodetaos/taosobjects.js b/2.0/src/connector/nodejs/nodetaos/taosobjects.js similarity index 100% rename from src/connector/nodejs/nodetaos/taosobjects.js rename to 2.0/src/connector/nodejs/nodetaos/taosobjects.js diff --git a/src/connector/nodejs/nodetaos/taosquery.js b/2.0/src/connector/nodejs/nodetaos/taosquery.js similarity index 100% rename from src/connector/nodejs/nodetaos/taosquery.js rename to 2.0/src/connector/nodejs/nodetaos/taosquery.js diff --git a/src/connector/nodejs/nodetaos/taosresult.js b/2.0/src/connector/nodejs/nodetaos/taosresult.js similarity index 100% rename from src/connector/nodejs/nodetaos/taosresult.js rename to 2.0/src/connector/nodejs/nodetaos/taosresult.js diff --git a/src/connector/nodejs/package.json b/2.0/src/connector/nodejs/package.json similarity index 100% rename from src/connector/nodejs/package.json rename to 2.0/src/connector/nodejs/package.json diff --git a/src/connector/nodejs/readme.md b/2.0/src/connector/nodejs/readme.md similarity index 100% rename from src/connector/nodejs/readme.md rename to 2.0/src/connector/nodejs/readme.md diff --git a/src/connector/nodejs/tdengine.js b/2.0/src/connector/nodejs/tdengine.js similarity index 100% rename from src/connector/nodejs/tdengine.js rename to 2.0/src/connector/nodejs/tdengine.js diff --git a/src/connector/nodejs/test/performance.js b/2.0/src/connector/nodejs/test/performance.js similarity index 100% rename from src/connector/nodejs/test/performance.js rename to 2.0/src/connector/nodejs/test/performance.js diff --git a/src/connector/nodejs/test/test.js b/2.0/src/connector/nodejs/test/test.js similarity index 100% rename from src/connector/nodejs/test/test.js rename to 2.0/src/connector/nodejs/test/test.js diff --git a/src/connector/nodejs/test/testMicroseconds.js b/2.0/src/connector/nodejs/test/testMicroseconds.js similarity index 100% rename from src/connector/nodejs/test/testMicroseconds.js rename to 2.0/src/connector/nodejs/test/testMicroseconds.js diff --git a/src/connector/nodejs/test/testNanoseconds.js b/2.0/src/connector/nodejs/test/testNanoseconds.js similarity index 100% rename from src/connector/nodejs/test/testNanoseconds.js rename to 2.0/src/connector/nodejs/test/testNanoseconds.js diff --git a/src/connector/nodejs/test/testSubscribe.js b/2.0/src/connector/nodejs/test/testSubscribe.js similarity index 100% rename from src/connector/nodejs/test/testSubscribe.js rename to 2.0/src/connector/nodejs/test/testSubscribe.js diff --git a/src/connector/nodejs/test/testnchar.js b/2.0/src/connector/nodejs/test/testnchar.js similarity index 100% rename from src/connector/nodejs/test/testnchar.js rename to 2.0/src/connector/nodejs/test/testnchar.js diff --git a/src/connector/python/.gitignore b/2.0/src/connector/python/.gitignore similarity index 100% rename from src/connector/python/.gitignore rename to 2.0/src/connector/python/.gitignore diff --git a/src/connector/python/LICENSE b/2.0/src/connector/python/LICENSE similarity index 100% rename from src/connector/python/LICENSE rename to 2.0/src/connector/python/LICENSE diff --git a/src/connector/python/README.md b/2.0/src/connector/python/README.md similarity index 100% rename from src/connector/python/README.md rename to 2.0/src/connector/python/README.md diff --git a/src/connector/python/examples/bind-multi.py b/2.0/src/connector/python/examples/bind-multi.py similarity index 100% rename from src/connector/python/examples/bind-multi.py rename to 2.0/src/connector/python/examples/bind-multi.py diff --git a/src/connector/python/examples/bind-row.py b/2.0/src/connector/python/examples/bind-row.py similarity index 100% rename from src/connector/python/examples/bind-row.py rename to 2.0/src/connector/python/examples/bind-row.py diff --git a/src/connector/python/examples/demo.py b/2.0/src/connector/python/examples/demo.py similarity index 100% rename from src/connector/python/examples/demo.py rename to 2.0/src/connector/python/examples/demo.py diff --git a/src/connector/python/examples/insert-lines.py b/2.0/src/connector/python/examples/insert-lines.py similarity index 100% rename from src/connector/python/examples/insert-lines.py rename to 2.0/src/connector/python/examples/insert-lines.py diff --git a/src/connector/python/examples/pep-249.py b/2.0/src/connector/python/examples/pep-249.py similarity index 100% rename from src/connector/python/examples/pep-249.py rename to 2.0/src/connector/python/examples/pep-249.py diff --git a/src/connector/python/examples/query-async.py b/2.0/src/connector/python/examples/query-async.py similarity index 100% rename from src/connector/python/examples/query-async.py rename to 2.0/src/connector/python/examples/query-async.py diff --git a/src/connector/python/examples/query-objectively.py b/2.0/src/connector/python/examples/query-objectively.py similarity index 100% rename from src/connector/python/examples/query-objectively.py rename to 2.0/src/connector/python/examples/query-objectively.py diff --git a/src/connector/python/examples/subscribe-async.py b/2.0/src/connector/python/examples/subscribe-async.py similarity index 100% rename from src/connector/python/examples/subscribe-async.py rename to 2.0/src/connector/python/examples/subscribe-async.py diff --git a/src/connector/python/examples/subscribe-sync.py b/2.0/src/connector/python/examples/subscribe-sync.py similarity index 100% rename from src/connector/python/examples/subscribe-sync.py rename to 2.0/src/connector/python/examples/subscribe-sync.py diff --git a/src/connector/python/pyproject.toml b/2.0/src/connector/python/pyproject.toml similarity index 100% rename from src/connector/python/pyproject.toml rename to 2.0/src/connector/python/pyproject.toml diff --git a/src/connector/python/setup.py b/2.0/src/connector/python/setup.py similarity index 100% rename from src/connector/python/setup.py rename to 2.0/src/connector/python/setup.py diff --git a/src/connector/python/taos/__init__.py b/2.0/src/connector/python/taos/__init__.py similarity index 100% rename from src/connector/python/taos/__init__.py rename to 2.0/src/connector/python/taos/__init__.py diff --git a/src/connector/python/taos/bind.py b/2.0/src/connector/python/taos/bind.py similarity index 100% rename from src/connector/python/taos/bind.py rename to 2.0/src/connector/python/taos/bind.py diff --git a/src/connector/python/taos/cinterface.py b/2.0/src/connector/python/taos/cinterface.py similarity index 100% rename from src/connector/python/taos/cinterface.py rename to 2.0/src/connector/python/taos/cinterface.py diff --git a/src/connector/python/taos/connection.py b/2.0/src/connector/python/taos/connection.py similarity index 100% rename from src/connector/python/taos/connection.py rename to 2.0/src/connector/python/taos/connection.py diff --git a/src/connector/python/taos/constants.py b/2.0/src/connector/python/taos/constants.py similarity index 100% rename from src/connector/python/taos/constants.py rename to 2.0/src/connector/python/taos/constants.py diff --git a/src/connector/python/taos/cursor.py b/2.0/src/connector/python/taos/cursor.py similarity index 100% rename from src/connector/python/taos/cursor.py rename to 2.0/src/connector/python/taos/cursor.py diff --git a/src/connector/python/taos/error.py b/2.0/src/connector/python/taos/error.py similarity index 100% rename from src/connector/python/taos/error.py rename to 2.0/src/connector/python/taos/error.py diff --git a/src/connector/python/taos/field.py b/2.0/src/connector/python/taos/field.py similarity index 100% rename from src/connector/python/taos/field.py rename to 2.0/src/connector/python/taos/field.py diff --git a/src/connector/python/taos/precision.py b/2.0/src/connector/python/taos/precision.py similarity index 100% rename from src/connector/python/taos/precision.py rename to 2.0/src/connector/python/taos/precision.py diff --git a/src/connector/python/taos/result.py b/2.0/src/connector/python/taos/result.py similarity index 100% rename from src/connector/python/taos/result.py rename to 2.0/src/connector/python/taos/result.py diff --git a/src/connector/python/taos/statement.py b/2.0/src/connector/python/taos/statement.py similarity index 100% rename from src/connector/python/taos/statement.py rename to 2.0/src/connector/python/taos/statement.py diff --git a/src/connector/python/taos/stream.py b/2.0/src/connector/python/taos/stream.py similarity index 100% rename from src/connector/python/taos/stream.py rename to 2.0/src/connector/python/taos/stream.py diff --git a/src/connector/python/taos/subscription.py b/2.0/src/connector/python/taos/subscription.py similarity index 100% rename from src/connector/python/taos/subscription.py rename to 2.0/src/connector/python/taos/subscription.py diff --git a/src/connector/python/taos/timestamp.py b/2.0/src/connector/python/taos/timestamp.py similarity index 100% rename from src/connector/python/taos/timestamp.py rename to 2.0/src/connector/python/taos/timestamp.py diff --git a/src/connector/python/tests/test-td6231.py b/2.0/src/connector/python/tests/test-td6231.py similarity index 100% rename from src/connector/python/tests/test-td6231.py rename to 2.0/src/connector/python/tests/test-td6231.py diff --git a/src/connector/python/tests/test_ctaos.py b/2.0/src/connector/python/tests/test_ctaos.py similarity index 100% rename from src/connector/python/tests/test_ctaos.py rename to 2.0/src/connector/python/tests/test_ctaos.py diff --git a/src/connector/python/tests/test_info.py b/2.0/src/connector/python/tests/test_info.py similarity index 100% rename from src/connector/python/tests/test_info.py rename to 2.0/src/connector/python/tests/test_info.py diff --git a/src/connector/python/tests/test_lines.py b/2.0/src/connector/python/tests/test_lines.py similarity index 100% rename from src/connector/python/tests/test_lines.py rename to 2.0/src/connector/python/tests/test_lines.py diff --git a/src/connector/python/tests/test_query.py b/2.0/src/connector/python/tests/test_query.py similarity index 100% rename from src/connector/python/tests/test_query.py rename to 2.0/src/connector/python/tests/test_query.py diff --git a/src/connector/python/tests/test_query_a.py b/2.0/src/connector/python/tests/test_query_a.py similarity index 100% rename from src/connector/python/tests/test_query_a.py rename to 2.0/src/connector/python/tests/test_query_a.py diff --git a/src/connector/python/tests/test_stmt.py b/2.0/src/connector/python/tests/test_stmt.py similarity index 100% rename from src/connector/python/tests/test_stmt.py rename to 2.0/src/connector/python/tests/test_stmt.py diff --git a/src/connector/python/tests/test_stream.py b/2.0/src/connector/python/tests/test_stream.py similarity index 100% rename from src/connector/python/tests/test_stream.py rename to 2.0/src/connector/python/tests/test_stream.py diff --git a/src/connector/python/tests/test_subscribe.py b/2.0/src/connector/python/tests/test_subscribe.py similarity index 100% rename from src/connector/python/tests/test_subscribe.py rename to 2.0/src/connector/python/tests/test_subscribe.py From a50c3f3d5e65996b2a6148b77389a828d2793968 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 19 Jan 2022 18:19:48 +0800 Subject: [PATCH 052/103] add client --- source/libs/transport/inc/transComm.h | 19 ++++ source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 145 +++++++++++++++++++------- source/libs/transport/src/transSrv.c | 30 +++--- source/libs/transport/test/rclient.c | 6 +- 5 files changed, 143 insertions(+), 58 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 792200639a..4b14f9f2c7 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -68,6 +68,25 @@ typedef void* queue[2]; QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \ QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \ } +#define QUEUE_SPLIT(h, q, n) \ + do { \ + QUEUE_PREV(n) = QUEUE_PREV(h); \ + QUEUE_PREV_NEXT(n) = (n); \ + QUEUE_NEXT(n) = (q); \ + QUEUE_PREV(h) = QUEUE_PREV(q); \ + QUEUE_PREV_NEXT(h) = (h); \ + QUEUE_PREV(q) = (n); \ + } while (0) + +#define QUEUE_MOVE(h, n) \ + do { \ + if (QUEUE_IS_EMPTY(h)) { \ + QUEUE_INIT(n); \ + } else { \ + queue* q = QUEUE_HEAD(h); \ + QUEUE_SPLIT(h, q, n); \ + } \ + } while (0) /* Return the element at the front of the queue. */ #define QUEUE_HEAD(q) (QUEUE_NEXT(q)) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 89361b13ad..cb8ef87b48 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -35,6 +35,7 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->label) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } + pRpc->cfp = pInit->cfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->connType = pInit->connType; pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 29f3361b10..7e5874d8cd 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -20,12 +20,14 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; + uv_write_t* writeReq; void* data; queue conn; } SCliConn; typedef struct SCliMsg { SRpcReqContext* context; queue q; + uint64_t st; } SCliMsg; typedef struct SCliThrdObj { @@ -45,86 +47,153 @@ typedef struct SClientObj { SCliThrdObj** pThreadObj; } SClientObj; -static void clientWriteCb(uv_write_t* req, int status); +static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -static void clientConnCb(struct uv_connect_s* req, int status); +static void clientWriteCb(uv_write_t* req, int status); +static void clientConnCb(uv_connect_t* req, int status); static void clientAsyncCb(uv_async_t* handle); +static void clientDestroy(uv_handle_t* handle); +static void clientConnDestroy(SCliConn* pConn); static void* clientThread(void* arg); +static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); + +static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + // impl later +} +static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { + // impl later + SCliConn* conn = handle->data; + if (nread > 0) { + return; + } + // + uv_close((uv_handle_t*)handle, clientDestroy); +} + +static void clientConnDestroy(SCliConn* conn) { + // impl later + // +} +static void clientDestroy(uv_handle_t* handle) { + SCliConn* conn = handle->data; + clientConnDestroy(conn); +} + static void clientWriteCb(uv_write_t* req, int status) { - // impl later -} -static void clientFailedCb(uv_handle_t* handle) { - // impl later - tDebug("close handle"); -} -static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { - // impl later -} -static void clientConnCb(struct uv_connect_s* req, int status) { SCliConn* pConn = req->data; - SCliMsg* pMsg = pConn->data; - SEpSet* pEpSet = &pMsg->context->epSet; + if (status == 0) { + tDebug("data already was written on stream"); + } else { + uv_close((uv_handle_t*)pConn->stream, clientDestroy); + return; + } + + uv_read_start((uv_stream_t*)pConn->stream, clientAllocrReadBufferCb, clientReadCb); + // impl later +} +static void clientConnCb(uv_connect_t* req, int status) { + // impl later + SCliConn* pConn = req->data; + if (status != 0) { + tError("failed to connect %s", uv_err_name(status)); + clientConnDestroy(pConn); + return; + } + + SCliMsg* pMsg = pConn->data; + SEpSet* pEpSet = &pMsg->context->epSet; + SRpcMsg rpcMsg; + rpcMsg.ahandle = pMsg->context->ahandle; + rpcMsg.pCont = NULL; char* fqdn = pEpSet->fqdn[pEpSet->inUse]; uint32_t port = pEpSet->port[pEpSet->inUse]; if (status != 0) { // call user fp later tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status)); - uv_close((uv_handle_t*)req->handle, clientFailedCb); + SRpcInfo* pRpc = pMsg->context->pRpc; + (pRpc->cfp)(NULL, &rpcMsg, pEpSet); + uv_close((uv_handle_t*)req->handle, clientDestroy); return; } assert(pConn->stream == req->handle); - // impl later + uv_buf_t wb; + uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { // impl later return NULL; } -static void clientAsyncCb(uv_async_t* handle) { - SCliThrdObj* pThrd = handle->data; - SCliMsg* pMsg = NULL; - pthread_mutex_lock(&pThrd->msgMtx); - if (!QUEUE_IS_EMPTY(&pThrd->msg)) { - queue* head = QUEUE_HEAD(&pThrd->msg); - pMsg = QUEUE_DATA(head, SCliMsg, q); - QUEUE_REMOVE(head); - } - pthread_mutex_unlock(&pThrd->msgMtx); - SEpSet* pEpSet = &pMsg->context->epSet; +static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { + SEpSet* pEpSet = &pMsg->context->epSet; + char* fqdn = pEpSet->fqdn[pEpSet->inUse]; uint32_t port = pEpSet->port[pEpSet->inUse]; + uint64_t el = taosGetTimestampUs() - pMsg->st; + tDebug("msg tran time cost: %" PRIu64 "", el); + SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); if (conn != NULL) { + conn->data = pMsg; + conn->writeReq->data = conn; + uv_buf_t wb; + uv_write(conn->writeReq, (uv_stream_t*)conn->stream, &wb, 1, clientWriteCb); // impl later } else { SCliConn* conn = malloc(sizeof(SCliConn)); conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); + conn->writeReq = malloc(sizeof(uv_write_t)); conn->connReq.data = conn; conn->data = pMsg; - struct sockaddr_in addr; uv_ip4_addr(fqdn, port, &addr); - // handle error in callback if connect error + // handle error in callback if fail to connect uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); + + // SRpcMsg rpcMsg; + // SEpSet* pEpSet = &pMsg->context->epSet; + // SRpcInfo* pRpc = pMsg->context->pRpc; + //// rpcMsg.ahandle = pMsg->context->ahandle; + // rpcMsg.pCont = NULL; + // rpcMsg.ahandle = pMsg->context->ahandle; + // uint64_t el1 = taosGetTimestampUs() - et; + // tError("msg tran back first: time cost: %" PRIu64 "", el1); + // et = taosGetTimestampUs(); + //(pRpc->cfp)(NULL, &rpcMsg, pEpSet); + // uint64_t el2 = taosGetTimestampUs() - et; + // tError("msg tran back second: time cost: %" PRIu64 "", el2); } +} +static void clientAsyncCb(uv_async_t* handle) { + SCliThrdObj* pThrd = handle->data; + SCliMsg* pMsg = NULL; + queue wq; - // SRpcReqContext* pCxt = pMsg->context; + // batch process to avoid to lock/unlock frequently + pthread_mutex_lock(&pThrd->msgMtx); + QUEUE_MOVE(&pThrd->msg, &wq); + pthread_mutex_unlock(&pThrd->msgMtx); - // SRpcHead* pHead = rpcHeadFromCont(pCtx->pCont); - // char* msg = (char*)pHead; - // int len = rpcMsgLenFromCont(pCtx->contLen); - // tmsg_t msgType = pCtx->msgType; - - // impl later + int count = 0; + while (!QUEUE_IS_EMPTY(&wq)) { + queue* h = QUEUE_HEAD(&wq); + QUEUE_REMOVE(h); + pMsg = QUEUE_DATA(h, SCliMsg, q); + clientHandleReq(pMsg, pThrd); + count++; + if (count >= 2) { + tError("send batch size: %d", count); + } + } } static void* clientThread(void* arg) { @@ -142,9 +211,6 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - - // QUEUE_INIT(&pThrd->clientCache); - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); @@ -186,6 +252,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* } SCliMsg* msg = malloc(sizeof(SCliMsg)); msg->context = pContext; + msg->st = taosGetTimestampUs(); SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 0bf39b9985..bc4cc695b0 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -277,10 +277,6 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } - if (terrno != 0) { - // handle err code - } - if (nread != UV_EOF) { tDebug("Read error %s\n", uv_err_name(nread)); } @@ -309,21 +305,23 @@ void uvOnWriteCb(uv_write_t* req, int status) { void uvWorkerAsyncCb(uv_async_t* handle) { SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync); SConn* conn = NULL; - - // opt later + queue wq; + // batch process to avoid to lock/unlock frequently pthread_mutex_lock(&pThrd->connMtx); - if (!QUEUE_IS_EMPTY(&pThrd->conn)) { - queue* head = QUEUE_HEAD(&pThrd->conn); - conn = QUEUE_DATA(head, SConn, queue); - QUEUE_REMOVE(head); - } + QUEUE_MOVE(&pThrd->conn, &wq); pthread_mutex_unlock(&pThrd->connMtx); - if (conn == NULL) { - tError("except occurred, do nothing"); - return; + + while (!QUEUE_IS_EMPTY(&wq)) { + queue* head = QUEUE_HEAD(&wq); + QUEUE_REMOVE(head); + SConn* conn = QUEUE_DATA(head, SConn, queue); + if (conn == NULL) { + tError("except occurred, do nothing"); + return; + } + uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); + uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } - uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); - uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } void uvOnAcceptCb(uv_stream_t* stream, int status) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 58fbf6ae85..6339e58560 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -34,8 +34,8 @@ typedef struct { static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); + // tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + // pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -57,7 +57,7 @@ static void *sendRequest(void *param) { rpcMsg.contLen = pInfo->msgSize; rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; - tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); tsem_wait(&pInfo->rspSem); From a637882baed91cec4d2664c92d2b6171da799607 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:19:54 -0800 Subject: [PATCH 053/103] rename files --- {src => 2.0/src}/connector/jdbc/CMakeLists.txt | 0 {src => 2.0/src}/connector/jdbc/deploy-pom.xml | 0 {src => 2.0/src}/connector/jdbc/pom.xml | 0 {src => 2.0/src}/connector/jdbc/readme.md | 0 .../src}/connector/jdbc/src/main/assembly/assembly-jar.xml | 0 {src => 2.0/src}/connector/jdbc/src/main/assembly/assembly.xml | 0 .../jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java | 0 .../src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java | 0 .../main/java/com/taosdata/jdbc/AbstractParameterMetaData.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java | 0 .../main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java | 0 .../connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java | 0 .../src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java | 0 .../src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java | 0 .../src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java | 0 .../src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java | 0 .../main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java | 0 .../main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java | 0 .../main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java | 0 .../main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java | 0 .../src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java | 0 .../src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java | 0 .../jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java | 0 .../jdbc/src/main/resources/META-INF/services/java.sql.Driver | 0 .../jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java | 0 .../src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 0 .../src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java | 0 .../test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java | 0 .../test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java | 0 .../src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java | 0 .../src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java | 0 .../test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java | 0 .../test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java | 0 .../src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java | 0 .../cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java | 0 .../java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java | 0 .../test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java | 0 .../test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java | 0 .../src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java | 0 .../java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java | 0 .../com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java | 0 .../taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java | 0 .../java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java | 0 .../java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java | 0 .../java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java | 0 .../com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java | 0 .../taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java | 0 .../taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java | 0 .../java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java | 0 .../com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java | 0 .../java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java | 0 .../com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java | 0 .../java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java | 0 .../taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java | 0 .../jdbc/cases/PreparedStatementBatchInsertRestfulTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java | 0 .../test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java | 0 .../jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java | 0 .../test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java | 0 .../taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java | 0 .../com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java | 0 .../test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java | 0 .../java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java | 0 .../java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java | 0 .../src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java | 0 .../src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java | 0 .../java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java | 0 .../src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java | 0 .../java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java | 0 .../java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java | 0 .../java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java | 0 .../src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java | 0 .../src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java | 0 .../jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java | 0 114 files changed, 0 insertions(+), 0 deletions(-) rename {src => 2.0/src}/connector/jdbc/CMakeLists.txt (100%) rename {src => 2.0/src}/connector/jdbc/deploy-pom.xml (100%) rename {src => 2.0/src}/connector/jdbc/pom.xml (100%) rename {src => 2.0/src}/connector/jdbc/readme.md (100%) rename {src => 2.0/src}/connector/jdbc/src/main/assembly/assembly-jar.xml (100%) rename {src => 2.0/src}/connector/jdbc/src/main/assembly/assembly.xml (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java (100%) rename {src => 2.0/src}/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java (100%) rename {src => 2.0/src}/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java (100%) diff --git a/src/connector/jdbc/CMakeLists.txt b/2.0/src/connector/jdbc/CMakeLists.txt similarity index 100% rename from src/connector/jdbc/CMakeLists.txt rename to 2.0/src/connector/jdbc/CMakeLists.txt diff --git a/src/connector/jdbc/deploy-pom.xml b/2.0/src/connector/jdbc/deploy-pom.xml similarity index 100% rename from src/connector/jdbc/deploy-pom.xml rename to 2.0/src/connector/jdbc/deploy-pom.xml diff --git a/src/connector/jdbc/pom.xml b/2.0/src/connector/jdbc/pom.xml similarity index 100% rename from src/connector/jdbc/pom.xml rename to 2.0/src/connector/jdbc/pom.xml diff --git a/src/connector/jdbc/readme.md b/2.0/src/connector/jdbc/readme.md similarity index 100% rename from src/connector/jdbc/readme.md rename to 2.0/src/connector/jdbc/readme.md diff --git a/src/connector/jdbc/src/main/assembly/assembly-jar.xml b/2.0/src/connector/jdbc/src/main/assembly/assembly-jar.xml similarity index 100% rename from src/connector/jdbc/src/main/assembly/assembly-jar.xml rename to 2.0/src/connector/jdbc/src/main/assembly/assembly-jar.xml diff --git a/src/connector/jdbc/src/main/assembly/assembly.xml b/2.0/src/connector/jdbc/src/main/assembly/assembly.xml similarity index 100% rename from src/connector/jdbc/src/main/assembly/assembly.xml rename to 2.0/src/connector/jdbc/src/main/assembly/assembly.xml diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/EmptyResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TaosGlobalConfig.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampFormat.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/enums/TimestampPrecision.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/NullType.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java similarity index 100% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java rename to 2.0/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java diff --git a/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver b/2.0/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver similarity index 100% rename from src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver rename to 2.0/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DoubleQuoteInSqlTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/JDBCTypeAndTypeCompareTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MicroSecondPrecisionRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NanoSecondTimestampRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertJNITest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/PreparedStatementBatchInsertRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResultSetMetaShouldNotBeNullRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimeZoneTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisonInNanoRestTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UseNowInsertTimestampTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java b/2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java similarity index 100% rename from src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java rename to 2.0/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java From 0399125bb829f2c5576fe064dc2a521104beef63 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 19 Jan 2022 18:22:25 +0800 Subject: [PATCH 054/103] [td-11818] Refactor. --- source/client/test/clientTests.cpp | 100 ++++++++++----------- source/libs/planner/src/physicalPlanJson.c | 43 ++++----- source/libs/planner/src/planner.c | 6 +- 3 files changed, 70 insertions(+), 79 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 59b133338e..4361131f01 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -521,29 +521,29 @@ TEST(testCase, connect_Test) { // taosHashCleanup(phash); //} // -//TEST(testCase, create_topic_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == nullptr); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// char* sql = "select * from tu"; -// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_topic_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from tu"; + pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -614,30 +614,30 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} -TEST(testCase, projection_query_stables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "select ts from m1"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, projection_query_stables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select ts from m1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index bf38712b13..db1b1c84d1 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -29,6 +29,14 @@ static void copyString(const cJSON* json, const char* name, char* dst) { strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); } +static uint64_t getBigintFromString(const cJSON* json, const char* name) { + char* val = getString(json, name); + uint64_t intVal = strtoul(val, NULL, 10); + tfree(val); + + return intVal; +} + static int64_t getNumber(const cJSON* json, const char* name) { double d = cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); return (int64_t) d; @@ -543,13 +551,13 @@ static const char* jkTimeWindowEndKey = "EndKey"; static bool timeWindowToJson(const void* obj, cJSON* json) { const STimeWindow* win = (const STimeWindow*)obj; - char tmp[32] = {0}; - sprintf(tmp, "%"PRId64, win->skey); + char tmp[40] = {0}; + snprintf(tmp, tListLen(tmp),"%"PRId64, win->skey); bool res = cJSON_AddStringToObject(json, jkTimeWindowStartKey, tmp); if (res) { memset(tmp, 0, tListLen(tmp)); - sprintf(tmp, "%"PRId64, win->ekey); + snprintf(tmp, tListLen(tmp),"%"PRId64, win->ekey); res = cJSON_AddStringToObject(json, jkTimeWindowEndKey, tmp); } return res; @@ -557,16 +565,8 @@ static bool timeWindowToJson(const void* obj, cJSON* json) { static bool timeWindowFromJson(const cJSON* json, void* obj) { STimeWindow* win = (STimeWindow*)obj; - - char* pStartKey = getString(json, jkTimeWindowStartKey); - win->skey = strtoul(pStartKey, NULL, 10); - - char* pEndKey = getString(json, jkTimeWindowEndKey); - win->ekey = strtoul(pEndKey, NULL, 10); - - tfree(pStartKey); - tfree(pEndKey); - + win->skey = getBigintFromString(json, jkTimeWindowStartKey); + win->ekey = getBigintFromString(json, jkTimeWindowEndKey); return true; } @@ -605,10 +605,7 @@ static bool scanNodeToJson(const void* obj, cJSON* json) { static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* pNode = (SScanPhyNode*)obj; - char* val = getString(json, jkScanNodeTableId); - pNode->uid = strtoull(val, NULL, 10); - tfree(val); - + pNode->uid = getBigintFromString(json, jkScanNodeTableId); pNode->tableType = getNumber(json, jkScanNodeTableType); pNode->count = getNumber(json, jkScanNodeTableCount); pNode->order = getNumber(json, jkScanNodeTableOrder); @@ -782,10 +779,7 @@ static bool nodeAddrFromJson(const cJSON* json, void* obj) { SDownstreamSource* pSource = (SDownstreamSource*)obj; pSource->taskId = getNumber(json, jkNodeTaskId); - char* pSchedId = getString(json, jkNodeTaskSchedId); - pSource->schedId = strtoul(pSchedId, NULL, 10); - tfree(pSchedId); - + pSource->schedId = getBigintFromString(json, jkNodeTaskSchedId); bool res = fromObject(json, jkNodeAddr, queryNodeAddrFromJson, &pSource->addr, true); return res; } @@ -1031,12 +1025,9 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) { static bool subplanIdFromJson(const cJSON* json, void* obj) { SSubplanId* id = (SSubplanId*)obj; - char* queryId = getString(json, jkIdQueryId); - id->queryId = strtoul(queryId, NULL, 0); - tfree(queryId); - + id->queryId = getBigintFromString(json, jkIdQueryId); id->templateId = getNumber(json, jkIdTemplateId); - id->subplanId = getNumber(json, jkIdSubplanId); + id->subplanId = getNumber(json, jkIdSubplanId); return true; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 6b3f37741e..9b32213ad7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From c20af74a69d496de213f25646cbe5059eaea8fe5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 02:45:19 -0800 Subject: [PATCH 055/103] add tests for tfs --- source/libs/tfs/CMakeLists.txt | 6 +++++- source/libs/tfs/test/CMakeLists.txt | 14 ++++++++++++++ source/libs/tfs/test/tfsTest.cpp | 27 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 source/libs/tfs/test/CMakeLists.txt create mode 100644 source/libs/tfs/test/tfsTest.cpp diff --git a/source/libs/tfs/CMakeLists.txt b/source/libs/tfs/CMakeLists.txt index 1b6f662507..607ccd4c48 100644 --- a/source/libs/tfs/CMakeLists.txt +++ b/source/libs/tfs/CMakeLists.txt @@ -6,4 +6,8 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(tfs os util common) \ No newline at end of file +target_link_libraries(tfs os util common) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt new file mode 100644 index 0000000000..ee28dcf723 --- /dev/null +++ b/source/libs/tfs/test/CMakeLists.txt @@ -0,0 +1,14 @@ +enable_testing() + +aux_source_directory(. TFS_TEST_SRC) +add_executable(tfs_test ${TFS_TEST_SRC}) +target_link_libraries( + tfs_test + PUBLIC tfs + PUBLIC gtest_main +) + +add_test( + NAME tfs_test + COMMAND tfs_test +) diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp new file mode 100644 index 0000000000..a85bea192c --- /dev/null +++ b/source/libs/tfs/test/tfsTest.cpp @@ -0,0 +1,27 @@ +/** + * @file tfsTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief TFS module tests + * @version 1.0 + * @date 2022-01-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "os.h" + +class TfsTest : public ::testing::Test { + protected: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(TfsTest, 01_Open_Close) { + +} From a5253f2517a8b50802a0b76fbba24f0c1bb427d1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 19 Jan 2022 19:14:25 +0800 Subject: [PATCH 056/103] add client --- source/libs/transport/src/transCli.c | 34 +++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7e5874d8cd..f197e72ec5 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -23,6 +23,8 @@ typedef struct SCliConn { uv_write_t* writeReq; void* data; queue conn; + char spi; + char secured; } SCliConn; typedef struct SCliMsg { SRpcReqContext* context; @@ -47,6 +49,10 @@ typedef struct SClientObj { SCliThrdObj** pThreadObj; } SClientObj; +// conn pool +static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); +static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); + static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void clientWriteCb(uv_write_t* req, int status); @@ -93,6 +99,16 @@ static void clientWriteCb(uv_write_t* req, int status) { uv_read_start((uv_stream_t*)pConn->stream, clientAllocrReadBufferCb, clientReadCb); // impl later } + +static void clientWrite(SCliConn* pConn) { + SCliMsg* pMsg = pConn->data; + SRpcHead* pHead = rpcHeadFromCont(pMsg->context->pCont); + int msgLen = rpcMsgLenFromCont(pMsg->context->contLen); + char* msg = (char*)(pHead); + + uv_buf_t wb = uv_buf_init(msg, msgLen); + uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); +} static void clientConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; @@ -105,8 +121,8 @@ static void clientConnCb(uv_connect_t* req, int status) { SCliMsg* pMsg = pConn->data; SEpSet* pEpSet = &pMsg->context->epSet; SRpcMsg rpcMsg; - rpcMsg.ahandle = pMsg->context->ahandle; - rpcMsg.pCont = NULL; + // rpcMsg.ahandle = pMsg->context->ahandle; + // rpcMsg.pCont = NULL; char* fqdn = pEpSet->fqdn[pEpSet->inUse]; uint32_t port = pEpSet->port[pEpSet->inUse]; @@ -119,15 +135,16 @@ static void clientConnCb(uv_connect_t* req, int status) { return; } assert(pConn->stream == req->handle); - - uv_buf_t wb; - uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { // impl later + return NULL; } +static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { + // impl later +} static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SEpSet* pEpSet = &pMsg->context->epSet; @@ -140,11 +157,12 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); if (conn != NULL) { + // impl later conn->data = pMsg; conn->writeReq->data = conn; - uv_buf_t wb; - uv_write(conn->writeReq, (uv_stream_t*)conn->stream, &wb, 1, clientWriteCb); - // impl later + clientWrite(conn); + // uv_buf_t wb; + // uv_write(conn->writeReq, (uv_stream_t*)conn->stream, &wb, 1, clientWriteCb); } else { SCliConn* conn = malloc(sizeof(SCliConn)); From b14ce58ba5672f322ac9d085268b46588a6d7d5c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 19 Jan 2022 19:42:44 +0800 Subject: [PATCH 057/103] feature/qnode --- include/common/tmsgdef.h | 1 - include/libs/executor/dataSinkMgt.h | 1 - source/dnode/vnode/src/vnd/vnodeQuery.c | 2 - source/libs/executor/src/dataDispatcher.c | 1 - source/libs/qworker/inc/qworkerInt.h | 17 +- source/libs/qworker/src/qworker.c | 305 ++++++++++++++-------- source/libs/qworker/src/qworkerMsg.c | 55 ---- 7 files changed, 210 insertions(+), 172 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index bfeba885d0..f16dbd1f22 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -171,7 +171,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 371cb12405..19438b5dd4 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -48,7 +48,6 @@ typedef struct SOutputData { int8_t compressed; char* pData; bool queryEnd; - int32_t scheduleJobNo; int32_t bufStatus; int64_t useconds; int8_t precision; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2e9c77c59b..0d7e322b70 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -29,8 +29,6 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); case TDMT_VND_QUERY_CONTINUE: return qWorkerProcessCQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); - case TDMT_VND_SCHEDULE_DATA_SINK: - return qWorkerProcessDataSinkMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index edba4fc97d..f9e61f91de 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -196,7 +196,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->bufStatus = updateStatus(pDispatcher); pthread_mutex_lock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; - pOutput->scheduleJobNo = 0; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->schema.precision; pthread_mutex_unlock(&pDispatcher->mutex); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 913057787e..193287eeac 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -31,8 +31,6 @@ enum { QW_PHASE_POST_QUERY, QW_PHASE_PRE_CQUERY, QW_PHASE_POST_CQUERY, - QW_PHASE_PRE_SINK, - QW_PHASE_POST_SINK, QW_PHASE_PRE_FETCH, QW_PHASE_POST_FETCH, }; @@ -105,10 +103,11 @@ typedef struct SQWTaskStatus { typedef struct SQWTaskCtx { SRWLatch lock; - int32_t phase; - - int32_t sinkId; - int32_t readyCode; + int8_t phase; + + int8_t queryContinue; + int8_t inQueue; + int32_t rspCode; int8_t events[QW_EVENT_MAX]; @@ -144,7 +143,11 @@ typedef struct SQWorkerMgmt { #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) -#define QW_IN_EXECUTOR(ctx) ((ctx)->phase == QW_PHASE_PRE_QUERY || (ctx)->phase == QW_PHASE_PRE_CQUERY || (ctx)->phase == QW_PHASE_PRE_FETCH || (ctx)->phase == QW_PHASE_PRE_SINK) +#define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) + +#define QW_SET_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) + +#define QW_IN_EXECUTOR(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_FETCH) #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index a0beaba61d..375851ff18 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -268,6 +268,8 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ if (HASH_NODE_EXIST(code)) { if (rwType && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + } else if (ctx) { + QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } else { QW_TASK_ELOG("task ctx already exist, id:%s", id); QW_ERR_RET(TSDB_CODE_QRY_TASK_ALREADY_EXIST); @@ -281,6 +283,8 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ if (rwType && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + } else if (ctx) { + QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } return TSDB_CODE_SUCCESS; @@ -309,6 +313,11 @@ int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint return qwAddTaskCtxImpl(QW_FPARAMS(), rwType, 0, ctx); } +int32_t qwAddGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { + return qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, ctx); +} + + void qwReleaseTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->ctxLock); } @@ -416,7 +425,7 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx *ctx = NULL; bool locked = false; - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -447,12 +456,12 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, _return: - if (locked) { - QW_UNLOCK(QW_WRITE, &ctx->lock); + if (code) { + QW_SET_RSP_CODE(ctx, code); } - if (ctx) { - qwReleaseTaskCtx(QW_READ, mgmt); + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); } QW_RET(code); @@ -463,8 +472,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHa bool qcontinue = true; SSDataBlock* pRes = NULL; uint64_t useconds = 0; - - while (qcontinue) { + int32_t i = 0; + + while (true) { + QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); + code = qExecTask(taskHandle, &pRes, &useconds); if (code) { QW_TASK_ELOG("qExecTask failed, code:%x", code); @@ -472,7 +484,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHa } if (NULL == pRes) { - QW_TASK_DLOG("query done, useconds:%"PRIu64, useconds); + QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); dsEndPut(sinkHandle, useconds); break; } @@ -483,6 +495,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHa QW_TASK_ELOG("dsPutDataBlock failed, code:%x", code); QW_ERR_JRET(code); } + + QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); + + if (!qcontinue) { + break; + } } _return: @@ -517,9 +535,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - *rspMsg = rsp; - *dataLen = 0; return TSDB_CODE_SUCCESS; @@ -538,15 +554,12 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_TASK_DLOG("task got data in sink, dataLength:%d", len); QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); - *rspMsg = rsp; pOutput->pData = rsp->data; - code = dsGetDataBlock(ctx->sinkHandle, pOutput); if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x", code); - qwFreeFetchRsp(rsp); QW_ERR_RET(code); } @@ -574,7 +587,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S switch (phase) { case QW_PHASE_PRE_QUERY: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); ctx->phase = phase; @@ -607,7 +620,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_POST_QUERY: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -616,7 +629,9 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S ctx->taskHandle = input->taskHandle; ctx->sinkHandle = input->sinkHandle; - ctx->readyCode = input->code; + if (input->code) { + QW_SET_RSP_CODE(ctx, input->code); + } assert(!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)); @@ -652,8 +667,78 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } break; } + case QW_PHASE_PRE_CQUERY: { + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + ctx->phase = phase; + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task is dropping, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task is cancelling, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + break; + } + case QW_PHASE_POST_CQUERY: { + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + if (input->code) { + QW_SET_RSP_CODE(ctx, input->code); + } + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task is dropping, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task is cancelling, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + break; + } case QW_PHASE_PRE_FETCH: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -691,15 +776,26 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->rspCode = TSDB_CODE_QRY_TASK_MSG_ERROR; QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } break; } case QW_PHASE_POST_FETCH: { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); locked = true; + if (input->code) { + QW_SET_RSP_CODE(ctx, input->code); + } + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); output->needStop = true; @@ -716,12 +812,17 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } break; } - } - _return: if (locked) { @@ -730,10 +831,6 @@ _return: QW_UNLOCK(QW_WRITE, &ctx->lock); } - if (ctx) { - qwReleaseTaskCtx(QW_READ, mgmt); - } - QW_RET(code); } @@ -817,24 +914,32 @@ _return: int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { int32_t code = 0; SQWTaskCtx *ctx = NULL; - - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + int8_t phase = 0; + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_LOCK(QW_WRITE, &ctx->lock); - if (ctx->phase == QW_PHASE_PRE_QUERY) { + phase = QW_GET_PHASE(ctx); + + if (phase == QW_PHASE_PRE_QUERY) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); - } else if (ctx->phase == QW_PHASE_POST_QUERY) { + } else if (phase == QW_PHASE_POST_QUERY) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - QW_ERR_JRET(qwBuildAndSendReadyRsp(qwMsg->connection, ctx->readyCode)); + QW_ERR_JRET(qwBuildAndSendReadyRsp(qwMsg->connection, ctx->rspCode)); + } else { + QW_TASK_ELOG("invalid phase when got ready msg, phase:%d", phase); + assert(0); } _return: + if (code) { + QW_SET_RSP_CODE(ctx, code); + } + if (ctx) { QW_UNLOCK(QW_WRITE, &ctx->lock); - - qwReleaseTaskCtx(QW_READ, mgmt); } QW_RET(code); @@ -842,84 +947,74 @@ _return: int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { + SQWTaskCtx *ctx = NULL; int32_t code = 0; bool queryRsped = false; bool needStop = false; struct SSubplan *plan = NULL; - int32_t rspCode = 0; SQWPhaseInput input = {0}; SQWPhaseOutput output = {0}; - SQWTaskCtx *ctx = NULL; void *rsp = NULL; int32_t dataLen = 0; - - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); - - needStop = output.needStop; - code = output.rspCode; - if (needStop) { - QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_CQUERY); - QW_ERR_JRET(code); - } + do { + QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - qTaskInfo_t taskHandle = ctx->taskHandle; - DataSinkHandle sinkHandle = ctx->sinkHandle; - - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), taskHandle, sinkHandle)); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CQUERY); - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - SOutputData sOutput = {0}; - QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + needStop = output.needStop; + code = output.rspCode; - // Note: schedule data sink firstly and will schedule query after it's done - if (sOutput.scheduleJobNo) { - if (sOutput.scheduleJobNo > ctx->sinkId) { - QW_TASK_DLOG("sink need schedule, scheduleJobNo:%d", sOutput.scheduleJobNo); - - ctx->sinkId = sOutput.scheduleJobNo; - QW_ERR_JRET(qwBuildAndSendSchSinkMsg(QW_FPARAMS(), qwMsg->connection)); + if (needStop) { + QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_CQUERY); + QW_ERR_JRET(code); + } + + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + atomic_store_8(&ctx->inQueue, 0); + + qTaskInfo_t taskHandle = ctx->taskHandle; + DataSinkHandle sinkHandle = ctx->sinkHandle; + + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), taskHandle, sinkHandle)); + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + SOutputData sOutput = {0}; + QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + + if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { + QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); + + atomic_store_8(&ctx->queryContinue, 1); } - } else if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { - QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); - - if (!QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY)) { - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY); - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + + if (rsp) { + qwBuildFetchRsp(rsp, &sOutput, dataLen); - QW_ERR_RET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + } else { + atomic_store_8(&ctx->queryContinue, 1); } } - - if (rsp) { - qwBuildFetchRsp(rsp, &sOutput, dataLen); - } - } + _return: -_return: - - qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - if (code) { + if (code && QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); rsp = NULL; qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); - } else if (rsp) { - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); } - } - - QW_RET(rspCode); + + input.code = code; + qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); + + needStop = output.needStop; + code = output.rspCode; + } while ((!needStop) && (0 == code) && atomic_val_compare_exchange_8(&ctx->queryContinue, 1, 0)); + + QW_RET(code); } @@ -956,35 +1051,35 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t if (NULL == rsp) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); + } else { + qwBuildFetchRsp(rsp, &sOutput, dataLen); } - // Note: schedule data sink firstly and will schedule query after it's done - if (sOutput.scheduleJobNo) { - if (sOutput.scheduleJobNo > ctx->sinkId) { - QW_TASK_DLOG("sink need schedule, scheduleJobNo:%d", sOutput.scheduleJobNo); - - ctx->sinkId = sOutput.scheduleJobNo; - - QW_ERR_JRET(qwBuildAndSendSchSinkMsg(QW_FPARAMS(), qwMsg->connection)); - } - } else if ((!sOutput.queryEnd) && (/* DS_BUF_LOW == sOutput.bufStatus || */ DS_BUF_EMPTY == sOutput.bufStatus)) { + if ((!sOutput.queryEnd) && (/* DS_BUF_LOW == sOutput.bufStatus || */ DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); - if (!QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY)) { - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_CQUERY); + QW_LOCK(QW_WRITE, &ctx->lock); + locked = true; + if (QW_IN_EXECUTOR(ctx)) { + atomic_store_8(&ctx->queryContinue, 1); + } else if (0 == atomic_load_8(&ctx->inQueue)) { QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); - - QW_ERR_RET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); - } - } - if (rsp) { - qwBuildFetchRsp(rsp, &sOutput, dataLen); + atomic_store_8(&ctx->inQueue, 1); + + QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + } } _return: + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + input.code = code; + qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); if (code) { diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index b9fd8e78b6..dad3a5d73b 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -229,42 +229,6 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe return TSDB_CODE_SUCCESS; } - -int32_t qwBuildAndSendSchSinkMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection) { - SRpcMsg *pMsg = (SRpcMsg *)connection; - SSinkDataReq * req = (SSinkDataReq *)rpcMallocCont(sizeof(SSinkDataReq)); - if (NULL == req) { - qError("rpcMallocCont %d failed", (int32_t)sizeof(SSinkDataReq)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - req->header.vgId = mgmt->nodeId; - req->sId = sId; - req->queryId = qId; - req->taskId = tId; - - SRpcMsg pNewMsg = { - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, - .msgType = TDMT_VND_SCHEDULE_DATA_SINK, - .pCont = req, - .contLen = sizeof(SSinkDataReq), - .code = 0, - }; - - int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); - if (TSDB_CODE_SUCCESS != code) { - qError("put data sink schedule msg to queue failed, code:%x", code); - rpcFreeCont(req); - QW_ERR_RET(code); - } - - qDebug("put data sink schedule msg to query queue"); - - return TSDB_CODE_SUCCESS; -} - - int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection) { SRpcMsg *pMsg = (SRpcMsg *)connection; SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); @@ -366,25 +330,6 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } - - -int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ - if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { - return TSDB_CODE_QRY_INVALID_INPUT; - } - - SSinkDataReq *msg = pMsg->pCont; - if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - qError("invalid sink data msg"); - QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - //dsScheduleProcess(); - //TODO - - return TSDB_CODE_SUCCESS; -} - int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; From 71729218863c06a0861ad55e6b2cbf479444a56d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 19 Jan 2022 21:07:29 +0800 Subject: [PATCH 058/103] add client --- source/libs/transport/src/transCli.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f197e72ec5..2ecc9e2343 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -53,7 +53,7 @@ typedef struct SClientObj { static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); -static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void clientWriteCb(uv_write_t* req, int status); static void clientConnCb(uv_connect_t* req, int status); @@ -65,7 +65,7 @@ static void* clientThread(void* arg); static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void clientAllocrReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { // impl later } static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { @@ -96,7 +96,7 @@ static void clientWriteCb(uv_write_t* req, int status) { return; } - uv_read_start((uv_stream_t*)pConn->stream, clientAllocrReadBufferCb, clientReadCb); + uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } @@ -121,8 +121,6 @@ static void clientConnCb(uv_connect_t* req, int status) { SCliMsg* pMsg = pConn->data; SEpSet* pEpSet = &pMsg->context->epSet; SRpcMsg rpcMsg; - // rpcMsg.ahandle = pMsg->context->ahandle; - // rpcMsg.pCont = NULL; char* fqdn = pEpSet->fqdn[pEpSet->inUse]; uint32_t port = pEpSet->port[pEpSet->inUse]; @@ -135,6 +133,7 @@ static void clientConnCb(uv_connect_t* req, int status) { return; } assert(pConn->stream == req->handle); + clientWrite(pConn); } static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { From f04292b8d49995e6ce1c99378863b4132d270e92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 19 Jan 2022 22:35:25 +0800 Subject: [PATCH 059/103] [td-11818]remove SParseBasicCtx --- source/client/inc/clientInt.h | 2 +- source/client/src/clientImpl.c | 28 ++++++++++++++++++++++---- source/client/src/clientMsgHandler.c | 2 +- source/libs/parser/src/dCDAstProcess.c | 9 +++++++++ source/util/src/tconfig.c | 2 +- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b6f38b12ca..1d10869e30 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -179,7 +179,7 @@ extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); -SMsgSendInfo* buildMsgInfoImpl(SRequestObj*); +SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); int taos_init(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 10d1d09c85..75b652e40d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -195,8 +195,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { } asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } else { - SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, pSendMsg); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } tsem_wait(&pRequest->body.rspSem); @@ -706,6 +705,8 @@ void* doFetchRow(SRequestObj* pRequest) { assert(pRequest != NULL); SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + SEpSet epSet = {0}; + if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { // All data has returned to App already, no need to try again @@ -729,6 +730,17 @@ void* doFetchRow(SRequestObj* pRequest) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; + SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); + + epSet.numOfEps = pVgroupInfo->numOfEps; + epSet.inUse = pVgroupInfo->inUse; + + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); + epSet.port[i] = pVgroupInfo->epAddr[i].port; + } + } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { pRequest->type = TDMT_VND_SHOW_TABLES; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -746,9 +758,17 @@ void* doFetchRow(SRequestObj* pRequest) { SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + epSet.numOfEps = pVgroupInfo->numOfEps; + epSet.inUse = pVgroupInfo->inUse; + + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); + epSet.port[i] = pVgroupInfo->epAddr[i].port; + } + int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; @@ -758,7 +778,7 @@ void* doFetchRow(SRequestObj* pRequest) { int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); tsem_wait(&pRequest->body.rspSem); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 497ef1ac95..02e36043dc 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -115,7 +115,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { } } else { assert(pRequest != NULL); - pMsgSendInfo->msgInfo = pRequest->body.requestMsg; + pMsgSendInfo->msgInfo = pRequest->body.requestMsg; } pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 955507ae1b..3b7b9e44d9 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -768,6 +768,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)buildUserManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_USER) ? TDMT_MND_CREATE_USER : TDMT_MND_ALTER_USER; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -809,6 +810,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)buildAcctManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_ACCT) ? TDMT_MND_CREATE_ACCT : TDMT_MND_ALTER_ACCT; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -816,6 +818,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch case TSDB_SQL_DROP_USER: { pDcl->pMsg = (char*)buildDropUserMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); pDcl->msgType = (pInfo->type == TSDB_SQL_DROP_ACCT) ? TDMT_MND_DROP_ACCT : TDMT_MND_DROP_USER; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -852,6 +855,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)pUseDbMsg; pDcl->msgLen = sizeof(SUseDbReq); pDcl->msgType = TDMT_MND_USE_DB; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -880,6 +884,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } + pDcl->epSet = pCtx->mgmtEpSet; pDcl->pMsg = (char*)pCreateMsg; pDcl->msgLen = sizeof(SCreateDbReq); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB) ? TDMT_MND_CREATE_DB : TDMT_MND_ALTER_DB; @@ -908,6 +913,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->msgType = TDMT_MND_DROP_DB; pDcl->msgLen = sizeof(SDropDbReq); pDcl->pMsg = (char*)pDropDbMsg; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -920,6 +926,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)buildCreateStbMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); pDcl->msgType = TDMT_MND_CREATE_STB; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -940,6 +947,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch } pDcl->msgType = TDMT_MND_CREATE_DNODE; + pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -950,6 +958,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch } pDcl->msgType = TDMT_MND_DROP_DNODE; + pDcl->epSet = pCtx->mgmtEpSet; break; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 726247d450..469da11d93 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tconfig.h" -#include "ulog.h" #include "tutil.h" +#include "ulog.h" SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; int32_t tsGlobalConfigNum = 0; From 75338f80f92c986887faaabae81d671c67d1ea74 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 20 Jan 2022 09:14:39 +0800 Subject: [PATCH 060/103] feature/qnode --- source/libs/qworker/src/qworker.c | 76 +++++++++++++++++++------------ 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 375851ff18..57bb73c500 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -254,6 +254,19 @@ int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ return TSDB_CODE_SUCCESS; } +int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); + if (NULL == (*ctx)) { + QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); + QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, int32_t status, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -294,19 +307,6 @@ int32_t qwAddTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tI QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, NULL)); } -int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { - char id[sizeof(qId) + sizeof(tId)] = {0}; - QW_SET_QTID(id, qId, tId); - - *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); - if (NULL == (*ctx)) { - QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); - } - - return TSDB_CODE_SUCCESS; - -} int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWTaskCtx **ctx) { @@ -322,16 +322,33 @@ void qwReleaseTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->ctxLock); } +void qwFreeTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + // RC WARNING + qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); + if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { + qDestroyTask(taskHandle); + } +} -void qwFreeTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx *ctx) { - if (ctx->taskHandle) { - qDestroyTask(ctx->taskHandle); - ctx->taskHandle = NULL; +int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + int32_t code = 0; + // RC WARNING + qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); + if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { + code = qKillTask(taskHandle); + atomic_store_ptr(&ctx->taskHandle, taskHandle); } - // TODO - if (ctx->sinkHandle) { + QW_RET(code); +} + +void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + qwFreeTaskHandle(QW_FPARAMS(), ctx); + + if (ctx->sinkHandle) { + dsDestroyDataSinker(ctx->sinkHandle); + ctx->sinkHandle = NULL; } } @@ -425,7 +442,7 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx *ctx = NULL; bool locked = false; - QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -437,9 +454,7 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, } if (QW_IN_EXECUTOR(ctx)) { - if (ctx->taskHandle) { - QW_ERR_JRET(qKillTask(ctx->taskHandle)); - } + QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); } else if (ctx->phase > 0) { @@ -587,7 +602,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S switch (phase) { case QW_PHASE_PRE_QUERY: { - QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); ctx->phase = phase; @@ -607,7 +622,6 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->needStop = true; QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - qwFreeTask(QW_FPARAMS(), ctx); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); @@ -620,7 +634,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_POST_QUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -702,7 +716,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_POST_CQUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -738,7 +752,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_PRE_FETCH: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -786,7 +800,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S break; } case QW_PHASE_POST_FETCH: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -983,7 +997,8 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); - + + // RC WARNING atomic_store_8(&ctx->queryContinue, 1); } @@ -1061,6 +1076,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_LOCK(QW_WRITE, &ctx->lock); locked = true; + // RC WARNING if (QW_IN_EXECUTOR(ctx)) { atomic_store_8(&ctx->queryContinue, 1); } else if (0 == atomic_load_8(&ctx->inQueue)) { From 6cf90b0a92061a85a7deea655c556c8fbbd1bb36 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 09:49:27 +0800 Subject: [PATCH 061/103] add timer for mq rebalance --- include/common/tmsg.h | 4 + include/common/tmsgdef.h | 1 + source/client/test/clientTests.cpp | 38 +++---- source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 114 +++++++++++++++++++-- source/dnode/mnode/impl/src/mnode.c | 25 +++-- source/dnode/vnode/src/vnd/vnodeWrite.c | 9 ++ 9 files changed, 158 insertions(+), 37 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 15daa97dbf..f6bee57c94 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1089,6 +1089,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq return buf; } +typedef struct SMqTmrMsg { + int32_t reserved; +} SMqTmrMsg; + typedef struct { int64_t status; } SMVSubscribeRsp; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index bfeba885d0..93adc58eca 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -140,6 +140,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-timer", SMqTmrMsg, SMqTmrMsg) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 415d6a57ce..1250402caf 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -148,30 +148,30 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} // -//TEST(testCase, create_db_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} + TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); - //taos_free_result(pRes); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_close(pConn); +} - //pRes = taos_query(pConn, "create database abc1 vgroups 4"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_close(pConn); -//} -// //TEST(testCase, create_dnode_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index ab4ae4ac53..f4fda75bd8 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -100,7 +100,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 45759117b5..ed88b42d39 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -354,6 +354,7 @@ typedef struct SMqSubscribeObj { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t epoch; //TODO: replace with priority queue + int32_t nextConsumerIdx; SArray* availConsumer; // SArray (consumerId) SArray* assigned; // SArray SArray* unassignedConsumer; // SArray diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index d2107b9d07..29ccd43622 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -80,6 +80,7 @@ typedef struct SMnode { SReplica replicas[TSDB_MAX_REPLICA]; tmr_h timer; tmr_h transTimer; + tmr_h mqTimer; char *path; SMnodeCfg cfg; int64_t checkTime; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 52f33fa89c..7b95e93257 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ +#include "mndSubscribe.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndShow.h" #include "mndStb.h" -#include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" @@ -40,6 +40,10 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); +static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); + +static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic); int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -54,9 +58,90 @@ int32_t mndInitSubscribe(SMnode *pMnode) { /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/ /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/ mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); return sdbSetTable(pMnode->pSdb, table); } +static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { + int i = 0; + while (key[i] != ':') { + i++; + } + key[i] = 0; + *topic = strdup(key); + key[i] = ':'; + *cgroup = strdup(&key[i + 1]); + return 0; +} + +static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + SMqSubscribeObj *pSub = NULL; + void *pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); + int sz; + while (pIter != NULL) { + if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) { + char *topic = NULL; + char *cgroup = NULL; + mndSplitSubscribeKey(pSub->key, &topic, &cgroup); + + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + + // create trans + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + for (int i = 0; i < sz; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx); + SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); + pCEp->consumerId = consumerId; + taosArrayPush(pSub->assigned, pCEp); + pSub->nextConsumerIdx++; + + // build msg + SMqSetCVgReq req = { + .vgId = pCEp->vgId, + .consumerId = consumerId, + }; + strcpy(req.cGroup, cgroup); + strcpy(req.topicName, topic); + strcpy(req.sql, pTopic->sql); + strcpy(req.logicalPlan, pTopic->logicalPlan); + strcpy(req.physicalPlan, pTopic->physicalPlan); + int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + void *reqStr = malloc(tlen); + if (reqStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + void *abuf = reqStr; + tEncodeSMqSetCVgReq(abuf, &req); + + // persist msg + STransAction action = {0}; + action.epSet = pCEp->epset; + action.pCont = reqStr; + action.contLen = tlen; + action.msgType = TDMT_VND_MQ_SET_CONN; + mndTransAppendRedoAction(pTrans, &action); + + // persist raw + SSdbRaw *pRaw = mndSubActionEncode(pSub); + mndTransAppendRedolog(pTrans, pRaw); + + tfree(topic); + tfree(cgroup); + } + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + } + mndReleaseTopic(pMnode, pTopic); + mndTransDrop(pTrans); + } + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); + } + return 0; +} + static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { SMqConsumerEp CEp; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; @@ -76,7 +161,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic) { + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic) { int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); for (int32_t i = 0; i < sz; i++) { int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i); @@ -86,7 +171,10 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume .consumerId = pConsumer->consumerId, }; strcpy(req.cGroup, pConsumer->cgroup); - strcpy(req.topicName, pConsumerTopic->name); + strcpy(req.topicName, pTopic->name); + strcpy(req.sql, pTopic->sql); + strcpy(req.logicalPlan, pTopic->logicalPlan); + strcpy(req.physicalPlan, pTopic->physicalPlan); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *reqStr = malloc(tlen); if (reqStr == NULL) { @@ -94,7 +182,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume return -1; } void *abuf = reqStr; - tEncodeSMqSetCVgReq(abuf, &req); + tEncodeSMqSetCVgReq(&abuf, &req); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); @@ -278,20 +366,20 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { int i = 0, j = 0; while (i < newTopicNum || j < oldTopicNum) { - char* newTopicName = NULL; - char* oldTopicName = NULL; + char *newTopicName = NULL; + char *oldTopicName = NULL; if (i >= newTopicNum) { // encode unset topic msg to all vnodes related to that topic - oldTopicName = ((SMqConsumerTopic*)taosArrayGet(oldSub, j))->name; + oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name; j++; } else if (j >= oldTopicNum) { newTopicName = taosArrayGet(newSub, i); i++; } else { newTopicName = taosArrayGet(newSub, i); - oldTopicName = ((SMqConsumerTopic*)taosArrayGet(oldSub, j))->name; + oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name; - int comp = compareLenPrefixedStr(newTopicName, oldTopicName); + int comp = compareLenPrefixedStr(newTopicName, oldTopicName); if (comp == 0) { // do nothing oldTopicName = newTopicName = NULL; @@ -374,19 +462,25 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { // set unassigned vg mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); } + taosArrayPush(pSub->availConsumer, &consumerId); + + //TODO: no need SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); taosArrayPush(pConsumer->topics, pConsumerTopic); if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); // send setmsg to vnode - if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic) < 0) { + if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) { // TODO return -1; } } taosArrayDestroy(pConsumerTopic->pVgInfo); free(pConsumerTopic); + SSdbRaw *pRaw = mndSubActionEncode(pSub); + /*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/ + mndTransAppendRedolog(pTrans, pRaw); #if 0 SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); if (pGroup == NULL) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index cab30702ea..e57ee3eabc 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -18,6 +18,7 @@ #include "mndAuth.h" #include "mndBnode.h" #include "mndCluster.h" +#include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" @@ -27,6 +28,7 @@ #include "mndShow.h" #include "mndSnode.h" #include "mndStb.h" +#include "mndSubscribe.h" #include "mndSync.h" #include "mndTelem.h" #include "mndTopic.h" @@ -69,15 +71,15 @@ static void mndTransReExecute(void *param, void *tmrId) { taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); } -static void mndCalMqRebalance(void* param, void* tmrId) { - SMnode* pMnode = param; +static void mndCalMqRebalance(void *param, void *tmrId) { + SMnode *pMnode = param; if (mndIsMaster(pMnode)) { - // iterate cgroup, cal rebalance - // sync with raft - // write sdb + SMqTmrMsg *pMsg = rpcMallocCont(sizeof(SMqTmrMsg)); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pMsg, .contLen = sizeof(SMqTmrMsg)}; + pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer); } static int32_t mndInitTimer(SMnode *pMnode) { @@ -95,6 +97,11 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } + if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; } @@ -102,6 +109,8 @@ static void mndCleanupTimer(SMnode *pMnode) { if (pMnode->timer != NULL) { taosTmrStop(pMnode->transTimer); pMnode->transTimer = NULL; + taosTmrStop(pMnode->mqTimer); + pMnode->mqTimer = NULL; taosTmrCleanUp(pMnode->timer); pMnode->timer = NULL; } @@ -171,6 +180,8 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -377,7 +388,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS) { + if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index bb863d6ed0..ea69f4c870 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -108,6 +108,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } break; + case TDMT_VND_MQ_SET_CONN: { + char* reqStr = ptr; + SMqSetCVgReq req; + /*tDecodeSMqSetCVgReq(reqStr, &req);*/ + // create topic if not exist + // convert to task + // write mq meta + } + break; default: ASSERT(0); break; From bc1d8f6a7e1d07d21cdf2e3809993a37f5306571 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Jan 2022 10:12:06 +0800 Subject: [PATCH 062/103] [modify insert] --- tests/test/c/create_table.c | 87 +++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 6df31c7b8b..d387bf483b 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -28,9 +28,14 @@ int32_t numOfThreads = 1; int64_t numOfTables = 200000; int32_t createTable = 1; int32_t insertData = 0; -int32_t batchNum = 100; +int32_t batchNumOfTbl = 100; +int32_t batchNumOfRow = 1; int32_t numOfVgroups = 2; int32_t showTablesFlag = 0; +int32_t queryFlag = 0; + +int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 + typedef struct { int64_t tableBeginIndex; @@ -167,7 +172,7 @@ void showTables() { void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; - char *qstr = malloc(2000 * 1000); + char *qstr = malloc(batchNumOfTbl * batchNumOfRow * 128); int32_t code = 0; TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); @@ -192,7 +197,7 @@ void *threadFunc(void *param) { // batch = MIN(batch, batchNum); int32_t len = sprintf(qstr, "create table"); - for (int32_t i = 0; i < batchNum;) { + for (int32_t i = 0; i < batchNumOfTbl;) { len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); t++; i++; @@ -205,7 +210,7 @@ void *threadFunc(void *param) { TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - pError("failed to create table t%" PRId64 ", code: %d, reason:%s", t, code, tstrerror(code)); + pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); } taos_free_result(pRes); int64_t endTs = taosGetTimestampUs(); @@ -227,31 +232,49 @@ void *threadFunc(void *param) { if (insertData) { int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex;) { + // int64_t batch = (pInfo->tableEndIndex - t); + // batch = MIN(batch, batchNum); - pInfo->startMs = taosGetTimestampMs(); - for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { - int64_t batch = (pInfo->tableEndIndex - t); - batch = MIN(batch, batchNum); - - int32_t len = sprintf(qstr, "insert into"); - for (int32_t i = 0; i < batch; ++i) { - len += sprintf(qstr + len, " t%" PRId64 " values(now, %" PRId64 ")", t + i, t + i); - } + int32_t len = sprintf(qstr, "insert into "); + + for (int32_t i = 0; i < batchNumOfTbl;) { + int64_t ts = startTimestamp; + len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); + for (int32_t j = 0; j < batchNumOfRow; j++) { + len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + } + + t++; + i++; + if (t > pInfo->tableEndIndex) { + break; + } + } + int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); - if (code != 0) { - pError("failed to insert table t%" PRId64 ", reason:%s", t, tstrerror(code)); + if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); } taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; curMs = taosGetTimestampMs(); if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); printInsertProgress(pInfo, t); } - t += (batch - 1); } - printInsertProgress(pInfo, pInfo->tableEndIndex); + printInsertProgress(pInfo, t); } taos_close(con); @@ -280,9 +303,13 @@ void printHelp() { printf("%s%s\n", indent, "-i"); printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); printf("%s%s\n", indent, "-b"); - printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum); + printf("%s%s%s%d\n", indent, indent, "batchNumOfTbl, default is ", batchNumOfTbl); printf("%s%s\n", indent, "-w"); printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); exit(EXIT_SUCCESS); } @@ -309,10 +336,15 @@ void parseArgument(int32_t argc, char *argv[]) { } else if (strcmp(argv[i], "-i") == 0) { insertData = atoi(argv[++i]); } else if (strcmp(argv[i], "-b") == 0) { - batchNum = atoi(argv[++i]); + batchNumOfTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + batchNumOfRow = atoi(argv[++i]); } else if (strcmp(argv[i], "-w") == 0) { showTablesFlag = atoi(argv[++i]); + } else if (strcmp(argv[i], "-q") == 0) { + queryFlag = atoi(argv[++i]); } else { + pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); } } @@ -324,8 +356,10 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); pPrint("%s createTable:%d %s", GREEN, createTable, NC); pPrint("%s insertData:%d %s", GREEN, insertData, NC); - pPrint("%s batchNum:%d %s", GREEN, batchNum, NC); + pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); + pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); + pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); pPrint("%s start create table performace test %s", GREEN, NC); } @@ -338,8 +372,15 @@ int32_t main(int32_t argc, char *argv[]) { return 0; } - createDbAndStb(); + if (queryFlag) { + //selectRowsFromTable(); + return 0; + } + if (createTable) { + createDbAndStb(); + } + pPrint("%d threads are spawned to create %" PRId64 " tables", numOfThreads, numOfTables); pthread_attr_t thattr; @@ -396,9 +437,11 @@ int32_t main(int32_t argc, char *argv[]) { insertDataSpeed += pInfo[i].insertDataSpeed; } - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 + if (createTable) { + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); + } if (insertData) { pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, From b9033a75015e1c74e592867483446759ca7b67e7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 20 Jan 2022 10:36:06 +0800 Subject: [PATCH 063/103] feature/qnode --- source/libs/qworker/inc/qworkerInt.h | 1 + source/libs/qworker/src/qworker.c | 47 +++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 193287eeac..7d043a0e02 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -105,6 +105,7 @@ typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; + bool emptyRes; int8_t queryContinue; int8_t inQueue; int32_t rspCode; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 50a6158631..c6e2bad421 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -532,6 +532,20 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void bool queryEnd = false; int32_t code = 0; + if (ctx->emptyRes) { + QW_TASK_DLOG("query empty result, query end, phase:%d", ctx->phase); + + QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); + + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + + *rspMsg = rsp; + *dataLen = 0; + pOutput->queryEnd = true; + + return TSDB_CODE_SUCCESS; + } + dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); if (len < 0) { @@ -644,6 +658,10 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S ctx->taskHandle = input->taskHandle; ctx->sinkHandle = input->sinkHandle; + + if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { + ctx->emptyRes = true; + } if (input->code) { QW_SET_RSP_CODE(ctx, input->code); @@ -885,12 +903,14 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_ERR_JRET(code); } - QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, TSDB_CODE_SUCCESS)); + QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, code)); + QW_TASK_DLOG("query msg rsped, code:%d", code); queryRsped = true; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), pTaskInfo, sinkHandle)); - + if (pTaskInfo && sinkHandle) { + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), pTaskInfo, sinkHandle)); + } _return: if (code) { @@ -899,6 +919,10 @@ _return: if (!queryRsped) { code = qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); + if (TSDB_CODE_SUCCESS == code) { + QW_TASK_DLOG("query msg rsped, code:%d", rspCode); + } + if (TSDB_CODE_SUCCESS == rspCode && code) { rspCode = code; } @@ -921,7 +945,8 @@ _return: QW_ERR_RET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); if (queryRsped && output.needRsp) { - qwBuildAndSendReadyRsp(qwMsg->connection, output.rspCode); + qwBuildAndSendReadyRsp(qwMsg->connection, output.rspCode); + QW_TASK_DLOG("ready msg rsped, code:%x", output.rspCode); } QW_RET(rspCode); @@ -940,9 +965,11 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t if (phase == QW_PHASE_PRE_QUERY) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); + QW_TASK_DLOG("ready msg not rsped, phase:%d", phase); } else if (phase == QW_PHASE_POST_QUERY) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); QW_ERR_JRET(qwBuildAndSendReadyRsp(qwMsg->connection, ctx->rspCode)); + QW_TASK_DLOG("ready msg rsped, code:%x", ctx->rspCode); } else { QW_TASK_ELOG("invalid phase when got ready msg, phase:%d", phase); assert(0); @@ -1009,7 +1036,8 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); } else { atomic_store_8(&ctx->queryContinue, 1); } @@ -1022,6 +1050,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t qwFreeFetchRsp(rsp); rsp = NULL; qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, 0); } input.code = code; @@ -1072,7 +1101,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t qwBuildFetchRsp(rsp, &sOutput, dataLen); } - if ((!sOutput.queryEnd) && (/* DS_BUF_LOW == sOutput.bufStatus || */ DS_BUF_EMPTY == sOutput.bufStatus)) { + if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end, need to continue, bufStatus:%d", sOutput.bufStatus); QW_LOCK(QW_WRITE, &ctx->lock); @@ -1103,12 +1132,14 @@ _return: if (code) { qwFreeFetchRsp(rsp); rsp = NULL; - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); + dataLen = 0; + qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); } else if (rsp) { qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); + QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); } - QW_RET(code); } From 5c5d593e41849dd92d19179bdd3f683c0f3436df Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 18:43:32 -0800 Subject: [PATCH 064/103] add test cases --- source/libs/tfs/src/tfs.c | 5 +- source/libs/tfs/test/tfsTest.cpp | 135 ++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 4 deletions(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index be411744cf..582422dc94 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -291,6 +291,8 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); if (taosRenameFile(oaname, naname) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); return -1; } } @@ -402,8 +404,7 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { } if (tfsFormatDir(pCfg->dir, dirName) < 0) { - fError("failed to mount %s to FS since invalid dir format", pCfg->dir); - terrno = TSDB_CODE_FS_INVLD_CFG; + fError("failed to mount %s to FS since %s", pCfg->dir, terrstr()); return -1; } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index a85bea192c..09f634bf4a 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -12,16 +12,147 @@ #include #include "os.h" +#include "tfs.h" + class TfsTest : public ::testing::Test { protected: - static void SetUpTestSuite() {} + static void SetUpTestSuite() { root = "/tmp/tfsTest"; } static void TearDownTestSuite() {} public: void SetUp() override {} void TearDown() override {} + + static const char *root; }; +const char *TfsTest::root; + TEST_F(TfsTest, 01_Open_Close) { - + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosRemoveDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_EQ(pTfs, nullptr); + + taosMkDir(root); + pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + tfsUpdateSize(pTfs); + SDiskSize size = tfsGetSize(pTfs); + + EXPECT_GT(size.avail, 0); + EXPECT_GT(size.used, 0); + EXPECT_GT(size.total, size.avail); + EXPECT_GT(size.total, size.used); + + tfsClose(pTfs); } + +TEST_F(TfsTest, 02_AllocDisk) { + int32_t code = 0; + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosMkDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + SDiskID did; + did.id = 0; + did.level = 0; + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + did.id = 1; + did.level = 1; + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + did.id = 1; + did.level = 2; + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + did.id = 1; + did.level = 3; + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + + const char *primary = tfsGetPrimaryPath(pTfs); + EXPECT_STREQ(primary, root); + + const char *path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root); + + tfsClose(pTfs); +} + +TEST_F(TfsTest, 03_Dir) { + int32_t code = 0; + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosMkDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + char p1[] = "p1"; + char ap1[128] = {0}; + snprintf(ap1, 128, "%s%s%s", root, TD_DIRSEP, p1); + EXPECT_NE(taosDirExist(ap1), 0); + EXPECT_EQ(tfsMkdir(pTfs, p1), 0); + EXPECT_EQ(taosDirExist(ap1), 0); + + char p2[] = "p2"; + char ap2[128] = {0}; + snprintf(ap2, 128, "%s%s%s", root, TD_DIRSEP, p2); + SDiskID did = {0}; + EXPECT_NE(taosDirExist(ap2), 0); + EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); + EXPECT_EQ(taosDirExist(ap2), 0); + + char p3[] = "p3/p2/p1/p0"; + char ap3[128] ={0}; + snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3); + EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(tfsMkdir(pTfs, p3), 0); + EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); + EXPECT_EQ(taosDirExist(ap3), 0); + + EXPECT_EQ(tfsRmdir(pTfs, p3), 0); + EXPECT_NE(taosDirExist(ap3), 0); + + char p45[] = "p5"; + char p44[] = "p4"; + char p4[] = "p4/p2/p1/p0"; + char ap4[128] = {0}; + snprintf(ap4, 128, "%s%s%s", root, TD_DIRSEP, p4); + + EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); + EXPECT_EQ(taosDirExist(ap4), 0); + EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRmdir(pTfs, p4), 0); + EXPECT_NE(taosDirExist(ap4), 0); + + tfsClose(pTfs); +} \ No newline at end of file From 5d77fab535473d59f1f6fbca47d341adb9a8fd67 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 10:45:15 +0800 Subject: [PATCH 065/103] add msg for vnode consume --- include/common/tmsg.h | 19 +++++++ include/common/tmsgdef.h | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 7 ++- source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/inc/tq.h | 60 +++++++++++++++------- source/dnode/vnode/src/inc/vnd.h | 2 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 + source/dnode/vnode/src/vnd/vnodeWrite.c | 30 ++++++++++- 8 files changed, 99 insertions(+), 23 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f6bee57c94..1f7ec3be1e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1552,6 +1552,25 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { return buf; } +typedef struct SMqCVConsumeReq { + int64_t reqId; + int64_t offset; + int64_t clientId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; +} SMqCVConsumeReq; + +typedef struct SMqCVConsumeRsp { + int64_t reqId; + int64_t clientId; + int64_t committedOffset; + int64_t receiveOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t bodyLen; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char body[]; +} SMqCvConsumeRsp; #ifdef __cplusplus } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 93adc58eca..ed7fdea2a8 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -175,6 +175,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) + TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) // Requests handled by QNODE diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 7b95e93257..e786a8972c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -464,7 +464,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } taosArrayPush(pSub->availConsumer, &consumerId); - //TODO: no need + // TODO: no need SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); taosArrayPush(pConsumer->topics, pConsumerTopic); @@ -542,7 +542,10 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 9dc4bb1873..75734d2d29 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -25,6 +25,7 @@ target_link_libraries( PUBLIC bdb PUBLIC tfs PUBLIC wal + PUBLIC scheduler PUBLIC qworker ) diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 8089826a80..1a41a02673 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -18,14 +18,15 @@ #include "common.h" #include "mallocator.h" +#include "meta.h" #include "os.h" +#include "scheduler.h" #include "taoserror.h" -#include "tmsg.h" #include "tlist.h" +#include "tmsg.h" #include "trpc.h" #include "ttimer.h" #include "tutil.h" -#include "meta.h" #ifdef __cplusplus extern "C" { @@ -150,15 +151,40 @@ typedef struct STqListHandle { } STqList; typedef struct STqGroup { - int64_t clientId; - int64_t cgId; - void* ahandle; - int32_t topicNum; + int64_t clientId; + int64_t cgId; + void* ahandle; + int32_t topicNum; STqList* head; SList* topicList; // SList STqRspHandle rspHandle; } STqGroup; +typedef struct STqTaskItem { + int32_t status; + void* dst; + SSubQueryMsg* pMsg; +} STqTaskItem; + +// new version +typedef struct STqBuffer { + int64_t firstOffset; + int64_t lastOffset; + STqTaskItem output[TQ_BUFFER_SIZE]; +} STqBuffer; + +typedef struct STqClientHandle { + int64_t clientId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_TOPIC_FNAME_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + int64_t committedOffset; + int64_t currentOffset; + STqBuffer buffer; +} STqClientHandle; + typedef struct STqQueryMsg { STqMsgItem* item; struct STqQueryMsg* next; @@ -253,7 +279,7 @@ typedef struct STqMetaStore { // a table head STqMetaList* unpersistHead; // topics that are not connectted - STqMetaList* unconnectTopic; + STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile int fileFd; @@ -316,24 +342,22 @@ const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**); static int tqQueryExecuting(int32_t status) { return status; } typedef struct STqReadHandle { - int64_t ver; - SSubmitMsg* pMsg; - SSubmitBlk* pBlock; + int64_t ver; + SSubmitMsg* pMsg; + SSubmitBlk* pBlock; SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; - SMeta* pMeta; + SMeta* pMeta; } STqReadHandle; typedef struct SSubmitBlkScanInfo { - } SSubmitBlkScanInfo; -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg *pMsg); -bool tqNextDataBlock(STqReadHandle* pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo *pBlockInfo); -//return SArray -SArray *tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList); -//int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t status); +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg* pMsg); +bool tqNextDataBlock(STqReadHandle* pHandle); +int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); +// return SArray +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 6b99b71b62..be32ed6829 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -177,4 +177,4 @@ bool vmaIsFull(SVMemAllocator* pVMA); } #endif -#endif /*_TD_VNODE_DEF_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_DEF_H_*/ diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2e9c77c59b..729d64f8b3 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -57,6 +57,8 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg, pRsp); + case TDMT_VND_CONSUME: + return 0; default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ea69f4c870..2f3a4d5409 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -14,6 +14,7 @@ */ #include "vnd.h" +#include "tq.h" int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { @@ -111,9 +112,34 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TDMT_VND_MQ_SET_CONN: { char* reqStr = ptr; SMqSetCVgReq req; - /*tDecodeSMqSetCVgReq(reqStr, &req);*/ - // create topic if not exist + tDecodeSMqSetCVgReq(reqStr, &req); + STqClientHandle* pHandle = calloc(sizeof(STqClientHandle), 1); + if (pHandle == NULL) { + // TODO: handle error + } + strcpy(pHandle->topicName, req.topicName); + strcpy(pHandle->cGroup, req.cGroup); + strcpy(pHandle->sql, req.sql); + strcpy(pHandle->logicalPlan, req.logicalPlan); + strcpy(pHandle->physicalPlan, req.physicalPlan); + SArray *pArray; + //TODO: deserialize to SQueryDag + SQueryDag *pDag; // convert to task + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + // TODO: handle error + } + ASSERT(taosArrayGetSize(pArray) == 0); + STaskInfo *pInfo = taosArrayGet(pArray, 0); + SArray* pTasks; + schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); + pHandle->buffer.firstOffset = -1; + pHandle->buffer.lastOffset = -1; + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { + SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); + pHandle->buffer.output[i].pMsg = pMsg; + pHandle->buffer.output[i].status = 0; + } // write mq meta } break; From a0a61fa91dbde00c88d84abeb6130181cb6172d8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 19 Jan 2022 21:51:23 -0500 Subject: [PATCH 066/103] TD-13120 SELECT statement data structure definition --- include/libs/function/functionMgt.h | 66 +++++ include/nodes/nodes.h | 271 +++++++++++++++++++++ source/CMakeLists.txt | 3 +- source/libs/function/CMakeLists.txt | 2 +- source/libs/function/inc/functionMgtInt.h | 76 ++++++ source/libs/function/inc/taggfunction.h | 4 + source/libs/function/src/buildins.c | 21 ++ source/libs/function/src/functionMgt.c | 80 ++++++ source/libs/function/src/taggfunction.c | 6 + source/libs/planner/src/logicPlan.c | 2 +- source/libs/planner/src/physicalPlanJson.c | 32 ++- source/nodes/CMakeLists.txt | 15 ++ source/nodes/src/nodesClone.c | 20 ++ source/nodes/src/nodesCode.c | 24 ++ source/nodes/src/nodesEqual.c | 141 +++++++++++ source/nodes/src/nodesTraverse.c | 83 +++++++ source/nodes/src/nodesUtil.c | 24 ++ source/nodes/test/CMakeLists.txt | 19 ++ source/nodes/test/nodesTest.cpp | 25 ++ 19 files changed, 907 insertions(+), 7 deletions(-) create mode 100644 include/libs/function/functionMgt.h create mode 100644 include/nodes/nodes.h create mode 100644 source/libs/function/inc/functionMgtInt.h create mode 100644 source/libs/function/src/buildins.c create mode 100644 source/libs/function/src/functionMgt.c create mode 100644 source/nodes/CMakeLists.txt create mode 100644 source/nodes/src/nodesClone.c create mode 100644 source/nodes/src/nodesCode.c create mode 100644 source/nodes/src/nodesEqual.c create mode 100644 source/nodes/src/nodesTraverse.c create mode 100644 source/nodes/src/nodesUtil.c create mode 100644 source/nodes/test/CMakeLists.txt create mode 100644 source/nodes/test/nodesTest.cpp diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h new file mode 100644 index 0000000000..3c365335be --- /dev/null +++ b/include/libs/function/functionMgt.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_FUNCTION_MGT_H_ +#define _TD_FUNCTION_MGT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nodes.h" + +struct SQLFunctionCtx; +struct SResultRowEntryInfo; +struct STimeWindow; + +typedef struct SFuncExecEnv { + int32_t calcMemSize; +} SFuncExecEnv; + +typedef void* FuncMgtHandle; +typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +typedef bool (*FExecInit)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +typedef void (*FExecProcess)(struct SQLFunctionCtx *pCtx); +typedef void (*FExecFinalize)(struct SQLFunctionCtx *pCtx); + +typedef struct SFuncExecFuncs { + FExecGetEnv getEnv; + FExecInit init; + FExecProcess process; + FExecFinalize finalize; +} SFuncExecFuncs; + +int32_t fmFuncMgtInit(); + +int32_t fmGetHandle(FuncMgtHandle* pHandle); + +int32_t fmGetFuncId(FuncMgtHandle handle, const char* name); +int32_t fmGetFuncResultType(FuncMgtHandle handle, SFunctionNode* pFunc); +bool fmIsAggFunc(int32_t funcId); +bool fmIsStringFunc(int32_t funcId); +bool fmIsTimestampFunc(int32_t funcId); +bool fmIsTimelineFunc(int32_t funcId); +bool fmIsTimeorderFunc(int32_t funcId); +bool fmIsNonstandardSQLFunc(int32_t funcId); +int32_t fmFuncScanType(int32_t funcId); + +int32_t fmGetFuncExecFuncs(FuncMgtHandle handle, int32_t funcId, SFuncExecFuncs* pFpSet); + +#ifdef __cplusplus +} +#endif + +#endif // _TD_FUNCTION_MGT_H_ diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h new file mode 100644 index 0000000000..e6b41adf14 --- /dev/null +++ b/include/nodes/nodes.h @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_NODES_H_ +#define _TD_NODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" +#include "tdef.h" + +typedef enum ENodeType { + QUERY_NODE_COLUMN = 1, + QUERY_NODE_VALUE, + QUERY_NODE_OPERATOR, + QUERY_NODE_LOGIC_CONDITION, + QUERY_NODE_IS_NULL_CONDITION, + QUERY_NODE_FUNCTION, + QUERY_NODE_REAL_TABLE, + QUERY_NODE_TEMP_TABLE, + QUERY_NODE_JOIN_TABLE, + QUERY_NODE_GROUPING_SET, + QUERY_NODE_ORDER_BY_EXPR, + QUERY_NODE_STATE_WINDOW, + QUERY_NODE_SESSION_WINDOW, + QUERY_NODE_INTERVAL_WINDOW, + + QUERY_NODE_SET_OPERATOR, + QUERY_NODE_SELECT_STMT +} ENodeType; + +/** + * The first field of a node of any type is guaranteed to be the ENodeType. + * Hence the type of any node can be gotten by casting it to SNode. + */ +typedef struct SNode { + ENodeType type; +} SNode; + +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) + +typedef struct SDataType { + uint8_t type; + uint8_t precision; + uint8_t scale; + int32_t bytes; +} SDataType; + +typedef struct SExprNode { + ENodeType nodeType; + SDataType resType; + char aliasName[TSDB_COL_NAME_LEN]; +} SExprNode; + +typedef enum EColumnType { + COLUMN_TYPE_COLUMN = 1, + COLUMN_TYPE_TAG +} EColumnType; + +typedef struct SColumnNode { + SExprNode node; // QUERY_NODE_COLUMN + int16_t colId; + EColumnType colType; // column or tag + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; +} SColumnNode; + +typedef struct SValueNode { + SExprNode type; // QUERY_NODE_VALUE + char* literal; +} SValueNode; + +typedef enum EOperatorType { + // arithmetic operator + OP_TYPE_ADD = 1, + OP_TYPE_SUB, + OP_TYPE_MULTI, + OP_TYPE_DIV, + OP_TYPE_MOD, + + // comparison operator + OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_EQUAL, + OP_TYPE_LOWER_THAN, + OP_TYPE_LOWER_EQUAL, + OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, + OP_TYPE_IN, + OP_TYPE_NOT_IN, + OP_TYPE_LIKE, + OP_TYPE_NOT_LIKE, + OP_TYPE_MATCH, + OP_TYPE_NMATCH, + + // json operator + OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_CONTAINS +} EOperatorType; + +typedef struct SOperatorNode { + SExprNode type; // QUERY_NODE_OPERATOR + EOperatorType opType; + SNode* pLeft; + SNode* pRight; +} SOperatorNode; + +typedef enum ELogicConditionType { + LOGIC_COND_TYPE_AND, + LOGIC_COND_TYPE_OR, + LOGIC_COND_TYPE_NOT, +} ELogicConditionType; + +typedef struct SLogicConditionNode { + ENodeType type; // QUERY_NODE_LOGIC_CONDITION + ELogicConditionType condType; + SArray* pParameterList; +} SLogicConditionNode; + +typedef struct SIsNullCondNode { + ENodeType type; // QUERY_NODE_IS_NULL_CONDITION + SNode* pExpr; + bool isNot; +} SIsNullCondNode; + +typedef struct SFunctionNode { + SExprNode type; // QUERY_NODE_FUNCTION + char functionName[TSDB_FUNC_NAME_LEN]; + int32_t funcId; + SArray* pParameterList; // SNode +} SFunctionNode; + +typedef struct STableNode { + ENodeType type; + char tableName[TSDB_TABLE_NAME_LEN]; + char tableAliasName[TSDB_COL_NAME_LEN]; +} STableNode; + +typedef struct SRealTableNode { + STableNode type; // QUERY_NODE_REAL_TABLE + char dbName[TSDB_DB_NAME_LEN]; +} SRealTableNode; + +typedef struct STempTableNode { + STableNode type; // QUERY_NODE_TEMP_TABLE + SNode* pSubquery; +} STempTableNode; + +typedef enum EJoinType { + JOIN_TYPE_INNER = 1 +} EJoinType; + +typedef struct SJoinTableNode { + STableNode type; // QUERY_NODE_JOIN_TABLE + EJoinType joinType; + SNode* pLeft; + SNode* pRight; + SNode* pOnCond; +} SJoinTableNode; + +typedef enum EGroupingSetType { + GP_TYPE_NORMAL = 1 +} EGroupingSetType; + +typedef struct SGroupingSetNode { + ENodeType type; // QUERY_NODE_GROUPING_SET + EGroupingSetType groupingSetType; + SArray* pParameterList; +} SGroupingSetNode; + +typedef enum EOrder { + ORDER_ASC = 1, + ORDER_DESC +} EOrder; + +typedef enum ENullOrder { + NULL_ORDER_FIRST = 1, + NULL_ORDER_LAST +} ENullOrder; + +typedef struct SOrderByExprNode { + ENodeType type; // QUERY_NODE_ORDER_BY_EXPR + SNode* pExpr; + EOrder order; + ENullOrder nullOrder; +} SOrderByExprNode; + +typedef struct SLimitInfo { + uint64_t limit; + uint64_t offset; +} SLimitInfo; + +typedef struct SStateWindowNode { + ENodeType type; // QUERY_NODE_STATE_WINDOW + SNode* pCol; +} SStateWindowNode; + +typedef struct SSessionWindowNode { + ENodeType type; // QUERY_NODE_SESSION_WINDOW + int64_t gap; // gap between two session window(in microseconds) + SNode* pCol; +} SSessionWindowNode; + +typedef struct SIntervalWindowNode { + ENodeType type; // QUERY_NODE_INTERVAL_WINDOW + int64_t interval; + int64_t sliding; + int64_t offset; +} SIntervalWindowNode; + +typedef struct SSelectStmt { + ENodeType type; // QUERY_NODE_SELECT_STMT + bool isDistinct; + SArray* pProjectionList; // SNode + SNode* pFromTable; + SNode* pWhereCond; + SArray* pPartitionByList; // SNode + SNode* pWindowClause; + SArray* pGroupByList; // SGroupingSetNode + SArray* pOrderByList; // SOrderByExprNode + SLimitInfo limit; + SLimitInfo slimit; +} SSelectStmt; + +typedef enum ESetOperatorType { + SET_OP_TYPE_UNION_ALL = 1 +} ESetOperatorType; + +typedef struct SSetOperator { + ENodeType type; // QUERY_NODE_SET_OPERATOR + ESetOperatorType opType; + SNode* pLeft; + SNode* pRight; +} SSetOperator; + +typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); + +bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext); +bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext); + +bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext); + +bool nodeEqual(const SNode* a, const SNode* b); + +void cloneNode(const SNode* pNode); + +int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen); +int32_t stringToNode(const char* pStr, SNode** pNode); + +bool isTimeorderQuery(const SNode* pQuery); +bool isTimelineQuery(const SNode* pQuery); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_NODES_H_*/ diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 2833b329a7..fbf045b99c 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -3,4 +3,5 @@ add_subdirectory(util) add_subdirectory(common) add_subdirectory(libs) add_subdirectory(client) -add_subdirectory(dnode) \ No newline at end of file +add_subdirectory(dnode) +add_subdirectory(nodes) \ No newline at end of file diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index a4aa7025e4..9f700dbb3c 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -8,5 +8,5 @@ target_include_directories( target_link_libraries( function - PRIVATE os util common + PRIVATE os util common nodes ) \ No newline at end of file diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h new file mode 100644 index 0000000000..9b9d82f0e1 --- /dev/null +++ b/source/libs/function/inc/functionMgtInt.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_FUNCTION_MGT_INT_H_ +#define _TD_FUNCTION_MGT_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "functionMgt.h" + +#define FUNC_MGT_DATA_TYPE_MASK(n) (1 << n) + +#define FUNC_MGT_DATA_TYPE_NULL 0 +#define FUNC_MGT_DATA_TYPE_BOOL FUNC_MGT_DATA_TYPE_MASK(0) +#define FUNC_MGT_DATA_TYPE_TINYINT FUNC_MGT_DATA_TYPE_MASK(1) +#define FUNC_MGT_DATA_TYPE_SMALLINT FUNC_MGT_DATA_TYPE_MASK(2) +#define FUNC_MGT_DATA_TYPE_INT FUNC_MGT_DATA_TYPE_MASK(3) +#define FUNC_MGT_DATA_TYPE_BIGINT FUNC_MGT_DATA_TYPE_MASK(4) +#define FUNC_MGT_DATA_TYPE_FLOAT FUNC_MGT_DATA_TYPE_MASK(5) +#define FUNC_MGT_DATA_TYPE_DOUBLE FUNC_MGT_DATA_TYPE_MASK(6) +#define FUNC_MGT_DATA_TYPE_BINARY FUNC_MGT_DATA_TYPE_MASK(7) +#define FUNC_MGT_DATA_TYPE_TIMESTAMP FUNC_MGT_DATA_TYPE_MASK(8) +#define FUNC_MGT_DATA_TYPE_NCHAR FUNC_MGT_DATA_TYPE_MASK(9) +#define FUNC_MGT_DATA_TYPE_UTINYINT FUNC_MGT_DATA_TYPE_MASK(10) +#define FUNC_MGT_DATA_TYPE_USMALLINT FUNC_MGT_DATA_TYPE_MASK(11) +#define FUNC_MGT_DATA_TYPE_UINT FUNC_MGT_DATA_TYPE_MASK(12) +#define FUNC_MGT_DATA_TYPE_UBIGINT FUNC_MGT_DATA_TYPE_MASK(13) +#define FUNC_MGT_DATA_TYPE_VARCHAR FUNC_MGT_DATA_TYPE_MASK(14) +#define FUNC_MGT_DATA_TYPE_VARBINARY FUNC_MGT_DATA_TYPE_MASK(15) +#define FUNC_MGT_DATA_TYPE_JSON FUNC_MGT_DATA_TYPE_MASK(16) +#define FUNC_MGT_DATA_TYPE_DECIMAL FUNC_MGT_DATA_TYPE_MASK(17) +#define FUNC_MGT_DATA_TYPE_BLOB FUNC_MGT_DATA_TYPE_MASK(18) + +#define FUNC_MGT_EXACT_NUMERIC_DATA_TYPE \ + (FUNC_MGT_DATA_TYPE_TINYINT | FUNC_MGT_DATA_TYPE_SMALLINT | FUNC_MGT_DATA_TYPE_INT | FUNC_MGT_DATA_TYPE_BIGINT \ + | FUNC_MGT_DATA_TYPE_UTINYINT | FUNC_MGT_DATA_TYPE_USMALLINT | FUNC_MGT_DATA_TYPE_UINT | FUNC_MGT_DATA_TYPE_UBIGINT) + +#define FUNC_MGT_APPRO_NUMERIC_DATA_TYPE (FUNC_MGT_DATA_TYPE_FLOAT | FUNC_MGT_DATA_TYPE_DOUBLE) + +#define FUNC_MGT_NUMERIC_DATA_TYPE (FUNC_MGT_EXACT_NUMERIC_DATA_TYPE | FUNC_MGT_APPRO_NUMERIC_DATA_TYPE) + +typedef void* FuncDef; + +typedef struct SFuncElement { + FuncDef (*defineFunc)(); +} SFuncElement; + +extern const SFuncElement gBuiltinFuncs[]; + +FuncDef createFuncDef(const char* name, int32_t maxNumOfParams); +FuncDef setOneParamSignature(FuncDef def, int64_t resDataType, int64_t paramDataType); +FuncDef setTwoParamsSignature(FuncDef def, int64_t resDataType, int64_t p1DataType, int64_t p2DataType); +FuncDef setFollowParamSignature(FuncDef def, int64_t paramDataType); +FuncDef setFollowParamsSignature(FuncDef def, int64_t p1DataType, int64_t p2DataType, int32_t followNo); + +FuncDef setExecFuncs(FuncDef def, FExecGetEnv getEnv, FExecInit init, FExecProcess process, FExecFinalize finalize); + +#ifdef __cplusplus +} +#endif + +#endif // _TD_FUNCTION_MGT_INT_H_ diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 41c7309a18..9192c19951 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -95,6 +95,10 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32 memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); } +#include "functionMgtInt.h" + +FuncDef defineCount(); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/buildins.c b/source/libs/function/src/buildins.c new file mode 100644 index 0000000000..ea2e9f3f2f --- /dev/null +++ b/source/libs/function/src/buildins.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "functionMgtInt.h" +#include "taggfunction.h" + +const SFuncElement gBuiltinFuncs[] = { + {.defineFunc = defineCount} +}; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c new file mode 100644 index 0000000000..c6ed2c8c03 --- /dev/null +++ b/source/libs/function/src/functionMgt.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "functionMgt.h" + +#include "functionMgtInt.h" +#include "taos.h" +#include "taoserror.h" +#include "thash.h" + +typedef struct SFuncMgtService { + SHashObj* pFuncNameHashTable; +} SFuncMgtService; + +static SFuncMgtService gFunMgtService; + +int32_t fmFuncMgtInit() { + gFunMgtService.pFuncNameHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == gFunMgtService.pFuncNameHashTable) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +typedef struct SFuncDef { + char name[TSDB_FUNC_NAME_LEN]; + int32_t maxNumOfParams; + SFuncExecFuncs execFuncs; +} SFuncDef ; + +FuncDef createFuncDef(const char* name, int32_t maxNumOfParams) { + SFuncDef* pDef = calloc(1, sizeof(SFuncDef)); + if (NULL == pDef) { + return NULL; + } + strcpy(pDef->name, name); + pDef->maxNumOfParams = maxNumOfParams; + return pDef; +} + +FuncDef setOneParamSignature(FuncDef def, int64_t resDataType, int64_t paramDataType) { + // todo +} + +FuncDef setTwoParamsSignature(FuncDef def, int64_t resDataType, int64_t p1DataType, int64_t p2DataType) { + // todo +} + +FuncDef setFollowParamSignature(FuncDef def, int64_t paramDataType) { + // todo +} + +FuncDef setFollowParamsSignature(FuncDef def, int64_t p1DataType, int64_t p2DataType, int32_t followNo) { + // todo +} + +FuncDef setExecFuncs(FuncDef def, FExecGetEnv getEnv, FExecInit init, FExecProcess process, FExecFinalize finalize) { + SFuncDef* pDef = (SFuncDef*)def; + pDef->execFuncs.getEnv = getEnv; + pDef->execFuncs.init = init; + pDef->execFuncs.process = process; + pDef->execFuncs.finalize = finalize; + return def; +} + +int32_t registerFunc(FuncDef func) { + +} diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 3af4a8fe57..3b3edf1501 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -4835,3 +4835,9 @@ SAggFunctionInfo aggFunc[35] = {{ statisRequired, } }; + +FuncDef defineCount() { + FuncDef def = createFuncDef("count", 1); + // todo define signature + return setExecFuncs(def, NULL, function_setup, count_function, doFinalizer); +} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 93f72bba95..a0feaca72b 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -212,7 +212,7 @@ static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, SQ pExpr[i] = p; } - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); + // pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); tfree(pExpr); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index c7a4e438ba..04282a2f80 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -109,6 +109,26 @@ static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void* return func(jObj, *obj); } +static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func, SArray** array) { + const cJSON* jArray = cJSON_GetObjectItem(json, name); + int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + if (size > 0) { + *array = taosArrayInit(size, POINTER_BYTES); + if (NULL == *array) { + return false; + } + } + for (int32_t i = 0; i < size; ++i) { + cJSON* jItem = cJSON_GetArrayItem(jArray, i); + void* item = calloc(1, getPnodeTypeSize(jItem)); + if (NULL == item || !func(jItem, item)) { + return false; + } + taosArrayPush(*array, &item); + } + return true; +} + static bool addTarray(cJSON* json, const char* name, FToJson func, const SArray* array, bool isPoint) { size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); if (size > 0) { @@ -379,8 +399,8 @@ static const char* jkFunctionChild = "Child"; static bool functionToJson(const void* obj, cJSON* jFunc) { const tExprNode* exprInfo = (const tExprNode*)obj; bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName); - if (res) { - res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); + if (res && NULL != exprInfo->_function.pChild) { + res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, *(exprInfo->_function.pChild), sizeof(tExprNode*), exprInfo->_function.num); } return res; } @@ -388,6 +408,10 @@ static bool functionToJson(const void* obj, cJSON* jFunc) { static bool functionFromJson(const cJSON* json, void* obj) { tExprNode* exprInfo = (tExprNode*)obj; copyString(json, jkFunctionName, exprInfo->_function.functionName); + exprInfo->_function.pChild = calloc(1, sizeof(tExprNode*)); + if (NULL == exprInfo->_function.pChild) { + return false; + } return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num); } @@ -808,7 +832,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { case OP_SystemTableScan: return scanNodeFromJson(json, obj); case OP_Aggregate: - break; // todo + return aggNodeFromJson(json, obj); case OP_Project: return true; // case OP_Groupby: @@ -879,7 +903,7 @@ static bool phyNodeFromJson(const cJSON* json, void* obj) { res = fromObject(json, jkPnodeSchema, dataBlockSchemaFromJson, &node->targetSchema, true); } if (res) { - res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); + res = fromPnodeArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren); } if (res) { res = fromObject(json, node->info.name, specificPhyNodeFromJson, node, true); diff --git a/source/nodes/CMakeLists.txt b/source/nodes/CMakeLists.txt new file mode 100644 index 0000000000..b30534f3f2 --- /dev/null +++ b/source/nodes/CMakeLists.txt @@ -0,0 +1,15 @@ +aux_source_directory(src NODES_SRC) +add_library(nodes STATIC ${NODES_SRC}) +target_include_directories( + nodes + PUBLIC "${CMAKE_SOURCE_DIR}/include/nodes" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + nodes + PRIVATE os util +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/nodes/src/nodesClone.c b/source/nodes/src/nodesClone.c new file mode 100644 index 0000000000..04f6df5623 --- /dev/null +++ b/source/nodes/src/nodesClone.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +void cloneNode(const SNode* pNode) { + +} diff --git a/source/nodes/src/nodesCode.c b/source/nodes/src/nodesCode.c new file mode 100644 index 0000000000..7fe919ffe8 --- /dev/null +++ b/source/nodes/src/nodesCode.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { + +} + +int32_t stringToNode(const char* pStr, SNode** pNode) { + +} diff --git a/source/nodes/src/nodesEqual.c b/source/nodes/src/nodesEqual.c new file mode 100644 index 0000000000..bef025fbea --- /dev/null +++ b/source/nodes/src/nodesEqual.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +#define COMPARE_SCALAR_FIELD(fldname) \ + do { \ + if (a->fldname != b->fldname) \ + return false; \ + } while (0) + +#define COMPARE_STRING(a, b) \ + (((a) != NULL && (b) != NULL) ? (strcmp(a, b) == 0) : (a) == (b)) + +#define COMPARE_STRING_FIELD(fldname) \ + do { \ + if (!COMPARE_STRING(a->fldname, b->fldname)) \ + return false; \ + } while (0) + +#define COMPARE_NODE_FIELD(fldname) \ + do { \ + if (!nodeEqual(a->fldname, b->fldname)) \ + return false; \ + } while (0) + +#define COMPARE_ARRAY_FIELD(fldname) \ + do { \ + if (!nodeArrayEqual(a->fldname, b->fldname)) \ + return false; \ + } while (0) + +static bool nodeArrayEqual(const SArray* a, const SArray* b) { + if (a == b) { + return true; + } + + if (NULL == a || NULL == b) { + return false; + } + + if (taosArrayGetSize(a) != taosArrayGetSize(b)) { + return false; + } + + size_t size = taosArrayGetSize(a); + for (size_t i = 0; i < size; ++i) { + if (!nodeEqual((SNode*)taosArrayGetP(a, i), (SNode*)taosArrayGetP(b, i))) { + return false; + } + } + return true; +} + +static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { + COMPARE_STRING_FIELD(dbName); + COMPARE_STRING_FIELD(tableName); + COMPARE_STRING_FIELD(colName); + return true; +} + +static bool valueNodeEqual(const SValueNode* a, const SValueNode* b) { + COMPARE_STRING_FIELD(literal); + return true; +} + +static bool operatorNodeEqual(const SOperatorNode* a, const SOperatorNode* b) { + COMPARE_SCALAR_FIELD(opType); + COMPARE_NODE_FIELD(pLeft); + COMPARE_NODE_FIELD(pRight); + return true; +} + +static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicConditionNode* b) { + COMPARE_SCALAR_FIELD(condType); + COMPARE_ARRAY_FIELD(pParameterList); + return true; +} + +static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCondNode* b) { + COMPARE_NODE_FIELD(pExpr); + COMPARE_SCALAR_FIELD(isNot); + return true; +} + +static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { + COMPARE_SCALAR_FIELD(funcId); + COMPARE_ARRAY_FIELD(pParameterList); + return true; +} + +bool nodeEqual(const SNode* a, const SNode* b) { + if (a == b) { + return true; + } + + if (NULL == a || NULL == b) { + return false; + } + + if (nodeType(a) != nodeType(b)) { + return false; + } + + switch (nodeType(a)) { + case QUERY_NODE_COLUMN: + return columnNodeEqual((const SColumnNode*)a, (const SColumnNode*)b); + case QUERY_NODE_VALUE: + return valueNodeEqual((const SValueNode*)a, (const SValueNode*)b); + case QUERY_NODE_OPERATOR: + return operatorNodeEqual((const SOperatorNode*)a, (const SOperatorNode*)b); + case QUERY_NODE_LOGIC_CONDITION: + return logicConditionNodeEqual((const SLogicConditionNode*)a, (const SLogicConditionNode*)b); + case QUERY_NODE_IS_NULL_CONDITION: + return isNullConditionNodeEqual((const SIsNullCondNode*)a, (const SIsNullCondNode*)b); + case QUERY_NODE_FUNCTION: + return functionNodeEqual((const SFunctionNode*)a, (const SFunctionNode*)b); + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + case QUERY_NODE_JOIN_TABLE: + case QUERY_NODE_GROUPING_SET: + case QUERY_NODE_ORDER_BY_EXPR: + return false; // todo + default: + break; + } + + return false; +} diff --git a/source/nodes/src/nodesTraverse.c b/source/nodes/src/nodesTraverse.c new file mode 100644 index 0000000000..eac8288099 --- /dev/null +++ b/source/nodes/src/nodesTraverse.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); + +bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext) { + size_t size = taosArrayGetSize(pArray); + for (size_t i = 0; i < size; ++i) { + if (!nodeTreeWalker((SNode*)taosArrayGetP(pArray, i), walker, pContext)) { + return false; + } + } + return true; +} + +bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + if (NULL == pNode) { + return true; + } + + if (!walker(pNode, pContext)) { + return false; + } + + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + // these node types with no subnodes + return true; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + if (!nodeTreeWalker(pOpNode->pLeft, walker, pContext)) { + return false; + } + return nodeTreeWalker(pOpNode->pRight, walker, pContext); + } + case QUERY_NODE_LOGIC_CONDITION: + return nodeArrayWalker(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext); + case QUERY_NODE_IS_NULL_CONDITION: + return nodeTreeWalker(((SIsNullCondNode*)pNode)->pExpr, walker, pContext); + case QUERY_NODE_FUNCTION: + return nodeArrayWalker(((SFunctionNode*)pNode)->pParameterList, walker, pContext); + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + return true; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + if (!nodeTreeWalker(pJoinTableNode->pLeft, walker, pContext)) { + return false; + } + if (!nodeTreeWalker(pJoinTableNode->pRight, walker, pContext)) { + return false; + } + return nodeTreeWalker(pJoinTableNode->pOnCond, walker, pContext); + } + case QUERY_NODE_GROUPING_SET: + return nodeArrayWalker(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext); + case QUERY_NODE_ORDER_BY_EXPR: + return nodeTreeWalker(((SOrderByExprNode*)pNode)->pExpr, walker, pContext); + default: + break; + } + + return false; +} + +bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + +} diff --git a/source/nodes/src/nodesUtil.c b/source/nodes/src/nodesUtil.c new file mode 100644 index 0000000000..fe5883d809 --- /dev/null +++ b/source/nodes/src/nodesUtil.c @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +bool isTimeorderQuery(const SNode* pQuery) { + +} + +bool isTimelineQuery(const SNode* pQuery) { + +} diff --git a/source/nodes/test/CMakeLists.txt b/source/nodes/test/CMakeLists.txt new file mode 100644 index 0000000000..80725a79fb --- /dev/null +++ b/source/nodes/test/CMakeLists.txt @@ -0,0 +1,19 @@ + +MESSAGE(STATUS "build nodes unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(nodesTest ${SOURCE_LIST}) + +TARGET_INCLUDE_DIRECTORIES( + nodesTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/nodes/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/nodes/inc" +) + +TARGET_LINK_LIBRARIES( + nodesTest + PUBLIC os util common nodes gtest +) diff --git a/source/nodes/test/nodesTest.cpp b/source/nodes/test/nodesTest.cpp new file mode 100644 index 0000000000..7df3cd8b4c --- /dev/null +++ b/source/nodes/test/nodesTest.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +TEST(NodesTest, traverseTest) { + // todo +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From c978eb12ff5bbd901a4ce81c7346401bf82500c2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 20:33:52 -0800 Subject: [PATCH 067/103] TD-13101 --- source/dnode/mnode/impl/src/mndVgroup.c | 34 ++++--- source/libs/tfs/test/tfsTest.cpp | 45 +++++++++- tests/script/sim/db/basic1.sim | 114 ++++++++++++++++++++---- tests/script/sim/db/basic6.sim | 2 +- 4 files changed, 161 insertions(+), 34 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 93d6d104ff..0bed943b60 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -540,35 +540,41 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t cols = 0; char *pWrite; + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); if (pShow->pIter == NULL) break; - cols = 0; + if (pVgroup->dbUid == pDb->uid) { + cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pVgroup->vgId; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pVgroup->numOfTables; - cols++; - - for (int32_t i = 0; i < pShow->replica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; + *(int32_t *)pWrite = pVgroup->vgId; cols++; - const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + *(int32_t *)pWrite = pVgroup->numOfTables; cols++; + + for (int32_t i = 0; i < pShow->replica; ++i) { + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; + cols++; + + const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + cols++; + } + numOfRows++; } sdbRelease(pSdb, pVgroup); - numOfRows++; } + mndReleaseDb(pMnode, pDb); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 09f634bf4a..6d68a5d87a 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -60,6 +60,7 @@ TEST_F(TfsTest, 02_AllocDisk) { dCfg.level = 0; dCfg.primary = 1; + taosRemoveDir(root); taosMkDir(root); STfs *pTfs = tfsOpen(&dCfg, 1); ASSERT_NE(pTfs, nullptr); @@ -110,6 +111,7 @@ TEST_F(TfsTest, 03_Dir) { dCfg.level = 0; dCfg.primary = 1; + taosRemoveDir(root); taosMkDir(root); STfs *pTfs = tfsOpen(&dCfg, 1); ASSERT_NE(pTfs, nullptr); @@ -130,7 +132,7 @@ TEST_F(TfsTest, 03_Dir) { EXPECT_EQ(taosDirExist(ap2), 0); char p3[] = "p3/p2/p1/p0"; - char ap3[128] ={0}; + char ap3[128] = {0}; snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3); EXPECT_NE(taosDirExist(ap3), 0); EXPECT_NE(tfsMkdir(pTfs, p3), 0); @@ -155,4 +157,43 @@ TEST_F(TfsTest, 03_Dir) { EXPECT_NE(taosDirExist(ap4), 0); tfsClose(pTfs); -} \ No newline at end of file +} +#if 0 +TEST_F(TfsTest, 04_File) { + int32_t code = 0; + SDiskCfg dCfg = {0}; + tstrncpy(dCfg.dir, root, TSDB_FILENAME_LEN); + dCfg.level = 0; + dCfg.primary = 1; + + taosRemoveDir(root); + taosMkDir(root); + STfs *pTfs = tfsOpen(&dCfg, 1); + ASSERT_NE(pTfs, nullptr); + + STfsFile file0; + STfsFile file1; + STfsFile file2; + STfsFile file3; + STfsFile file4; + SDiskID did0 = {0}; + SDiskID did1 = {0}; + SDiskID did2 = {0}; + SDiskID did3 = {0}; + SDiskID did4 = {0}; + did3.id = 1; + did4.level = 1; + tfsInitFile(pTfs, &file0, did0, "fname"); + tfsInitFile(pTfs, &file1, did1, "fname"); + tfsInitFile(pTfs, &file2, did2, "fnamex"); + tfsInitFile(pTfs, &file3, did3, "fname"); + tfsInitFile(pTfs, &file4, did4, "fname"); + + EXPECT_TRUE(tfsIsSameFile(&file0, &file1)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file2)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file3)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file4)); + + tfsClose(pTfs); +} +#endif \ No newline at end of file diff --git a/tests/script/sim/db/basic1.sim b/tests/script/sim/db/basic1.sim index 33af1c5b59..67f88575c9 100644 --- a/tests/script/sim/db/basic1.sim +++ b/tests/script/sim/db/basic1.sim @@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create database -sql create database d1 +sql create database d1 vgroups 2 sql show databases if $rows != 1 then return -1 @@ -22,6 +22,21 @@ if $data03 != 0 then return -1 endi +print =============== show vgroups1 +sql use d1 +sql show vgroups +if $rows != 2 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi + +if $data10 != 3 then + return -1 +endi + print =============== drop database sql drop database d1 sql show databases @@ -30,14 +45,68 @@ if $rows != 0 then endi print =============== more databases -sql create database d2 -sql create database d3 -sql create database d4 +sql create database d2 vgroups 2 +sql create database d3 vgroups 3 +sql create database d4 vgroups 4 sql show databases if $rows != 3 then return -1 endi +print =============== show vgroups2 +sql show d2.vgroups +if $rows != 2 then + return -1 +endi + +if $data00 != 4 then + return -1 +endi + +if $data10 != 5 then + return -1 +endi + +print =============== show vgroups3 +sql show d3.vgroups +if $rows != 3 then + return -1 +endi + +if $data00 != 6 then + return -1 +endi + +if $data10 != 7 then + return -1 +endi + +if $data20 != 8 then + return -1 +endi + +print =============== show vgroups4 +sql show d4.vgroups +if $rows != 4 then + return -1 +endi + +if $data00 != 9 then + return -1 +endi + +if $data10 != 10 then + return -1 +endi + +if $data20 != 11 then + return -1 +endi + +if $data30 != 12 then + return -1 +endi + print =============== drop database sql drop database d2 sql drop database d3 @@ -50,7 +119,7 @@ if $data00 != d4 then return -1 endi -if $data02 != 2 then +if $data02 != 4 then return -1 endi @@ -58,19 +127,12 @@ if $data03 != 0 then return -1 endi -print =============== show vgroups -sql show databases - -if $rows != 1 then - return -1 -endi - +print =============== show vgroups4 again sql_error use d1 sql use d4 sql show vgroups - -if $rows != 2 then +if $rows != 4 then return -1 endi @@ -81,15 +143,16 @@ if $data00 != 1 then return -1 endi -if $data02 != 2 then +if $data02 != 4 then return -1 endi +print =============== restart + system sh/exec.sh -n dnode1 -s stop -x SIGKILL system sh/exec.sh -n dnode1 -s start sql show databases - if $rows != 1 then return -1 endi @@ -99,7 +162,24 @@ sql_error use d1 sql use d4 sql show vgroups -if $rows != 2 then +if $rows != 4 then + return -1 +endi + +sql create database d5 vgroups 5; +sql use d5 +sql show vgroups +if $rows != 5 then + return -1 +endi + +sql show d4.vgroups +if $rows != 4 then + return -1 +endi + +sql show d5.vgroups +if $rows != 5 then return -1 endi diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index a688b4c2f3..1ad8499f92 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -94,8 +94,8 @@ sql use $db sql create table st (ts timestamp, i int) tags (j int) sql create table $tb using st tags(1) -return system sh/exec.sh -n dnode1 -s stop -x SIGINT +return sql show tables if $rows != 1 then From 1a1cd551113c8e259cfef902eabce97bc8edda3f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Jan 2022 13:52:46 +0800 Subject: [PATCH 068/103] [td-11818] refactor. --- include/libs/function/function.h | 7 + source/client/src/clientImpl.c | 5 +- source/client/test/clientTests.cpp | 1126 ++++++++++---------- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndStb.c | 4 + source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/executorimpl.h | 9 +- source/libs/executor/src/executil.c | 9 +- source/libs/executor/src/executorimpl.c | 190 +++- source/libs/function/inc/taggfunction.h | 7 - source/libs/parser/src/astValidate.c | 3 +- source/libs/parser/src/dCDAstProcess.c | 27 +- source/libs/parser/src/parserUtil.c | 5 + source/libs/planner/src/logicPlan.c | 22 +- source/libs/planner/src/physicalPlanJson.c | 4 +- source/libs/planner/src/planner.c | 6 +- source/libs/scheduler/src/scheduler.c | 2 +- 17 files changed, 823 insertions(+), 607 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index bf2937a220..ad1ccce58b 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -80,6 +80,13 @@ extern "C" { #define FUNCTION_COV 38 +typedef struct SResultRowEntryInfo { + int8_t hasResult; // result generated, not NULL value + bool initialized; // output buffer has been initialized + bool complete; // query has completed + uint32_t numOfRes; // num of output result in current buffer +} SResultRowEntryInfo; + // determine the real data need to calculated the result enum { BLK_DATA_NO_NEEDED = 0x0, diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 75b652e40d..34ab1fb05a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -259,7 +259,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr)); SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 2}; - addr.epAddr[0].port = 6030; + addr.epAddr[0].port = 7100; strcpy(addr.epAddr[0].fqdn, "localhost"); taosArrayPush(execNode, &addr); @@ -728,6 +728,7 @@ void* doFetchRow(SRequestObj* pRequest) { goto _return; } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; + epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -772,6 +773,8 @@ void* doFetchRow(SRequestObj* pRequest) { tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE && pResultInfo->pData != NULL) { + return NULL; } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 4361131f01..0dcdeb2044 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -46,481 +47,484 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -TEST(testCase, driverInit_Test) { taos_init(); } +TEST(testCase, driverInit_Test) { + taosInitGlobalCfg(); +// taos_init(); +} TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } - sleep(3); taos_close(pConn); } -//TEST(testCase, create_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show users"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show databases"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, create_db_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); +TEST(testCase, create_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); + taos_free_result(pRes); + taos_close(pConn); +} - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //taos_free_result(pRes); + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } - //pRes = taos_query(pConn, "create database abc1 vgroups 4"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_close(pConn); -//} -// -//TEST(testCase, create_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); -// if (taos_errno(pRes) != 0) { -// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// taos_close(pConn); -//} -// -//TEST(testCase, drop_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// pRes = taos_query(pConn, "drop dnode 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, use_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_close(pConn); -//} -// -// TEST(testCase, drop_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// taos_close(pConn); -//} + taos_free_result(pRes); + taos_close(pConn); +} -//TEST(testCase, create_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// pRes = taos_query(pConn, "drop stable `123_$^)`"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_close(pConn); -//} +TEST(testCase, drop_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -//TEST(testCase, create_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); -// ASSERT_EQ(taos_errno(pRes), 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); -// ASSERT_NE(taos_errno(pRes), 0); -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } -//TEST(testCase, create_ctable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + taos_free_result(pRes); + taos_close(pConn); +} -//TEST(testCase, show_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != nullptr); -// -//// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -// -// TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_vgroup_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show vgroups"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_multiple_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// taos_close(pConn); -// return; -// } -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// -// for (int32_t i = 0; i < 20; ++i) { -// char sql[512] = {0}; -// snprintf(sql, tListLen(sql), -// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, -// (i + 1) * 30, (i + 2) * 40); -// TAOS_RES* pres = taos_query(pConn, sql); -// if (taos_errno(pres) != 0) { -// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); -// } -// taos_free_result(pres); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, show_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show tables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in using db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop stable st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, generated_request_id_test) { -// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); -// -// for (int32_t i = 0; i < 50000; ++i) { -// uint64_t v = generateRequestId(); -// void* result = taosHashGet(phash, &v, sizeof(v)); -// if (result != nullptr) { -// printf("0x%lx, index:%d\n", v, i); -// } -// assert(result == nullptr); -// taosHashPut(phash, &v, sizeof(v), NULL, 0); -// } -// -// taosHashCleanup(phash); -//} +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_close(pConn); +} + +TEST(testCase, create_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); + if (taos_errno(pRes) != 0) { + printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); + if (taos_errno(pRes) != 0) { + printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + taos_close(pConn); +} + +TEST(testCase, drop_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + pRes = taos_query(pConn, "drop dnode 4"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, use_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + + TEST(testCase, drop_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + showDB(pConn); + + TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + showDB(pConn); + + pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("create to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + taos_close(pConn); +} // +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + pRes = taos_query(pConn, "drop stable `123_$^)`"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + taos_close(pConn); +} + +TEST(testCase, create_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + ASSERT_EQ(taos_errno(pRes), 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + ASSERT_NE(taos_errno(pRes), 0); + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); + + TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); + if (taos_errno(pRes) != 0) { + printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_vgroup_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show vgroups"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + taos_close(pConn); + return; + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + + for (int32_t i = 0; i < 20; ++i) { + char sql[512] = {0}; + snprintf(sql, tListLen(sql), + "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, + (i + 1) * 30, (i + 2) * 40); + TAOS_RES* pres = taos_query(pConn, sql); + if (taos_errno(pres) != 0) { + printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); + } + taos_free_result(pres); + } + + taos_close(pConn); +} + +TEST(testCase, show_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "show abc1.tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + int32_t count = 0; + char str[512] = {0}; + + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%d: %s\n", ++count, str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("error in creating db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in using db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop stable st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, generated_request_id_test) { + SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + for (int32_t i = 0; i < 50000; ++i) { + uint64_t v = generateRequestId(); + void* result = taosHashGet(phash, &v, sizeof(v)); + if (result != nullptr) { + printf("0x%lx, index:%d\n", v, i); + } + assert(result == nullptr); + taosHashPut(phash, &v, sizeof(v), NULL, 0); + } + + taosHashCleanup(phash); +} + TEST(testCase, create_topic_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -545,99 +549,127 @@ TEST(testCase, create_topic_Test) { taos_close(pConn); } -//TEST(testCase, insert_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, insert_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); -//TEST(testCase, projection_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -//// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// for(int32_t i = 0; i < 100; ++i) { -//// char sql[512] = {0}; -//// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); -//// TAOS_RES* p = taos_query(pConn, sql); -//// if (taos_errno(p) != 0) { -//// printf("failed to insert data, reason:%s\n", taos_errstr(p)); -//// } -//// -//// taos_free_result(p); -//// } -// -// pRes = taos_query(pConn, "select * from tu"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); -//TEST(testCase, projection_query_stables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "select ts from m1"); + pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, projection_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + +// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); // if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); +// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); // } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// // taos_free_result(pRes); -// taos_close(pConn); -//} +// +// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// for(int32_t i = 0; i < 100; ++i) { +// char sql[512] = {0}; +// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); +// TAOS_RES* p = taos_query(pConn, sql); +// if (taos_errno(p) != 0) { +// printf("failed to insert data, reason:%s\n", taos_errstr(p)); +// } +// +// taos_free_result(p); +// } + + pRes = taos_query(pConn, "select * from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, projection_query_stables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "select ts from m1"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, agg_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use db"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "select count(*) from st_t1"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 45759117b5..328ff99bb8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -230,7 +230,7 @@ typedef struct { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int64_t uid; + uint64_t uid; int32_t cfgVersion; int32_t vgVersion; int8_t hashMethod; // default is 1 diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a6fd2a3c58..d61ef8e856 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -855,6 +855,10 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pShow->pIter == NULL) break; if (pStb->dbUid != pDb->uid) { + if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { + mError("Inconsistent table data, name:%s, db:%s, dbUid:%"PRIu64, pStb->name, pDb->name, pDb->uid); + } + sdbRelease(pSdb, pStb); continue; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 79766b22ac..f552afcfbf 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -94,7 +94,7 @@ struct SUdfInfo; int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); -size_t getResultRowSize(struct STaskRuntimeEnv* pRuntimeEnv); +size_t getResultRowSize(SArray* pExprInfo); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index beb26572d0..5d8b8d04c0 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -419,6 +419,7 @@ typedef struct SOptrBasicInfo { int32_t *rowCellInfoOffset; // offset value for each row result cell info SQLFunctionCtx *pCtx; SSDataBlock *pRes; + void *keyBuf; } SOptrBasicInfo; typedef struct SOptrBasicInfo STableIntervalOperatorInfo; @@ -426,6 +427,12 @@ typedef struct SOptrBasicInfo STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { SOptrBasicInfo binfo; uint32_t seed; + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. } SAggOperatorInfo; typedef struct SProjectOperatorInfo { @@ -554,7 +561,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5253add5af..7e53149c7c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -168,16 +168,15 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_ } // TODO refactor: use macro -struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { +SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { assert(index >= 0 && offset != NULL); // return (SResultRowEntryInfo*)((char*) pRow->pCellInfo + offset[index]); return NULL; } -size_t getResultRowSize(STaskRuntimeEnv* pRuntimeEnv) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - return 0; -// return (pQueryAttr->numOfOutput * sizeof(SResultRowEntryInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); +size_t getResultRowSize(SArray* pExprInfo) { + size_t numOfOutput = taosArrayGetSize(pExprInfo); + return (numOfOutput * sizeof(SResultRowEntryInfo)) + /*pQueryAttr->interBufSize +*/ sizeof(SResultRow); } SResultRowPool* initResultRowPool(size_t size) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 27c24669cc..164b8ed9f7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -309,6 +309,31 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO return res; } +SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { + const static int32_t minSize = 8; + + size_t numOfOutput = taosArrayGetSize(pExprInfo); + + SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = numOfOutput; + res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {{0}}; + SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + + idata.info.type = pExpr->base.resSchema.type; + idata.info.bytes = pExpr->base.resSchema.bytes; + idata.info.colId = pExpr->base.resSchema.colId; + + int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform + taosArrayPush(res->pDataBlock, &idata); + } + + return res; +} + void* destroyOutputBuf(SSDataBlock* pBlock) { if (pBlock == NULL) { return NULL; @@ -369,7 +394,7 @@ static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { return true; } -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, STaskRuntimeEnv* pRuntimeEnv) { +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) { // more than the capacity, reallocate the resources if (pResultRowInfo->size < pResultRowInfo->capacity) { return; @@ -384,7 +409,7 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, STaskRuntime char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); } pResultRowInfo->pResult = (SResultRow **)t; @@ -473,7 +498,7 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR } if (!existed) { - prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); +// prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); SResultRow *pResult = NULL; if (p1 == NULL) { @@ -507,6 +532,80 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR return pResultRowInfo->pResult[pResultRowInfo->curPos]; } +static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, + bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggOperatorInfo* pAggInfo) { + bool existed = false; + SET_RES_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tableGroupId); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (isIntervalQuery) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + if (pResultRowInfo->size == 0) { + existed = false; + assert(pResultRowInfo->curPos == -1); + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + pResultRowInfo->curPos = 0; + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); + int64_t* index = taosHashGet(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (index != NULL) { + pResultRowInfo->curPos = (int32_t) *index; + existed = true; + } else { + existed = false; + } + } + } + } else { + // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + if (p1 != NULL) { + return *p1; + } + } + + if (!existed) { + prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); + + SResultRow *pResult = NULL; + if (p1 == NULL) { + pResult = getNewResultRow(pAggInfo->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; + taosArrayPush(pAggInfo->pResultRowArrayList, &cell); + } else { + pResult = *p1; + } + + pResultRowInfo->curPos = pResultRowInfo->size; + pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + + int64_t index = pResultRowInfo->curPos; + SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); + taosHashPut(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + } + + // too many time window in query + if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + + return pResultRowInfo->pResult[pResultRowInfo->curPos]; +} + static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); @@ -1918,7 +2017,9 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { +static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { + size_t numOfOutput = taosArrayGetSize(pExprInfo); + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); if (pFuncCtx == NULL) { return NULL; @@ -1931,7 +2032,9 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExpr, int32_t numOfOu } for (int32_t i = 0; i < numOfOutput; ++i) { - SSqlExpr *pSqlExpr = &pExpr[i].base; + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + + SSqlExpr *pSqlExpr = &pExpr->base; SQLFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2044,7 +2147,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); - pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); +// pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); @@ -3457,6 +3560,43 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } +void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t stage, SExecTaskInfo* pTaskInfo) { + SOptrBasicInfo *pInfo = &pAggInfo->binfo; + + SQLFunctionCtx* pCtx = pInfo->pCtx; + SSDataBlock* pDataBlock = pInfo->pRes; + int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + + int64_t tid = 0; + pInfo->keyBuf = realloc(pInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); + SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid, pTaskInfo, false, pAggInfo); + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); + cleanupResultRowEntry(pEntry); + + pCtx[i].resultInfo = pEntry; + pCtx[i].pOutput = pData->pData; + pCtx[i].currentStage = stage; + assert(pCtx[i].pOutput != NULL); + + // set the timestamp output buffer for top/bottom/diff query + int32_t fid = pCtx[i].functionId; + if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { + if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + } + + initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); +} + void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { SSDataBlock* pDataBlock = pBInfo->pRes; @@ -6516,19 +6656,31 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); -// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfRows = 1;//(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + size_t numOfOutput = taosArrayGetSize(pExprInfo); + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); + + pInfo->pResultRowHashTable = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pInfo->pResultRowListSet = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pInfo->keyBuf = NULL;//malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); + pInfo->pool = initResultRowPool(getResultRowSize(pExprInfo)); + pInfo->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); pInfo->seed = rand(); -// setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN); + setDefaultOutputBuf_rv(pInfo, pInfo->seed, MAIN_SCAN, pTaskInfo); + + SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + memcpy(&p[i], pExpr, sizeof(SExprInfo)); + } SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; @@ -6536,10 +6688,11 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = pExpr; + pOperator->pExpr = p; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = NULL; + pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; pOperator->cleanup = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); @@ -7485,8 +7638,17 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); - } else { - assert(0); + } + } + + if (pPhyNode->info.type == OP_Aggregate) { + size_t size = taosArrayGetSize(pPhyNode->pChildren); + assert(size == 1); + + for (int32_t i = 0; i < size; ++i) { + SPhyNode* pChildNode = taosArrayGet(pPhyNode->pChildren, i); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, param); + return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); } } } diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 41c7309a18..c1c6d7c2fa 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -30,13 +30,6 @@ extern "C" { extern SAggFunctionInfo aggFunc[35]; -typedef struct SResultRowEntryInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - uint32_t numOfRes; // num of output result in current buffer -} SResultRowEntryInfo; - #define FUNCSTATE_SO 0x0u #define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM #define FUNCSTATE_STREAM 0x2u // function avail for stream diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 512204b40b..8327b7c131 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2080,7 +2080,7 @@ static int32_t setColumnIndex(SQueryStmtInfo* pQueryInfo, SArray* pParamList, SC STableMeta* pTableMeta = getMetaInfo(pQueryInfo, 0)->pTableMeta; if (pParamList == NULL) { // count(*) is equalled to count(primary_timestamp_key) - *index = (SColumnIndex) {0, PRIMARYKEY_TIMESTAMP_COL_ID, false}; + *index = (SColumnIndex) {0, 0, false}; *columnSchema = *(SSchema*) getOneColumnSchema(pTableMeta, index->columnIndex); } else { tSqlExprItem* pParamElem = taosArrayGet(pParamList, 0); @@ -3955,6 +3955,7 @@ int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtI pQueryInfo->pTableMetaInfo[0]->name = *name; pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo[0]->tagColList = taosArrayInit(4, POINTER_BYTES); + strcpy(pQueryInfo->pTableMetaInfo[0]->aliasName, name->tname); code = setTableVgroupList(pCtx, name, &pQueryInfo->pTableMetaInfo[0]->vgroupList); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 3b7b9e44d9..662e550655 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -26,18 +26,26 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out const char* msg4 = "pattern is invalid"; const char* msg5 = "database name is empty"; const char* msg6 = "pattern string is empty"; - + const char* msg7 = "db is not specified"; /* * database prefix in pInfo->pMiscInfo->a[0] * wildcard in like clause in pInfo->pMiscInfo->a[1] */ int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE) { - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); - SArray* array = NULL; SName name = {0}; - tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + + if (pCtx->db == NULL && pShowInfo->prefix.n == 0) { + return buildInvalidOperationMsg(pMsgBuf, msg7); + } + + SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + if (pShowInfo->prefix.n > 0) { + tNameSetDbName(&name, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); + } else { + tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + } char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); @@ -715,6 +723,8 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf* pMsgBuf = &m; + pDcl->epSet = pCtx->mgmtEpSet; + switch (pInfo->type) { case TSDB_SQL_CREATE_USER: case TSDB_SQL_ALTER_USER: { @@ -768,7 +778,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)buildUserManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_USER) ? TDMT_MND_CREATE_USER : TDMT_MND_ALTER_USER; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -810,7 +819,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)buildAcctManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_ACCT) ? TDMT_MND_CREATE_ACCT : TDMT_MND_ALTER_ACCT; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -818,7 +826,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch case TSDB_SQL_DROP_USER: { pDcl->pMsg = (char*)buildDropUserMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen); pDcl->msgType = (pInfo->type == TSDB_SQL_DROP_ACCT) ? TDMT_MND_DROP_ACCT : TDMT_MND_DROP_USER; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -855,7 +862,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)pUseDbMsg; pDcl->msgLen = sizeof(SUseDbReq); pDcl->msgType = TDMT_MND_USE_DB; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -884,7 +890,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - pDcl->epSet = pCtx->mgmtEpSet; pDcl->pMsg = (char*)pCreateMsg; pDcl->msgLen = sizeof(SCreateDbReq); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB) ? TDMT_MND_CREATE_DB : TDMT_MND_ALTER_DB; @@ -913,7 +918,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->msgType = TDMT_MND_DROP_DB; pDcl->msgLen = sizeof(SDropDbReq); pDcl->pMsg = (char*)pDropDbMsg; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -926,7 +930,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch pDcl->pMsg = (char*)buildCreateStbMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); pDcl->msgType = TDMT_MND_CREATE_STB; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -947,7 +950,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch } pDcl->msgType = TDMT_MND_CREATE_DNODE; - pDcl->epSet = pCtx->mgmtEpSet; break; } @@ -958,7 +960,6 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch } pDcl->msgType = TDMT_MND_DROP_DNODE; - pDcl->epSet = pCtx->mgmtEpSet; break; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 4d506b84a0..1793c85e30 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1947,6 +1947,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) { int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; const char* msg2 = "invalid database name"; + const char* msg3 = "db is not specified"; int32_t code = TSDB_CODE_SUCCESS; char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, true); @@ -1984,6 +1985,10 @@ int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, strncpy(name, pTableName->z, pTableName->n); strdequote(name); + if (pParseCtx->db == NULL) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + code = tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); if (code != TSDB_CODE_SUCCESS) { code = buildInvalidOperationMsg(pMsgBuf, msg2); diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index ae058c1f85..00a099bb4b 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -197,23 +197,25 @@ static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, SQ SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); if (!pQueryInfo->info.projectionQuery) { + SArray* p = pQueryInfo->exprList[0]; STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); + int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); + + pNode->numOfExpr = numOfCols; + pNode->pExpr = taosArrayInit(numOfCols, POINTER_BYTES); + for(int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(p, i); + SColumn* pCol = pExprInfo->base.pColumns; SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, pCol); - SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); - pExpr[i] = p; - } + SSchema schema = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); - tfree(pExpr); + SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &schema, 0); + taosArrayPush(pNode->pExpr, &p); + } } return pNode; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index db1b1c84d1..f77e5c0c2b 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -443,7 +443,7 @@ static const char* jkExprNodeColumn = "Column"; static const char* jkExprNodeValue = "Value"; static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) { - const tExprNode* exprInfo = (const tExprNode*)obj; + const tExprNode* exprInfo = *(const tExprNode**)obj; bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType); if (res) { switch (exprInfo->nodeType) { @@ -531,7 +531,7 @@ static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) { const SExprInfo* exprInfo = (const SExprInfo*)obj; bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base); if (res) { - res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, exprInfo->pExpr); + res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, &exprInfo->pExpr); } return res; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 9b32213ad7..6b3f37741e 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 45c6936b62..b016dc6124 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1216,7 +1216,7 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_JRET(code); } - printf("physical plan:%s\n", pTask->msg); +// printf("physical plan:%s\n", pTask->msg); } SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); From 359d8427fc1ddc3f76f4b6610558b690dbcf37a5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 21:52:54 -0800 Subject: [PATCH 069/103] minor changes --- source/dnode/mnode/impl/src/mndSync.c | 5 ++++- tests/script/sim/db/basic1.sim | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 591367c519..035c05c286 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -92,6 +92,10 @@ static int32_t mndRestoreWal(SMnode *pMnode) { goto WAL_RESTORE_OVER; } + if (walCommit(pWal, sdbVer) != 0) { + goto WAL_RESTORE_OVER; + } + if (walBeginSnapshot(pWal, sdbVer) < 0) { goto WAL_RESTORE_OVER; } @@ -99,7 +103,6 @@ static int32_t mndRestoreWal(SMnode *pMnode) { if (walEndSnapshot(pWal) < 0) { goto WAL_RESTORE_OVER; } - } code = 0; diff --git a/tests/script/sim/db/basic1.sim b/tests/script/sim/db/basic1.sim index 67f88575c9..21e26784bf 100644 --- a/tests/script/sim/db/basic1.sim +++ b/tests/script/sim/db/basic1.sim @@ -152,6 +152,7 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGKILL system sh/exec.sh -n dnode1 -s start +print =============== show databases sql show databases if $rows != 1 then return -1 @@ -166,7 +167,10 @@ if $rows != 4 then return -1 endi +print =============== create databases sql create database d5 vgroups 5; + +print =============== show vgroups sql use d5 sql show vgroups if $rows != 5 then From fd6ec6d33b494601b3084558bceab902600f7cb1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 22:27:13 -0800 Subject: [PATCH 070/103] minor changes --- source/dnode/mnode/impl/src/mndStb.c | 8 +- tests/script/sim/db/basic6.sim | 154 ++++++++++++++++++++++++--- 2 files changed, 142 insertions(+), 20 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index a6fd2a3c58..a651142358 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -839,14 +839,12 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 SStbObj *pStb = NULL; int32_t cols = 0; char *pWrite; - char prefix[64] = {0}; + char prefix[TSDB_DB_FNAME_LEN] = {0}; SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) { - return TSDB_CODE_MND_INVALID_DB; - } + if (pDb == NULL) return 0; - tstrncpy(prefix, pShow->db, 64); + tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = (int32_t)strlen(prefix); diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index 1ad8499f92..18ac7ffeb1 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -7,8 +7,10 @@ print ============================ dnode1 start $i = 0 $dbPrefix = db +$stPrefix = st $tbPrefix = tb $db = $dbPrefix . $i +$st = $stPrefix . $i $tb = $tbPrefix . $i print =============== step1 @@ -68,15 +70,51 @@ if $data06 != 15 then return -1 endi +return + print =============== step6 $i = $i + 1 while $i < 5 $db = $dbPrefix . $i + $st = $stPrefix . $i + $tb = $tbPrefix . $i + + print create database $db sql create database $db + + print use $db sql use $db + + print create table $st (ts timestamp, i int) tags (j int) + sql create table $st (ts timestamp, i int) tags (j int) + + print create table $tb using $st tags(1) + sql create table $tb using $st tags(1) + + sql show stables + if $rows != 1 then + return -1 + endi + + print $data00 $data01 $data02 $data03 + if $data00 != $st then + return -1 + endi + + sql show tables + if $rows != 1 then + return -1 + endi + + print $data00 $data01 $data02 $data03 + if $data00 != $tb then + return -1 + endi + $i = $i + 1 endw +return print =============== step7 $i = 0 while $i < 5 @@ -86,93 +124,179 @@ while $i < 5 endw print =============== step8 -$i = 0 +$i = 1 $db = $dbPrefix . $i +$st = $stPrefix . $i $tb = $tbPrefix . $i sql create database $db sql use $db -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) -system sh/exec.sh -n dnode1 -s stop -x SIGINT -return +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi sql show tables if $rows != 1 then return -1 endi +if $data00 != $tb then + return -1 +endi + print =============== step9 sql drop database $db print =============== step10 sql create database $db sql use $db + +sql show stables +if $rows != 0 then + return -1 +endi + sql show tables if $rows != 0 then return -1 endi print =============== step11 -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) + +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi + sql show tables if $rows != 1 then return -1 endi +if $data00 != $tb then + return -1 +endi + print =============== step12 sql drop database $db print =============== step13 sql create database $db sql use $db + +sql show stables +if $rows != 0 then + return -1 +endi + sql show tables if $rows != 0 then return -1 endi -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +print ============== step14 +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) + +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi + sql show tables if $rows != 1 then return -1 endi +if $data00 != $tb then + return -1 +endi + sql insert into $tb values (now+1a, 0) sql insert into $tb values (now+2a, 1) sql insert into $tb values (now+3a, 2) sql insert into $tb values (now+4a, 3) sql insert into $tb values (now+5a, 4) - -return - sql select * from $tb if $rows != 5 then return -1 endi +sql select * from $stb +if $rows != 5 then + return -1 +endi + print =============== step14 sql drop database $db print =============== step15 sql create database $db sql use $db + +sql show stables +if $rows != 0 then + return -1 +endi + sql show tables if $rows != 0 then return -1 endi print =============== step16 -sql create table st (ts timestamp, i int) tags (j int) -sql create table $tb using st tags(1) +sql create table $st (ts timestamp, i int) tags (j int) +sql create table $tb using $st tags(1) + +sql show stables +if $rows != 1 then + return -1 +endi + +if $data00 != $st then + return -1 +endi + sql show tables if $rows != 1 then return -1 endi + +if $data00 != $tb then + return -1 +endi + +sql insert into $tb values (now+1a, 0) +sql insert into $tb values (now+2a, 1) +sql insert into $tb values (now+3a, 2) +sql insert into $tb values (now+4a, 3) +sql insert into $tb values (now+5a, 4) + sql select * from $tb -if $rows != 0 then +if $rows != 5 then + return -1 +endi + +sql select * from $stb +if $rows != 5 then return -1 endi From 6ceb6117241c511dbf5dc49851d1e53ad7827e34 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Jan 2022 22:37:19 -0800 Subject: [PATCH 071/103] minor changes --- tests/script/sim/dnode/basic1.sim | 95 ++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/sim/dnode/basic1.sim index 49b29a4ac8..94b29e877e 100644 --- a/tests/script/sim/dnode/basic1.sim +++ b/tests/script/sim/dnode/basic1.sim @@ -78,8 +78,8 @@ if $data02 != master then return -1 endi -print =============== create table -sql create database d1; +print =============== create database +sql create database d1 vgroups 4; sql create database d2; sql show databases @@ -90,10 +90,99 @@ endi sql use d1 sql show vgroups; -if $rows != 2 then +if $rows != 4 then return -1 endi +print =============== create table +sql use d1 + +sql create table st (ts timestamp, i int) tags (j int) +sql show stables +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +sql create table c1 using st tags(1) +sql create table c2 using st tags(2) +sql create table c3 using st tags(2) +sql create table c4 using st tags(2) +sql create table c5 using st tags(2) + +sql show tables +print $data00 $data01 $data02 + +#if $rows != 5 then +# return -1 +#endi + +print =============== insert data +sql insert into c1 values(now+1s, 1) +sql insert into c1 values(now+2s, 2) +sql insert into c1 values(now+3s, 3) +sql insert into c2 values(now+1s, 1) +sql insert into c2 values(now+2s, 2) +sql insert into c2 values(now+3s, 3) +sql insert into c3 values(now+1s, 1) +sql insert into c3 values(now+2s, 2) +sql insert into c3 values(now+3s, 3) +sql insert into c4 values(now+1s, 1) +sql insert into c4 values(now+2s, 2) +sql insert into c4 values(now+3s, 3) +sql insert into c5 values(now+1s, 1) +sql insert into c5 values(now+2s, 2) +sql insert into c5 values(now+3s, 3) + +print =============== query data +sql select * from c1 +if $rows != 3 then + return -1 +endi + +print $data00 $data01 +print $data10 $data11 +print $data20 $data11 + +if $data01 != 1 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data21 != 3 then + return -1 +endi + +sql select * from c2 +if $rows != 3 then + return -1 +endi + +sql select * from c3 +if $rows != 3 then + return -1 +endi + +sql select * from c4 +if $rows != 3 then + return -1 +endi + +sql select * from c5 +if $rows != 3 then + return -1 +endi + +sql select * from st +if $rows != 15 then + return -1 +endi + +return print =============== drop dnode sql drop dnode 2; sql show dnodes; From f7726ce8a437ffbb371a743945e92613d8d02a60 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 14:38:55 +0800 Subject: [PATCH 072/103] add tq process consumer msg --- include/common/tmsg.h | 9 +++++- include/common/tmsgdef.h | 2 +- include/libs/wal/wal.h | 4 +-- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- source/dnode/vnode/inc/tq.h | 21 +++++-------- source/dnode/vnode/src/inc/tqInt.h | 3 ++ source/dnode/vnode/src/tq/tq.c | 34 ++++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 1 + 9 files changed, 59 insertions(+), 19 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1f7ec3be1e..cb0c76eb71 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1527,7 +1527,6 @@ typedef struct SMqSetCVgReq { SArray* tasks; // SArray } SMqSetCVgReq; - static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->vgId); @@ -1552,10 +1551,18 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { return buf; } +typedef struct SMqSetCVgRsp { + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; +} SMqSetCVgRsp; + typedef struct SMqCVConsumeReq { int64_t reqId; int64_t offset; int64_t clientId; + int64_t blockingTime; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqCVConsumeReq; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index ed7fdea2a8..52fff978bc 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -160,7 +160,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 51aaa7d903..641b485f4c 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -80,8 +80,8 @@ typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWal typedef struct SWalReadHead { int8_t headVer; - uint8_t msgType; - int8_t reserved[2]; + int16_t msgType; + int8_t reserved; int32_t len; int64_t ingestTs; // not implemented int64_t version; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e786a8972c..7a57956f3b 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -95,7 +95,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); pCEp->consumerId = consumerId; taosArrayPush(pSub->assigned, pCEp); - pSub->nextConsumerIdx++; + pSub->nextConsumerIdx = (pSub->nextConsumerIdx+1) % taosArrayGetSize(pSub->availConsumer); // build msg SMqSetCVgReq req = { diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 1a41a02673..eb82d19a4f 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -27,6 +27,7 @@ #include "trpc.h" #include "ttimer.h" #include "tutil.h" +#include "wal.h" #ifdef __cplusplus extern "C" { @@ -161,7 +162,8 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int32_t status; + int8_t status; + int64_t offset; void* dst; SSubQueryMsg* pMsg; } STqTaskItem; @@ -183,6 +185,7 @@ typedef struct STqClientHandle { int64_t committedOffset; int64_t currentOffset; STqBuffer buffer; + SWalReadHandle* pReadhandle; } STqClientHandle; typedef struct STqQueryMsg { @@ -321,25 +324,20 @@ void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); +#if 0 +int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); int tqSetCursor(STQ*, STqSetCurReq* pMsg); int tqBufferSetOffset(STqTopic*, int64_t offset); - STqTopic* tqFindTopic(STqGroup*, int64_t topicId); - STqGroup* tqGetGroup(STQ*, int64_t clientId); - STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); int tqRegisterContext(STqGroup*, void* ahandle); int tqSendLaunchQuery(STqMsgItem*, int64_t offset); +#endif -int tqSerializeGroup(const STqGroup*, STqSerializedHead**); - -const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**); - -static int tqQueryExecuting(int32_t status) { return status; } +int32_t tqProcessConsume(STQ* pTq, SRpcMsg *pMsg, SRpcMsg **ppRsp); typedef struct STqReadHandle { int64_t ver; @@ -350,9 +348,6 @@ typedef struct STqReadHandle { SMeta* pMeta; } STqReadHandle; -typedef struct SSubmitBlkScanInfo { -} SSubmitBlkScanInfo; - STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg* pMsg); bool tqNextDataBlock(STqReadHandle* pHandle); int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 107f5d5103..b4e1f57384 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -43,6 +43,9 @@ extern int32_t tqDebugFlag; // delete persistent storage for meta info // int tqDropTCGroup(STQ*, const char* topic, int cgId); +int tqSerializeGroup(const STqGroup*, STqSerializedHead**); +const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); +static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a5be0ec29a..b83cfe99bc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -608,6 +608,40 @@ int tqItemSSize() { return 0; } +int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg **ppRsp) { + SMqCVConsumeReq* pReq = pMsg->pCont; + int64_t reqId = pReq->reqId; + int64_t clientId = pReq->clientId; + int64_t offset = pReq->offset; + int64_t blockingTime = pReq->blockingTime; + + STqClientHandle* pHandle = tqHandleGet(pTq->tqMeta, clientId); + int8_t pos = offset % TQ_BUFFER_SIZE; + int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); + if (old == 1) { + // do nothing + } + if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { + //TODO + } + SWalHead *pHead = pHandle->pReadhandle->pHead; + while (pHead->head.msgType != TDMT_VND_SUBMIT) { + // read until find TDMT_VND_SUBMIT + } + SSubmitMsg *pCont = (SSubmitMsg*)&pHead->head.body; + + SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; + + void* outputData; + + atomic_store_8(&pHandle->buffer.output[pos].status, 1); + // launch query + // get result + // put into + SMqCvConsumeRsp* pRsp; + return 0; +} + STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg *pMsg) { STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 729d64f8b3..96791488fa 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -58,7 +58,7 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg, pRsp); case TDMT_VND_CONSUME: - return 0; + return tqProcessConsume(pVnode->pTq, pMsg, pRsp); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 2f3a4d5409..deb8a714a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -110,6 +110,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_SET_CONN: { + //TODO: wrap in a function char* reqStr = ptr; SMqSetCVgReq req; tDecodeSMqSetCVgReq(reqStr, &req); From 51f02913f4e60636297b16f382d2a159aea54a31 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 14:53:09 +0800 Subject: [PATCH 073/103] tq handle multiple topic --- include/common/tmsg.h | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 +- source/dnode/vnode/inc/tq.h | 30 +++-- source/dnode/vnode/src/tq/tq.c | 145 +++++++++++---------- 4 files changed, 95 insertions(+), 86 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cb0c76eb71..fe96f4ee52 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1561,7 +1561,7 @@ typedef struct SMqSetCVgRsp { typedef struct SMqCVConsumeReq { int64_t reqId; int64_t offset; - int64_t clientId; + int64_t consumerId; int64_t blockingTime; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 7a57956f3b..4158d7e7c7 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -55,8 +55,6 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/ - /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/ mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); return sdbSetTable(pMnode->pSdb, table); @@ -95,7 +93,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); pCEp->consumerId = consumerId; taosArrayPush(pSub->assigned, pCEp); - pSub->nextConsumerIdx = (pSub->nextConsumerIdx+1) % taosArrayGetSize(pSub->availConsumer); + pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); // build msg SMqSetCVgReq req = { diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index eb82d19a4f..aaf7f33061 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -162,7 +162,7 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int8_t status; + int8_t status; int64_t offset; void* dst; SSubQueryMsg* pMsg; @@ -175,18 +175,22 @@ typedef struct STqBuffer { STqTaskItem output[TQ_BUFFER_SIZE]; } STqBuffer; -typedef struct STqClientHandle { - int64_t clientId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_TOPIC_FNAME_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - int64_t committedOffset; - int64_t currentOffset; - STqBuffer buffer; +typedef struct STqTopicHandle { + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_TOPIC_FNAME_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + int64_t committedOffset; + int64_t currentOffset; + STqBuffer buffer; SWalReadHandle* pReadhandle; -} STqClientHandle; +} STqTopicHandle; + +typedef struct STqConsumerHandle { + int64_t consumerId; + SArray* topics; // SArray +} STqConsumerHandle; typedef struct STqQueryMsg { STqMsgItem* item; @@ -337,7 +341,7 @@ int tqRegisterContext(STqGroup*, void* ahandle); int tqSendLaunchQuery(STqMsgItem*, int64_t offset); #endif -int32_t tqProcessConsume(STQ* pTq, SRpcMsg *pMsg, SRpcMsg **ppRsp); +int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); typedef struct STqReadHandle { int64_t ver; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b83cfe99bc..c4a4f087c6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -37,7 +37,7 @@ const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); - if(old == 1) return 0; + if (old == 1) return 0; tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ"); return 0; @@ -45,7 +45,7 @@ int tqInit() { void tqCleanUp() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0); - if(old == 0) return; + if (old == 0) return; taosTmrStop(tqMgmt.timer); taosTmrCleanUp(tqMgmt.timer); } @@ -150,7 +150,7 @@ int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup memset(pGroup, 0, sizeof(STqGroup)); pGroup->topicList = tdListNew(sizeof(STqTopic)); - if(pGroup->topicList == NULL) { + if (pGroup->topicList == NULL) { free(pGroup); return -1; } @@ -190,7 +190,7 @@ static int tqFetch(STqGroup* pGroup, STqConsumeRsp** pRsp) { int totSize = 0; int numOfMsgs = 0; // TODO: make it a macro - int sizeLimit = 4 * 1024; + int sizeLimit = 4 * 1024; void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + sizeLimit); if (ptr == NULL) { @@ -329,9 +329,9 @@ int tqProcessCMsg(STQ* pTq, STqConsumeReq* pMsg, STqRspHandle* pRsp) { } int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { - STqConsumeReq *pMsg = pReq->pCont; - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); + STqConsumeReq* pMsg = pReq->pCont; + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); if (pGroup == NULL) { terrno = TSDB_CODE_TQ_GROUP_NOT_SET; return -1; @@ -343,9 +343,8 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { int numOfMsgs = 0; int sizeLimit = 4096; - - STqConsumeRsp *pCsmRsp = (*pRsp)->pCont; - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); + STqConsumeRsp* pCsmRsp = (*pRsp)->pCont; + void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); if (ptr == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -356,16 +355,16 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { tdListInitIter(topicList, &iter, TD_LIST_FORWARD); STqMsgContent* buffer = NULL; - SArray* pArray = taosArrayInit(0, sizeof(void*)); + SArray* pArray = taosArrayInit(0, sizeof(void*)); - SListNode *pn; - while((pn = tdListNext(&iter)) != NULL) { - STqTopic* pTopic = *(STqTopic**)pn->data; - int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; + SListNode* pn; + while ((pn = tdListNext(&iter)) != NULL) { + STqTopic* pTopic = *(STqTopic**)pn->data; + int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; STqMsgItem* pItem = &pTopic->buffer[idx]; if (pItem->content != NULL && pItem->offset == pTopic->floatingCursor) { - if(pItem->status == TQ_ITEM_READY) { - //if has data + if (pItem->status == TQ_ITEM_READY) { + // if has data totSize += pTopic->buffer[idx].size; if (totSize > sizeLimit) { void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + totSize); @@ -388,13 +387,13 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { if (totSize > sizeLimit) { break; } - } else if(pItem->status == TQ_ITEM_PROCESS) { - //if not have data but in process + } else if (pItem->status == TQ_ITEM_PROCESS) { + // if not have data but in process - } else if(pItem->status == TQ_ITEM_EMPTY){ - //if not have data and not in process + } else if (pItem->status == TQ_ITEM_EMPTY) { + // if not have data and not in process int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_EMPTY, TQ_ITEM_PROCESS); - if(old != TQ_ITEM_EMPTY) { + if (old != TQ_ITEM_EMPTY) { continue; } pItem->offset = pTopic->floatingCursor; @@ -416,22 +415,22 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { } // fetched a num of msgs, rpc response - for(int i = 0; i < pArray->size; i++) { + for (int i = 0; i < pArray->size; i++) { STqMsgItem* pItem = taosArrayGet(pArray, i); - //read from wal + // read from wal void* raw = NULL; /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset); - if(code < 0) { - //TODO: error + if (code < 0) { + // TODO: error } - //get msgType - //if submitblk + // get msgType + // if submitblk pItem->executor->assign(pItem->executor->runtimeEnv, raw); SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); pItem->content = content; - //if other type, send just put into buffer + // if other type, send just put into buffer /*pItem->content = raw;*/ int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); @@ -608,41 +607,48 @@ int tqItemSSize() { return 0; } -int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg **ppRsp) { +int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SMqCVConsumeReq* pReq = pMsg->pCont; - int64_t reqId = pReq->reqId; - int64_t clientId = pReq->clientId; - int64_t offset = pReq->offset; - int64_t blockingTime = pReq->blockingTime; + int64_t reqId = pReq->reqId; + int64_t consumerId = pReq->consumerId; + int64_t offset = pReq->offset; + int64_t blockingTime = pReq->blockingTime; - STqClientHandle* pHandle = tqHandleGet(pTq->tqMeta, clientId); - int8_t pos = offset % TQ_BUFFER_SIZE; - int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); - if (old == 1) { - // do nothing + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); + int sz = taosArrayGetSize(pConsumer->topics); + + for (int i = 0 ; i < sz; i++) { + STqTopicHandle *pHandle = taosArrayGet(pConsumer->topics, i); + + int8_t pos = offset % TQ_BUFFER_SIZE; + int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); + if (old == 1) { + // do nothing + } + if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { + // TODO + } + SWalHead* pHead = pHandle->pReadhandle->pHead; + while (pHead->head.msgType != TDMT_VND_SUBMIT) { + // read until find TDMT_VND_SUBMIT + } + SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; + + SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; + + void* outputData; + atomic_store_8(&pHandle->buffer.output[pos].status, 1); + + // put output into rsp } - if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { - //TODO - } - SWalHead *pHead = pHandle->pReadhandle->pHead; - while (pHead->head.msgType != TDMT_VND_SUBMIT) { - // read until find TDMT_VND_SUBMIT - } - SSubmitMsg *pCont = (SSubmitMsg*)&pHead->head.body; - SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; - - void* outputData; - - atomic_store_8(&pHandle->buffer.output[pos].status, 1); // launch query // get result - // put into SMqCvConsumeRsp* pRsp; return 0; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg *pMsg) { +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg* pMsg) { STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; @@ -655,39 +661,39 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg *pMsg) { } bool tqNextDataBlock(STqReadHandle* pHandle) { - if(tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } return true; } int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - SMemRow row; - int32_t sversion = pHandle->pBlock->sversion; + SMemRow row; + int32_t sversion = pHandle->pBlock->sversion; SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false); pBlockInfo->numOfCols = pSchema->nCols; pBlockInfo->rows = pHandle->pBlock->numOfRows; pBlockInfo->uid = pHandle->pBlock->uid; - //TODO: filter out unused column + // TODO: filter out unused column return 0; } -SArray *tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) { - int32_t sversion = pHandle->pBlock->sversion; +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) { + int32_t sversion = pHandle->pBlock->sversion; SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true); - STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); - SArray *pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); + STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); + SArray* pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); if (pArray == NULL) { return NULL; } SColumnInfoData colInfo; - int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes; + int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes; colInfo.pData = malloc(sz); if (colInfo.pData == NULL) { return NULL; } for (int i = 0; i < pTschema->numOfCols; i++) { - //TODO: filter out unused column + // TODO: filter out unused column taosArrayPush(pColumnIdList, &(schemaColAt(pTschema, i)->colId)); } @@ -695,16 +701,17 @@ SArray *tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) { int32_t kvIdx; while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { for (int i = 0; i < pTschema->numOfCols && kvIdx < pTschema->numOfCols; i++) { - //TODO: filter out unused column - STColumn *pCol = schemaColAt(pTschema, i); + // TODO: filter out unused column + STColumn* pCol = schemaColAt(pTschema, i); void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - //TODO: handle varlen + // TODO: handle varlen memcpy(POINTER_SHIFT(colInfo.pData, pCol->offset), val, pCol->bytes); } } taosArrayPush(pArray, &colInfo); return pArray; } -/*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t status) {*/ - /*return 0;*/ +/*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t + * status) {*/ +/*return 0;*/ /*}*/ From fa73f1d497436170f9cdeae75455aaf8e31332f6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 15:04:40 +0800 Subject: [PATCH 074/103] fix compile error --- include/common/tmsg.h | 8 +++++++- source/dnode/vnode/inc/tq.h | 1 + source/dnode/vnode/src/vnd/vnodeWrite.c | 24 +++++++++++++----------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fe96f4ee52..a3d62cc52d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1567,6 +1567,12 @@ typedef struct SMqCVConsumeReq { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqCVConsumeReq; +typedef struct SMqConsumeRspBlock { + int32_t bodyLen; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char body[]; +} SMqConsumeRspBlock; + typedef struct SMqCVConsumeRsp { int64_t reqId; int64_t clientId; @@ -1576,7 +1582,7 @@ typedef struct SMqCVConsumeRsp { int32_t skipLogNum; int32_t bodyLen; char topicName[TSDB_TOPIC_FNAME_LEN]; - char body[]; + SMqConsumeRspBlock blocks[]; } SMqCvConsumeRsp; #ifdef __cplusplus diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index aaf7f33061..588305c8ae 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -189,6 +189,7 @@ typedef struct STqTopicHandle { typedef struct STqConsumerHandle { int64_t consumerId; + int64_t epoch; SArray* topics; // SArray } STqConsumerHandle; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index deb8a714a6..f05520a960 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -114,15 +114,17 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { char* reqStr = ptr; SMqSetCVgReq req; tDecodeSMqSetCVgReq(reqStr, &req); - STqClientHandle* pHandle = calloc(sizeof(STqClientHandle), 1); - if (pHandle == NULL) { + STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); + + STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); + if (pTopic == NULL) { // TODO: handle error } - strcpy(pHandle->topicName, req.topicName); - strcpy(pHandle->cGroup, req.cGroup); - strcpy(pHandle->sql, req.sql); - strcpy(pHandle->logicalPlan, req.logicalPlan); - strcpy(pHandle->physicalPlan, req.physicalPlan); + strcpy(pTopic->topicName, req.topicName); + strcpy(pTopic->cGroup, req.cGroup); + strcpy(pTopic->sql, req.sql); + strcpy(pTopic->logicalPlan, req.logicalPlan); + strcpy(pTopic->physicalPlan, req.physicalPlan); SArray *pArray; //TODO: deserialize to SQueryDag SQueryDag *pDag; @@ -134,12 +136,12 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { STaskInfo *pInfo = taosArrayGet(pArray, 0); SArray* pTasks; schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); - pHandle->buffer.firstOffset = -1; - pHandle->buffer.lastOffset = -1; + pTopic->buffer.firstOffset = -1; + pTopic->buffer.lastOffset = -1; for (int i = 0; i < TQ_BUFFER_SIZE; i++) { SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); - pHandle->buffer.output[i].pMsg = pMsg; - pHandle->buffer.output[i].status = 0; + pTopic->buffer.output[i].pMsg = pMsg; + pTopic->buffer.output[i].status = 0; } // write mq meta } From 3203e8fe42eec55890c44a96478bd6efde763c64 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 15:18:33 +0800 Subject: [PATCH 075/103] try integrating allocator --- source/dnode/vnode/inc/tq.h | 14 +------------- source/dnode/vnode/src/tq/tq.c | 15 +++++---------- source/dnode/vnode/src/vnd/vnodeMain.c | 4 ++-- 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 588305c8ae..ec886286f3 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -198,17 +198,6 @@ typedef struct STqQueryMsg { struct STqQueryMsg* next; } STqQueryMsg; -typedef struct STqLogHandle { - void* logHandle; - void* (*openLogReader)(void* logHandle); - void (*closeLogReader)(void* logReader); - int32_t (*logRead)(void* logReader, void** data, int64_t ver); - - int64_t (*logGetFirstVer)(void* logHandle); - int64_t (*logGetSnapshotVer)(void* logHandle); - int64_t (*logGetLastVer)(void* logHandle); -} STqLogHandle; - typedef struct STqCfg { // TODO } STqCfg; @@ -306,7 +295,6 @@ typedef struct STQ { // the handle of meta kvstore char* path; STqCfg* tqConfig; - STqLogHandle* tqLogHandle; STqMemRef tqMemRef; STqMetaStore* tqMeta; } STQ; @@ -323,7 +311,7 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c4a4f087c6..2b2272caea 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -50,7 +50,7 @@ void tqCleanUp() { taosTmrCleanUp(tqMgmt.timer); } -STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -58,20 +58,15 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemA } pTq->path = strdup(path); pTq->tqConfig = tqConfig; - pTq->tqLogHandle = tqLogHandle; -#if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { // TODO: error code of buffer pool } -#endif pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { - free(pTq); -#if 0 allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); -#endif + free(pTq); return NULL; } @@ -421,10 +416,10 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { // read from wal void* raw = NULL; /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ - int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset); - if (code < 0) { + /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ + /*if (code < 0) {*/ // TODO: error - } + /*}*/ // get msgType // if submitblk pItem->executor->assign(pItem->executor->runtimeEnv, raw); diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 76b7ccf0d9..cb0d76ed29 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -119,7 +119,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // TODO: Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; @@ -151,4 +151,4 @@ static void vnodeCloseImpl(SVnode *pVnode) { tqClose(pVnode->pTq); walClose(pVnode->pWal); } -} \ No newline at end of file +} From 6bf4614bfa0fbaa6fcf13024ef4f25b4b0d96677 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 15:25:13 +0800 Subject: [PATCH 076/103] revert and remove allocator --- source/dnode/vnode/src/tq/tq.c | 6 +++++- source/dnode/vnode/src/vnd/vnodeMain.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2b2272caea..ad71005153 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -58,15 +58,19 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) } pTq->path = strdup(path); pTq->tqConfig = tqConfig; +#if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { // TODO: error code of buffer pool } +#endif pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { - allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); free(pTq); +#if 0 + allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); +#endif return NULL; } diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index cb0d76ed29..fd79341adf 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -117,7 +117,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { return -1; } - // TODO: Open TQ + // Open TQ sprintf(dir, "%s/tq", pVnode->path); pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { From 5b5122957763351568557393eaa819a50b1bc37c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 15:39:28 +0800 Subject: [PATCH 077/103] integrating wal with tq --- source/dnode/vnode/inc/tq.h | 5 +++-- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/vnd/vnodeMain.c | 16 ++++++++-------- source/dnode/vnode/src/vnd/vnodeWrite.c | 3 ++- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index ec886286f3..e394297ab8 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -177,7 +177,7 @@ typedef struct STqBuffer { typedef struct STqTopicHandle { char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_TOPIC_FNAME_LEN]; char* sql; char* logicalPlan; char* physicalPlan; @@ -297,6 +297,7 @@ typedef struct STQ { STqCfg* tqConfig; STqMemRef tqMemRef; STqMetaStore* tqMeta; + SWal * pWal; } STQ; typedef struct STqMgmt { @@ -311,7 +312,7 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ad71005153..15c7971252 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -50,7 +50,7 @@ void tqCleanUp() { taosTmrCleanUp(tqMgmt.timer); } -STQ* tqOpen(const char* path, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index fd79341adf..c4bbd93eda 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -117,14 +117,6 @@ static int vnodeOpenImpl(SVnode *pVnode) { return -1; } - // Open TQ - sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); - if (pVnode->pTq == NULL) { - // TODO: handle error - return -1; - } - // Open WAL sprintf(dir, "%s/wal", pVnode->path); pVnode->pWal = walOpen(dir, &(pVnode->config.walCfg)); @@ -133,6 +125,14 @@ static int vnodeOpenImpl(SVnode *pVnode) { return -1; } + // Open TQ + sprintf(dir, "%s/tq", pVnode->path); + pVnode->pTq = tqOpen(dir, pVnode->pWal, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + if (pVnode->pTq == NULL) { + // TODO: handle error + return -1; + } + // Open Query if (vnodeQueryOpen(pVnode)) { return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index f05520a960..6e2d438970 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -121,7 +121,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } strcpy(pTopic->topicName, req.topicName); - strcpy(pTopic->cGroup, req.cGroup); + strcpy(pTopic->cgroup, req.cGroup); strcpy(pTopic->sql, req.sql); strcpy(pTopic->logicalPlan, req.logicalPlan); strcpy(pTopic->physicalPlan, req.physicalPlan); @@ -143,6 +143,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pTopic->buffer.output[i].pMsg = pMsg; pTopic->buffer.output[i].status = 0; } + pTopic->pReadhandle = walOpenReadHandle(pVnode->pTq->pWal); // write mq meta } break; From 275336e1484f457c921a4cfb5d6c4dd730a0e68f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 20 Jan 2022 01:09:54 -0800 Subject: [PATCH 078/103] add test cases --- source/libs/tfs/src/tfs.c | 18 ++++-- source/libs/tfs/test/tfsTest.cpp | 100 +++++++++++++++++++++++++++++- tests/script/sim/db/basic6.sim | 1 - tests/script/sim/dnode/basic1.sim | 9 ++- 4 files changed, 114 insertions(+), 14 deletions(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 582422dc94..623293f82b 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -141,6 +141,7 @@ const char *tfsGetDiskPath(STfs *pTfs, SDiskID diskId) { return TFS_DISK_AT(pTfs void tfsInitFile(STfs *pTfs, STfsFile *pFile, SDiskID diskId, const char *rname) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); + if (pDisk == NULL) return; pFile->did = diskId; tstrncpy(pFile->rname, rname, TSDB_FILENAME_LEN); @@ -197,9 +198,7 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } -int32_t tfsRemoveFile(const STfsFile *pFile) { - return remove(pFile->aname); -} +int32_t tfsRemoveFile(const STfsFile *pFile) { return remove(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { return taosCopyFile(pFile1->aname, pFile2->aname); @@ -332,7 +331,12 @@ const STfsFile *tfsReaddir(STfsDir *pDir) { // Skip . and .. if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; - snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); + if (pDir->dirname == NULL || pDir->dirname[0] == 0) { + snprintf(bname, TMPNAME_LEN * 2, "%s", dp->d_name); + } else { + snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name); + } + tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname); return &pDir->tfile; } @@ -502,7 +506,11 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) { pDir->did.level = pDisk->level; pDir->did.id = pDisk->id; - snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + if (pDisk->path == NULL || pDisk->path[0] == 0) { + snprintf(adir, TMPNAME_LEN * 2, "%s", pDir->dirname); + } else { + snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname); + } pDir->dir = opendir(adir); if (pDir->dir != NULL) break; } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 6d68a5d87a..abc00c5cd9 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -158,7 +158,7 @@ TEST_F(TfsTest, 03_Dir) { tfsClose(pTfs); } -#if 0 + TEST_F(TfsTest, 04_File) { int32_t code = 0; SDiskCfg dCfg = {0}; @@ -194,6 +194,100 @@ TEST_F(TfsTest, 04_File) { EXPECT_FALSE(tfsIsSameFile(&file0, &file3)); EXPECT_FALSE(tfsIsSameFile(&file0, &file4)); + { + int32_t size = 1024; + void *ret = malloc(size + sizeof(size_t)); + *(size_t *)ret = size; + void *buf = (void *)((char *)ret + sizeof(size_t)); + + file0.did.id = 0; + file0.did.level = 0; + int32_t len = tfsEncodeFile((void **)&buf, &file0); + EXPECT_EQ(len, 8); + + STfsFile outfile = {0}; + char *outbuf = (char *)tfsDecodeFile(pTfs, (void *)((char *)buf - len), &outfile); + int32_t decodeLen = (outbuf - (char *)buf); + + EXPECT_EQ(outfile.did.id, 0); + EXPECT_EQ(outfile.did.level, 0); + EXPECT_STREQ(outfile.aname, file0.aname); + EXPECT_STREQ(outfile.rname, "fname"); + EXPECT_EQ(outfile.pTfs, pTfs); + } + + { + char n1[] = "t3/t1.json"; + char n2[] = "t3/t2.json"; + STfsFile f1 = {0}; + STfsFile f2 = {0}; + SDiskID did; + did.id = 0; + did.level = 0; + + tfsInitFile(pTfs, &f1, did, n1); + tfsInitFile(pTfs, &f2, did, n2); + + EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); + + FILE *fp = fopen(f1.aname, "w"); + ASSERT_NE(fp, nullptr); + fwrite("12345678", 1, 5, fp); + fclose(fp); + + char base[128] = {0}; + tfsBasename(&f1, base); + char dir[128] = {0}; + tfsDirname(&f1, dir); + + EXPECT_STREQ(base, "t1.json"); + + char fulldir[128]; + snprintf(fulldir, 128, "%s%s%s", root, TD_DIRSEP, "t3"); + EXPECT_STREQ(dir, fulldir); + + EXPECT_NE(tfsCopyFile(&f1, &f2), 0); + + char af2[128] = {0}; + snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); + EXPECT_EQ(taosDirExist(af2), 0); + tfsRemoveFile(&f2); + EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(tfsCopyFile(&f1, &f2), 0); + + { + STfsDir *pDir = tfsOpendir(pTfs, ""); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_STREQ(pf1->rname, "t3"); + EXPECT_EQ(pf1->did.id, 0); + EXPECT_EQ(pf1->did.level, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_EQ(pf2, nullptr); + + tfsClosedir(pDir); + } + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_EQ(pf1->did.id, 0); + EXPECT_EQ(pf1->did.level, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_NE(pf2, nullptr); + + const STfsFile *pf3 = tfsReaddir(pDir); + EXPECT_EQ(pf3, nullptr); + + tfsClosedir(pDir); + } + } + tfsClose(pTfs); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index 18ac7ffeb1..eb12da2ccb 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -114,7 +114,6 @@ while $i < 5 $i = $i + 1 endw -return print =============== step7 $i = 0 while $i < 5 diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/sim/dnode/basic1.sim index 94b29e877e..65a04f80b3 100644 --- a/tests/script/sim/dnode/basic1.sim +++ b/tests/script/sim/dnode/basic1.sim @@ -177,12 +177,11 @@ if $rows != 3 then return -1 endi -sql select * from st -if $rows != 15 then - return -1 -endi +#sql select * from st +#if $rows != 15 then +# return -1 +#endi -return print =============== drop dnode sql drop dnode 2; sql show dnodes; From bdf03d8fa9ba3b3c675c2c21093e068b5fdfb129 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Jan 2022 17:10:28 +0800 Subject: [PATCH 079/103] [td-11818] refactor. --- include/libs/function/function.h | 1 + source/libs/executor/inc/executorimpl.h | 7 +- source/libs/executor/src/executorMain.c | 10 +- source/libs/executor/src/executorimpl.c | 204 +++++++++++++----------- 4 files changed, 116 insertions(+), 106 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 00f71f103f..39d8ec3420 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -165,6 +165,7 @@ typedef struct SQLFunctionCtx { SPoint1 start; SPoint1 end; + int32_t columnIndex; SFunctionFpSet* fpSet; } SQLFunctionCtx; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 85f7b9e42c..dd633002db 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -414,6 +414,10 @@ typedef struct STagScanInfo { int32_t curPos; } STagScanInfo; +typedef struct SStreamBlockScanInfo { + +} SStreamBlockScanInfo; + typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; int32_t *rowCellInfoOffset; // offset value for each row result cell info @@ -554,13 +558,12 @@ typedef struct SOrderOperatorInfo { SSDataBlock *pDataBlock; } SOrderOperatorInfo; -void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); - SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 8b7370f9c1..f39df4d4ae 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -73,15 +73,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ assert(tsdb != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = 0; - uint64_t uid = 0; - STimeWindow window = TSWINDOW_INITIALIZER; - int32_t tableType = 0; - - SPhyNode* pPhyNode = pSubplan->pNode; -// STableGroupInfo groupInfo = {0}; - - code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); + int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 97dc7e1dd5..2b5faba29d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,11 +12,12 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "tmsg.h" -#include "tglobal.h" -#include "ttime.h" +#include #include "exception.h" +#include "os.h" +#include "tglobal.h" +#include "tmsg.h" +#include "ttime.h" #include "executorimpl.h" #include "function.h" @@ -176,7 +177,7 @@ static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); +static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static bool hasMainOutput(STaskAttr *pQueryAttr); @@ -320,7 +321,7 @@ SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { for (int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData idata = {{0}}; - SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); idata.info.type = pExpr->base.resSchema.type; idata.info.bytes = pExpr->base.resSchema.bytes; @@ -382,8 +383,8 @@ static bool isProjQuery(STaskAttr *pQueryAttr) { return true; } -static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { - if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +static bool hasNull(SColumn* pColumn, SColumnDataAgg *pStatis) { + if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { return false; } @@ -1122,43 +1123,52 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pC } void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { - if (pCtx[0].functionId == FUNCTION_ARITHM) { +// if (pCtx[0].functionId == FUNCTION_ARITHM) { // SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; // if (pSupport->colList == NULL) { // doSetInputDataBlock(pOperator, pCtx, pBlock, order); // } else { // doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); // } - } else { +// } else { if (pBlock->pDataBlock != NULL) { doSetInputDataBlock(pOperator, pCtx, pBlock, order); } else { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } - } +// } } static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { -#if 0 for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; - pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + pCtx[i].currentStage = MAIN_SCAN/*(uint8_t)pOperator->pRuntimeEnv->scanFlag*/; - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns); if (pCtx[i].functionId == FUNCTION_ARITHM) { // setArithParams((SScalarFunctionSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); } else { - SColIndex* pCol = &pOperator->pExpr[i].base.pColumns->info.; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == FUNCTION_BLKINFO) || - (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + uint32_t flag = pOperator->pExpr[i].base.pColumns->flag; + if (TSDB_COL_IS_NORMAL_COL(flag) /*|| (pCtx[i].functionId == FUNCTION_BLKINFO) || + (TSDB_COL_IS_TAG(flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)*/) { + SColumn* pCol = pOperator->pExpr[i].base.pColumns; + if (pCtx[i].columnIndex == -1) { + for(int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + if (pColData->info.colId == pCol->info.colId) { + pCtx[i].columnIndex = j; + break; + } + } + } + + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pCtx[i].columnIndex); // in case of the block distribution query, the inputBytes is not a constant value. pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); + assert(p->info.colId == pCol->info.colId); if (pCtx[i].functionId < 0) { SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); @@ -1169,30 +1179,28 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, // uint32_t status = aAggs[pCtx[i].functionId].status; // if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { -// SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); -// // In case of the top/bottom query again the nest query result, which has no timestamp column -// // don't set the ptsList attribute. -// if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { -// pCtx[i].ptsList = (int64_t*) tsInfo->pData; -// } else { -// pCtx[i].ptsList = NULL; -// } + SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); + // In case of the top/bottom query again the nest query result, which has no timestamp column + // don't set the ptsList attribute. + if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + pCtx[i].ptsList = (int64_t*) tsInfo->pData; + } else { + pCtx[i].ptsList = NULL; + } +// } +// } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { +// SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; +// SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); +// +// pCtx[i].pInput = p->pData; +// assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); +// for(int32_t j = 0; j < pBlock->info.rows; ++j) { +// char* dst = p->pData + j * p->info.bytes; +// taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); // } - } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { - SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; - SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); - - pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); - for(int32_t j = 0; j < pBlock->info.rows; ++j) { - char* dst = p->pData + j * p->info.bytes; - taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); - } } } } -#endif - } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { @@ -1837,19 +1845,19 @@ static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx * } // in the reverse table scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv) || - (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { - return false; - } +// if (IS_REVERSE_SCAN(pRuntimeEnv) || +// (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { +// return false; +// } return true; } -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { +void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { SColumnDataAgg *pAgg = NULL; - if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - pAgg = &pSDataBlock->pBlockAgg[pColIndex->colIndex]; + if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColumn->flag)) { + pAgg = &pSDataBlock->pBlockAgg[pCtx->columnIndex]; pCtx->agg = *pAgg; pCtx->isAggSet = true; @@ -1858,10 +1866,10 @@ void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColInde pCtx->isAggSet = false; } - pCtx->hasNull = hasNull(pColIndex, pAgg); + pCtx->hasNull = hasNull(pColumn, pAgg); // set the statistics data for primary time stamp column - if (pCtx->functionId == FUNCTION_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pCtx->functionId == FUNCTION_SPREAD && pColumn->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pCtx->isAggSet = true; pCtx->agg.min = pSDataBlock->info.window.skey; pCtx->agg.max = pSDataBlock->info.window.ekey; @@ -2033,6 +2041,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC SSqlExpr *pSqlExpr = &pExpr->base; SQLFunctionCtx* pCtx = &pFuncCtx[i]; + #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2043,16 +2052,16 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC pCtx->requireNull = false; } #endif -// pCtx->inputBytes = pSqlExpr->colBytes; +// pCtx->inputBytes = pSqlExpr->; // pCtx->inputType = pSqlExpr->colType; pCtx->ptsOutputBuf = NULL; pCtx->fpSet = fpSet; - + pCtx->columnIndex = -1; pCtx->resDataInfo.bytes = pSqlExpr->resSchema.bytes; pCtx->resDataInfo.type = pSqlExpr->resSchema.type; -// pCtx->order = pQueryAttr->order.order; + pCtx->order = TSDB_ORDER_ASC; // pCtx->functionId = pSqlExpr->functionId; // pCtx->stableQuery = pQueryAttr->stableQuery; pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes; @@ -2108,12 +2117,12 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC } } -// for(int32_t i = 1; i < numOfOutput; ++i) { -// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes); -// } + for(int32_t i = 1; i < numOfOutput; ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i - 1); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr->base.interBytes); + } setCtxTagColumnInfo(pFuncCtx, numOfOutput); - return pFuncCtx; } @@ -3717,49 +3726,49 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfOutput = pOperator->numOfOutput; - if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { - // for each group result, call the finalize function for each column - if (pQueryAttr->groupbyColumn) { - closeAllResultRows(pResultRowInfo); - } - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *buf = pResultRowInfo->pResult[i]; - if (!isResultRowClosed(pResultRowInfo, i)) { - continue; - } - - setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); - - for (int32_t j = 0; j < numOfOutput; ++j) { -// pCtx[j].startTs = buf->win.skey; -// if (pCtx[j].functionId < 0) { -// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); -// } else { -// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); -// } - } - - - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); - } - - } else { +// if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { +// // for each group result, call the finalize function for each column +// if (pQueryAttr->groupbyColumn) { +// closeAllResultRows(pResultRowInfo); +// } +// +// for (int32_t i = 0; i < pResultRowInfo->size; ++i) { +// SResultRow *buf = pResultRowInfo->pResult[i]; +// if (!isResultRowClosed(pResultRowInfo, i)) { +// continue; +// } +// +// setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); +// +// for (int32_t j = 0; j < numOfOutput; ++j) { +//// pCtx[j].startTs = buf->win.skey; +//// if (pCtx[j].functionId < 0) { +//// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); +//// } else { +//// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); +//// } +// } +// +// +// /* +// * set the number of output results for group by normal columns, the number of output rows usually is 1 except +// * the top and bottom query +// */ +// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); +// } +// +// } else { for (int32_t j = 0; j < numOfOutput; ++j) { // if (pCtx[j].functionId < 0) { // doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); // } else { -// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); + pCtx[j].fpSet->finalize(&pCtx[j]); // } } - } +// } } static bool hasMainOutput(STaskAttr *pQueryAttr) { @@ -5362,6 +5371,11 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } +SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + +} + + void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { assert(pTableScanInfo != NULL && pDownstream != NULL); @@ -5752,9 +5766,9 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { break; } - if (pAggInfo->current != NULL) { +// if (pAggInfo->current != NULL) { // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - } +// } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); @@ -6667,8 +6681,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGet(pExprInfo, i); - memcpy(&p[i], pExpr, sizeof(SExprInfo)); + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + assignExprInfo(&p[i], pExpr); } SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); From e0d9ca04d4ab8fa458f2544cdc106d118e6389ca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Jan 2022 17:37:12 +0800 Subject: [PATCH 080/103] [td-11818] refactor. --- source/libs/planner/src/planner.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 6b3f37741e..9b32213ad7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From 59c0976d988c592ec4300cee863065e6ff04cefd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 20 Jan 2022 01:45:41 -0800 Subject: [PATCH 081/103] minor changes --- tests/script/sim/dnode/basic1.sim | 8 +-- tests/script/sim/table/basic1.sim | 90 +++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/sim/dnode/basic1.sim index 65a04f80b3..e689bb3261 100644 --- a/tests/script/sim/dnode/basic1.sim +++ b/tests/script/sim/dnode/basic1.sim @@ -114,9 +114,9 @@ sql create table c5 using st tags(2) sql show tables print $data00 $data01 $data02 -#if $rows != 5 then -# return -1 -#endi +if $rows != 5 then + return -1 +endi print =============== insert data sql insert into c1 values(now+1s, 1) @@ -177,7 +177,7 @@ if $rows != 3 then return -1 endi -#sql select * from st +sql select * from st #if $rows != 15 then # return -1 #endi diff --git a/tests/script/sim/table/basic1.sim b/tests/script/sim/table/basic1.sim index c33182dbf9..f47630b737 100644 --- a/tests/script/sim/table/basic1.sim +++ b/tests/script/sim/table/basic1.sim @@ -66,6 +66,24 @@ print =============== insert data sql insert into c1 values(now+1s, 1) sql insert into c1 values(now+2s, 2) sql insert into c1 values(now+3s, 3) +sql insert into c2 values(now+1s, 1) +sql insert into c2 values(now+2s, 2) +sql insert into c2 values(now+3s, 3) +sql insert into c3 values(now+1s, 1) +sql insert into c3 values(now+2s, 2) +sql insert into c3 values(now+3s, 3) +sql insert into c4 values(now+1s, 1) +sql insert into c4 values(now+2s, 2) +sql insert into c4 values(now+3s, 3) +sql insert into c5 values(now+1s, 1) +sql insert into c5 values(now+2s, 2) +sql insert into c5 values(now+3s, 3) +sql insert into c6 values(now+1s, 1) +sql insert into c6 values(now+2s, 2) +sql insert into c6 values(now+3s, 3) +sql insert into c7 values(now+1s, 1) +sql insert into c7 values(now+2s, 2) +sql insert into c7 values(now+3s, 3) print =============== query data sql select * from c1 @@ -89,6 +107,42 @@ if $data21 != 3 then return -1 endi +sql select * from c2 +if $rows != 3 then + return -1 +endi + +sql select * from c3 +if $rows != 3 then + return -1 +endi + +sql select * from c4 +if $rows != 3 then + return -1 +endi + +sql select * from c5 +if $rows != 3 then + return -1 +endi + +sql select * from c6 +if $rows != 3 then + return -1 +endi + +sql select * from c7 +if $rows != 3 then + return -1 +endi + +print =============== query data frpm st +sql select * from st +#if $rows != 21 then +# return -1 +#endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -114,4 +168,40 @@ if $data21 != 3 then return -1 endi +sql select * from c2 +if $rows != 3 then + return -1 +endi + +sql select * from c3 +if $rows != 3 then + return -1 +endi + +sql select * from c4 +if $rows != 3 then + return -1 +endi + +sql select * from c5 +if $rows != 3 then + return -1 +endi + +sql select * from c6 +if $rows != 3 then + return -1 +endi + +sql select * from c7 +if $rows != 3 then + return -1 +endi + +print =============== query data frpm st +sql select * from st +#if $rows != 21 then +# return -1 +#endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 6a3d98cda42e2a6d3f02eee562451bb32f5beb60 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 20 Jan 2022 18:09:28 +0800 Subject: [PATCH 082/103] refine tqRead interface --- include/common/tmsg.h | 6 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 +- source/dnode/vnode/inc/tq.h | 11 ++-- source/dnode/vnode/src/tq/tq.c | 72 +++++++++++++++++++--- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 44 ++----------- 6 files changed, 81 insertions(+), 58 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a3d62cc52d..7690e53594 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1520,7 +1520,7 @@ typedef struct SMqSetCVgReq { int32_t vgId; int64_t consumerId; char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_CONSUMER_GROUP_LEN]; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; char* sql; char* logicalPlan; char* physicalPlan; @@ -1532,7 +1532,7 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->topicName); - tlen += taosEncodeString(buf, pReq->cGroup); + tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); @@ -1543,7 +1543,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->topicName); - buf = taosDecodeStringTo(buf, pReq->cGroup); + buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 4158d7e7c7..4cf7505f74 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -100,7 +100,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { .vgId = pCEp->vgId, .consumerId = consumerId, }; - strcpy(req.cGroup, cgroup); + strcpy(req.cgroup, cgroup); strcpy(req.topicName, topic); strcpy(req.sql, pTopic->sql); strcpy(req.logicalPlan, pTopic->logicalPlan); @@ -168,7 +168,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume .vgId = vgId, .consumerId = pConsumer->consumerId, }; - strcpy(req.cGroup, pConsumer->cgroup); + strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.topicName, pTopic->name); strcpy(req.sql, pTopic->sql); strcpy(req.logicalPlan, pTopic->logicalPlan); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index e394297ab8..ec71777882 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -297,7 +297,7 @@ typedef struct STQ { STqCfg* tqConfig; STqMemRef tqMemRef; STqMetaStore* tqMeta; - SWal * pWal; + SWal* pWal; } STQ; typedef struct STqMgmt { @@ -331,7 +331,8 @@ int tqRegisterContext(STqGroup*, void* ahandle); int tqSendLaunchQuery(STqMsgItem*, int64_t offset); #endif -int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); +int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq); typedef struct STqReadHandle { int64_t ver; @@ -340,13 +341,15 @@ typedef struct STqReadHandle { SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; SMeta* pMeta; + SArray* pColumnIdList; } STqReadHandle; -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg* pMsg); +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList); +void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle* pHandle); int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); // return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList); +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 15c7971252..ead856a06b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -606,7 +606,7 @@ int tqItemSSize() { return 0; } -int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SMqCVConsumeReq* pReq = pMsg->pCont; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; @@ -623,6 +623,7 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); if (old == 1) { // do nothing + continue; } if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { // TODO @@ -635,7 +636,17 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; + // TODO: launch query and get output data void* outputData; + pHandle->buffer.output[pos].dst = outputData; + if (pHandle->buffer.firstOffset == -1 + || pReq->offset < pHandle->buffer.firstOffset) { + pHandle->buffer.firstOffset = pReq->offset; + } + if (pHandle->buffer.lastOffset == -1 + || pReq->offset > pHandle->buffer.lastOffset) { + pHandle->buffer.lastOffset = pReq->offset; + } atomic_store_8(&pHandle->buffer.output[pos].status, 1); // put output into rsp @@ -647,16 +658,62 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { return 0; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg* pMsg) { +int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { + STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); + if (pConsumer == NULL) { + return -1; + } + + STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); + if (pTopic == NULL) { + free(pConsumer); + return -1; + } + strcpy(pTopic->topicName, pReq->topicName); + strcpy(pTopic->cgroup, pReq->cgroup); + strcpy(pTopic->sql, pReq->sql); + strcpy(pTopic->logicalPlan, pReq->logicalPlan); + strcpy(pTopic->physicalPlan, pReq->physicalPlan); + SArray *pArray; + //TODO: deserialize to SQueryDag + SQueryDag *pDag; + // convert to task + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + // TODO: handle error + } + ASSERT(taosArrayGetSize(pArray) == 0); + STaskInfo *pInfo = taosArrayGet(pArray, 0); + SArray* pTasks; + schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); + pTopic->buffer.firstOffset = -1; + pTopic->buffer.lastOffset = -1; + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { + SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); + pTopic->buffer.output[i].pMsg = pMsg; + pTopic->buffer.output[i].status = 0; + } + pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); + // write mq meta + return 0; +} + +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) { STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; } pReadHandle->pMeta = pMeta; + pReadHandle->pMsg = NULL; + pReadHandle->ver = -1; + pReadHandle->pColumnIdList = pColumnIdList; + return NULL; +} + +void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { pReadHandle->pMsg = pMsg; tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); - pReadHandle->ver = -1; - return NULL; + pReadHandle->ver = ver; + memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); } bool tqNextDataBlock(STqReadHandle* pHandle) { @@ -676,7 +733,7 @@ int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) // TODO: filter out unused column return 0; } -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) { +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t sversion = pHandle->pBlock->sversion; SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true); STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); @@ -691,11 +748,6 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) { return NULL; } - for (int i = 0; i < pTschema->numOfCols; i++) { - // TODO: filter out unused column - taosArrayPush(pColumnIdList, &(schemaColAt(pTschema, i)->colId)); - } - SMemRow row; int32_t kvIdx; while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 96791488fa..03e123244f 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -58,7 +58,7 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg, pRsp); case TDMT_VND_CONSUME: - return tqProcessConsume(pVnode->pTq, pMsg, pRsp); + return tqProcessConsumeReq(pVnode->pTq, pMsg, pRsp); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 6e2d438970..d1b529f7fb 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnd.h" #include "tq.h" +#include "vnd.h" int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + void *ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -110,43 +110,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_SET_CONN: { - //TODO: wrap in a function - char* reqStr = ptr; SMqSetCVgReq req; - tDecodeSMqSetCVgReq(reqStr, &req); - STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); - - STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); - if (pTopic == NULL) { - // TODO: handle error + tDecodeSMqSetCVgReq(ptr, &req); + if (tqProcessSetConnReq(pVnode->pTq, &req) < 0) { } - strcpy(pTopic->topicName, req.topicName); - strcpy(pTopic->cgroup, req.cGroup); - strcpy(pTopic->sql, req.sql); - strcpy(pTopic->logicalPlan, req.logicalPlan); - strcpy(pTopic->physicalPlan, req.physicalPlan); - SArray *pArray; - //TODO: deserialize to SQueryDag - SQueryDag *pDag; - // convert to task - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - // TODO: handle error - } - ASSERT(taosArrayGetSize(pArray) == 0); - STaskInfo *pInfo = taosArrayGet(pArray, 0); - SArray* pTasks; - schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); - pTopic->buffer.firstOffset = -1; - pTopic->buffer.lastOffset = -1; - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); - pTopic->buffer.output[i].pMsg = pMsg; - pTopic->buffer.output[i].status = 0; - } - pTopic->pReadhandle = walOpenReadHandle(pVnode->pTq->pWal); - // write mq meta - } - break; + } break; default: ASSERT(0); break; From 9fc13dc750f438f11e0b9108fc5f9c993771e3a4 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Jan 2022 18:14:41 +0800 Subject: [PATCH 083/103] [fix memory leak, and temp get tables of vnode into log] --- source/dnode/vnode/src/vnd/vnodeQuery.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 96791488fa..fb36e82dec 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -166,16 +166,26 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { char * name = NULL; int32_t totalLen = 0; + int32_t numOfTables = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { - taosArrayPush(pArray, &name); - totalLen += strlen(name); + if (numOfTables < 1000) { // TODO: temp get tables of vnode, and should del when show tables commad ok. + taosArrayPush(pArray, &name); + totalLen += strlen(name); + } + numOfTables++; + } + + // TODO: temp debug, and should del when show tables command ok + vError("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); + if (numOfTables > 1000) { + numOfTables = 1000; } metaCloseTbCursor(pCur); int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); + //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; @@ -203,5 +213,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); + taosArrayDestroy(pArray); return 0; } From beffd609d23a71924463e6b8a5966e0ff27e770e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Jan 2022 18:18:12 +0800 Subject: [PATCH 084/103] [modify] --- tests/script/sh/massiveTable/deployCluster.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index 4febe40054..beb324321d 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -53,22 +53,22 @@ curr_dir=$(readlink -f "$(dirname "$0")") echo $curr_dir ${curr_dir}/cleanCluster.sh -r "/data" -${curr_dir}/cleanCluster.sh -r "/data2" +#${curr_dir}/cleanCluster.sh -r "/data2" if [[ "${updateSrc}" == "yes" ]]; then ${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" fi ${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000 -${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 +#${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 if [[ "${masterDnode}" == "master" ]]; then # create all dnode into cluster - taos -s "create dnode trd02 port 8000;" + #taos -s "create dnode trd02 port 8000;" taos -s "create dnode trd03 port 7000;" - taos -s "create dnode trd03 port 8000;" + #taos -s "create dnode trd03 port 8000;" taos -s "create dnode trd04 port 7000;" - taos -s "create dnode trd04 port 8000;" + #taos -s "create dnode trd04 port 8000;" fi From c063ea6913f35e9f9bc335d432bdbdca5ebe4823 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 18:38:25 +0800 Subject: [PATCH 085/103] add client --- source/libs/transport/inc/transComm.h | 103 ++++++++--- source/libs/transport/inc/transportInt.h | 3 + source/libs/transport/src/trans.c | 13 +- source/libs/transport/src/transCli.c | 105 ++++++------ source/libs/transport/src/transComm.c | 72 ++++++++ source/libs/transport/src/transSrv.c | 59 ++++--- source/libs/transport/test/uv.c | 207 +++++++++++++++++++++++ 7 files changed, 461 insertions(+), 101 deletions(-) create mode 100644 source/libs/transport/test/uv.c diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 4b14f9f2c7..506b085ecd 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -102,38 +102,101 @@ typedef void* queue[2]; #define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - struct SRpcConn* pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - SEpSet* pSet; // for synchronous API - char msg[0]; // RpcHead starts from here + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + // struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + // int32_t code; // error code + // int16_t numOfTry; // number of try for different servers + // int8_t oldInUse; // server EP inUse passed by app + // int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + char* ip; + uint32_t port; + // SEpSet* pSet; // for synchronous API } SRpcReqContext; -#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) +typedef struct { + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + // struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + // int32_t code; // error code + // int16_t numOfTry; // number of try for different servers + // int8_t oldInUse; // server EP inUse passed by app + // int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + char* ip; + uint32_t port; + // SEpSet* pSet; // for synchronous API +} STransConnCtx; -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#pragma pack(push, 1) + +typedef struct { + char version : 4; // RPC version + char comp : 4; // compression algorithm, 0:no compression 1:lz4 + char resflag : 2; // reserved bits + char spi : 3; // security parameter index + char encrypt : 3; // encrypt algorithm, 0: no encryption + + uint32_t code; // del later + uint32_t msgType; + int32_t msgLen; + uint8_t content[0]; // message body starts from here +} STransMsgHead; + +typedef struct { + int32_t reserved; + int32_t contLen; +} STransCompMsg; + +typedef struct { + uint32_t timeStamp; + uint8_t auth[TSDB_AUTH_LEN]; +} STransDigestMsg; + +#pragma pack(pop) + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) + +#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) #define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) #define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) +#define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) + +#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead) + sizeof(STransDigestMsg)) +#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) +#define transContFromHead(msg) (msg + sizeof(STransMsgHead)) +#define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead)) +#define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); +#define transIsReq(type) (type & 1U) + int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); +int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +bool transCompressMsg(char* msg, int32_t len, int32_t* flen); +bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); + #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index e39e0d9273..3c8c922d83 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -45,6 +45,9 @@ extern "C" { void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void taosCloseServer(void* arg); +void taosCloseClient(void* arg); + typedef struct { int sessions; // number of sessions allowed int numOfThreads; // number of threads to process incoming messages diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index cb8ef87b48..48398316f1 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -24,8 +24,9 @@ typedef struct SConnBuffer { int left; } SConnBuffer; -void* (*taosHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { taosInitServer, taosInitClient}; +void (*taosCloseHandle[])(void* arg) = {taosCloseServer, taosCloseClient}; void* rpcOpen(const SRpcInit* pInit) { SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); @@ -38,11 +39,15 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->connType = pInit->connType; - pRpc->tcphandle = (*taosHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; } -void rpcClose(void* arg) { return; } +void rpcClose(void* arg) { + SRpcInfo* pRpc = (SRpcInfo*)arg; + (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + return; +} void* rpcMallocCont(int contLen) { int size = contLen + RPC_MSG_OVERHEAD; @@ -53,7 +58,7 @@ void* rpcMallocCont(int contLen) { } else { tTrace("malloc mem:%p size:%d", start, size); } - return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); + return start + sizeof(STransMsgHead); } void rpcFreeCont(void* cont) { return; } void* rpcReallocCont(void* ptr, int contLen) { return NULL; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2ecc9e2343..8622db9b3f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -26,10 +26,12 @@ typedef struct SCliConn { char spi; char secured; } SCliConn; + typedef struct SCliMsg { - SRpcReqContext* context; - queue q; - uint64_t st; + STransConnCtx* ctx; + SRpcMsg msg; + queue q; + uint64_t st; } SCliMsg; typedef struct SCliThrdObj { @@ -74,6 +76,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { return; } + // uv_close((uv_handle_t*)handle, clientDestroy); } @@ -101,10 +104,12 @@ static void clientWriteCb(uv_write_t* req, int status) { } static void clientWrite(SCliConn* pConn) { - SCliMsg* pMsg = pConn->data; - SRpcHead* pHead = rpcHeadFromCont(pMsg->context->pCont); - int msgLen = rpcMsgLenFromCont(pMsg->context->contLen); - char* msg = (char*)(pHead); + SCliMsg* pCliMsg = pConn->data; + SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + + int msgLen = transMsgLenFromCont(pMsg->contLen); + char* msg = (char*)(pHead); uv_buf_t wb = uv_buf_init(msg, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); @@ -118,17 +123,17 @@ static void clientConnCb(uv_connect_t* req, int status) { return; } - SCliMsg* pMsg = pConn->data; - SEpSet* pEpSet = &pMsg->context->epSet; - SRpcMsg rpcMsg; + SCliMsg* pMsg = pConn->data; + STransConnCtx* pCtx = ((SCliMsg*)(pConn->data))->ctx; + + SRpcMsg rpcMsg; + rpcMsg.ahandle = pCtx->ahandle; - char* fqdn = pEpSet->fqdn[pEpSet->inUse]; - uint32_t port = pEpSet->port[pEpSet->inUse]; if (status != 0) { // call user fp later - tError("failed to connect server(%s, %d), errmsg: %s", fqdn, port, uv_strerror(status)); - SRpcInfo* pRpc = pMsg->context->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, pEpSet); + tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); + SRpcInfo* pRpc = pMsg->ctx->pRpc; + (pRpc->cfp)(NULL, &rpcMsg, NULL); uv_close((uv_handle_t*)req->handle, clientDestroy); return; } @@ -146,22 +151,18 @@ static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) } static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SEpSet* pEpSet = &pMsg->context->epSet; - - char* fqdn = pEpSet->fqdn[pEpSet->inUse]; - uint32_t port = pEpSet->port[pEpSet->inUse]; - - uint64_t el = taosGetTimestampUs() - pMsg->st; + uint64_t et = taosGetTimestampUs(); + uint64_t el = et - pMsg->st; tDebug("msg tran time cost: %" PRIu64 "", el); + et = taosGetTimestampUs(); - SCliConn* conn = getConnFromCache(pThrd->cache, fqdn, port); + STransConnCtx* pCtx = pMsg->ctx; + SCliConn* conn = getConnFromCache(pThrd->cache, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later conn->data = pMsg; conn->writeReq->data = conn; clientWrite(conn); - // uv_buf_t wb; - // uv_write(conn->writeReq, (uv_stream_t*)conn->stream, &wb, 1, clientWriteCb); } else { SCliConn* conn = malloc(sizeof(SCliConn)); @@ -171,23 +172,11 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn->connReq.data = conn; conn->data = pMsg; + struct sockaddr_in addr; - uv_ip4_addr(fqdn, port, &addr); + uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); - - // SRpcMsg rpcMsg; - // SEpSet* pEpSet = &pMsg->context->epSet; - // SRpcInfo* pRpc = pMsg->context->pRpc; - //// rpcMsg.ahandle = pMsg->context->ahandle; - // rpcMsg.pCont = NULL; - // rpcMsg.ahandle = pMsg->context->ahandle; - // uint64_t el1 = taosGetTimestampUs() - et; - // tError("msg tran back first: time cost: %" PRIu64 "", el1); - // et = taosGetTimestampUs(); - //(pRpc->cfp)(NULL, &rpcMsg, pEpSet); - // uint64_t el2 = taosGetTimestampUs() - et; - // tError("msg tran back second: time cost: %" PRIu64 "", el2); } } static void clientAsyncCb(uv_async_t* handle) { @@ -204,7 +193,8 @@ static void clientAsyncCb(uv_async_t* handle) { while (!QUEUE_IS_EMPTY(&wq)) { queue* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); - pMsg = QUEUE_DATA(h, SCliMsg, q); + + SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); clientHandleReq(pMsg, pThrd); count++; if (count >= 2) { @@ -220,6 +210,7 @@ static void* clientThread(void* arg) { void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SClientObj* cli = calloc(1, sizeof(SClientObj)); + memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); @@ -244,22 +235,31 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, } return cli; } +void taosCloseClient(void* arg) { + // impl later + SClientObj* cli = arg; +} void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later + char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]); + uint32_t port = pEpSet->port[pEpSet->inUse]; + SRpcInfo* pRpc = (SRpcInfo*)shandle; int len = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); - SRpcReqContext* pContext; - pContext = (SRpcReqContext*)((char*)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); - pContext->ahandle = pMsg->ahandle; - pContext->pRpc = (SRpcInfo*)shandle; - pContext->epSet = *pEpSet; - pContext->contLen = len; - pContext->pCont = pMsg->pCont; - pContext->msgType = pMsg->msgType; - pContext->oldInUse = pEpSet->inUse; + STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + + pCtx->pRpc = (SRpcInfo*)shandle; + pCtx->ahandle = pMsg->ahandle; + // pContext->contLen = len; + // pContext->pCont = pMsg->pCont; + pCtx->msgType = pMsg->msgType; + pCtx->ip = strdup(ip); + pCtx->port = port; + // pContext->epSet = *pEpSet; + // pContext->oldInUse = pEpSet->inUse; assert(pRpc->connType == TAOS_CONN_CLIENT); // atomic or not @@ -267,14 +267,15 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* if (pRpc->index++ >= pRpc->numOfThreads) { pRpc->index = 0; } - SCliMsg* msg = malloc(sizeof(SCliMsg)); - msg->context = pContext; - msg->st = taosGetTimestampUs(); + SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); + cliMsg->ctx = pCtx; + cliMsg->msg = *pMsg; + cliMsg->st = taosGetTimestampUs(); SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; pthread_mutex_lock(&thrd->msgMtx); - QUEUE_PUSH(&thrd->msg, &msg->q); + QUEUE_PUSH(&thrd->msg, &cliMsg->q); pthread_mutex_unlock(&thrd->msgMtx); uv_async_send(thrd->cliAsync); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f23cfb6e2d..27cff80586 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -30,6 +30,20 @@ int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { return ret; } +int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + int ret = -1; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; + + return ret; +} void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { T_MD5_CTX context; @@ -41,6 +55,17 @@ void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { memcpy(pAuth, context.digest, sizeof(context.digest)); } +void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + memcpy(pAuth, context.digest, sizeof(context.digest)); +} int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { SRpcHead* pHead = rpcHeadFromCont(pCont); @@ -81,6 +106,53 @@ int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { return finalLen; } +bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { + // SRpcHead* pHead = rpcHeadFromCont(pCont); + bool succ = false; + int overhead = sizeof(STransCompMsg); + if (!NEEDTO_COMPRESSS_MSG(len)) { + return succ; + } + + char* buf = malloc(len + overhead + 8); // 8 extra bytes + if (buf == NULL) { + tError("failed to allocate memory for rpc msg compression, contLen:%d", len); + *flen = len; + return succ; + } + + int32_t clen = LZ4_compress_default(msg, buf, len, len + overhead); + tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", len, clen, overhead); + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (clen > 0 && clen < len - overhead) { + STransCompMsg* pComp = (STransCompMsg*)msg; + pComp->reserved = 0; + pComp->contLen = htonl(len); + memcpy(msg + overhead, buf, clen); + + tDebug("compress rpc msg, before:%d, after:%d", len, clen); + *flen = clen + overhead; + succ = true; + } else { + *flen = len; + succ = false; + } + free(buf); + return succ; +} +bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { + // impl later + return false; + STransCompMsg* pComp = (STransCompMsg*)msg; + + int overhead = sizeof(STransCompMsg); + int clen = 0; + return false; +} + SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { int overhead = sizeof(SRpcComp); SRpcHead* pNewHead = NULL; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index bc4cc695b0..46caffd93c 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -107,24 +107,24 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b SConn* conn = handle->data; SConnBuffer* pBuf = &conn->connBuf; if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); + pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); pBuf->len = 0; pBuf->cap = CAPACITY; pBuf->left = -1; - buf->base = pBuf->buf + RPC_RESERVE_SIZE; + buf->base = pBuf->buf; buf->len = CAPACITY; } else { if (pBuf->len >= pBuf->cap) { if (pBuf->left == -1) { pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); + pBuf->buf = realloc(pBuf->buf, pBuf->cap); } else if (pBuf->len + pBuf->left > pBuf->cap) { pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); } } - buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; + buf->base = pBuf->buf + pBuf->len; buf->len = pBuf->cap - pBuf->len; } } @@ -133,11 +133,12 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b // static bool readComplete(SConnBuffer* data) { // TODO(yihao): handle pipeline later - SRpcHead rpcHead; - int32_t headLen = sizeof(rpcHead); + // SRpcHead rpcHead; + STransMsgHead head; + int32_t headLen = sizeof(head); if (data->len >= headLen) { - memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen); - int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + memcpy((char*)&head, data->buf, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); if (msgLen > data->len) { data->left = msgLen - data->len; return false; @@ -150,21 +151,21 @@ static bool readComplete(SConnBuffer* data) { } static void uvDoProcess(SRecvInfo* pRecv) { - SRpcHead* pHead = (SRpcHead*)pRecv->msg; - SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; - SConn* pConn = pRecv->thandle; - + // impl later + STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; + SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; + SConn* pConn = pRecv->thandle; tDump(pRecv->msg, pRecv->msgLen); - terrno = 0; - SRpcReqContext* pContest; + // SRpcReqContext* pContest; // do auth and check } static int uvAuthMsg(SConn* pConn, char* msg, int len) { - SRpcHead* pHead = (SRpcHead*)msg; - int code = 0; + STransMsgHead* pHead = (STransMsgHead*)msg; + + int code = 0; if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { // secured link, or no authentication @@ -224,7 +225,7 @@ static void uvProcessData(SConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->connBuf; - p->msg = pBuf->buf + RPC_RESERVE_SIZE; + p->msg = pBuf->buf; p->msgLen = pBuf->len; p->ip = 0; p->port = 0; @@ -233,11 +234,10 @@ static void uvProcessData(SConn* pConn) { p->chandle = NULL; // - SRpcHead* pHead = (SRpcHead*)p->msg; - assert(rpcIsReq(pHead->msgType)); + STransMsgHead* pHead = (STransMsgHead*)p->msg; + assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; - pConn->ahandle = (void*)pHead->ahandle; // auth here int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); @@ -247,14 +247,19 @@ static void uvProcessData(SConn* pConn) { } pHead->code = htonl(pHead->code); + int32_t dlen = 0; SRpcMsg rpcMsg; - - pHead = rpcDecompressRpcMsg(pHead); + if (transDecompressMsg(NULL, 0, NULL)) { + // add compress later + // pHead = rpcDecompressRpcMsg(pHead); + } else { + // impl later + } rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; rpcMsg.code = pHead->code; - rpcMsg.ahandle = pConn->ahandle; + rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); @@ -423,7 +428,7 @@ void* workerThread(void* arg) { uv_loop_init(pThrd->loop); // SRpcInfo* pRpc = pThrd->shandle; - uv_pipe_init(pThrd->loop, pThrd->pipe, 1); + uv_pipe_init(pThrd->loop, pThrd->pipe, 0); uv_pipe_open(pThrd->pipe, pThrd->fd); pThrd->pipe->data = pThrd; @@ -522,6 +527,10 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, return srv; } +void taosCloseServer(void* arg) { + // impl later + SServerObj* srv = arg; +} void rpcSendResponse(const SRpcMsg* pMsg) { SConn* pConn = pMsg->handle; diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c new file mode 100644 index 0000000000..4c7d30900b --- /dev/null +++ b/source/libs/transport/test/uv.c @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include + +#include "task.h" +#include + +#define NUM_OF_THREAD 1 +#define TIMEOUT 10000 + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t *pipe; + uv_loop_t *loop; + uv_async_t *workerAsync; // + int fd; +} SThreadObj; + +typedef struct SServerObj { + uv_tcp_t server; + uv_loop_t *loop; + int workerIdx; + int numOfThread; + SThreadObj **pThreadObj; + uv_pipe_t **pipe; +} SServerObj; + +typedef struct SConnCtx { + uv_tcp_t *pClient; + uv_timer_t *pTimer; + uv_async_t *pWorkerAsync; + int ref; +} SConnCtx; + +void echo_write(uv_write_t *req, int status) { + if (status < 0) { + fprintf(stderr, "Write error %s\n", uv_err_name(status)); + } + printf("write data to client\n"); + free(req); +} + +void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + + SConnCtx *pConn = container_of(client, SConnCtx, pClient); + pConn->ref += 1; + printf("read data %d\n", nread, buf->base, buf->len); + if (nread > 0) { + uv_write_t *req = (uv_write_t *)malloc(sizeof(uv_write_t)); + // dispatch request to database other process thread + // just write out + uv_buf_t write_out; + write_out.base = buf->base; + write_out.len = nread; + uv_write((uv_write_t *)req, client, &write_out, 1, echo_write); + free(buf->base); + return; + } + + if (nread < 0) { + if (nread != UV_EOF) + fprintf(stderr, "Read error %s\n", uv_err_name(nread)); + uv_close((uv_handle_t *)client, NULL); + } + free(buf->base); +} + +void alloc_buffer(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { + buf->base = malloc(suggested_size); + buf->len = suggested_size; +} + +void on_new_connection(uv_stream_t *s, int status) { + if (status == -1) { + // error! + return; + } + SServerObj *pObj = container_of(s, SServerObj, server); + printf("new_connection from client\n"); + + uv_tcp_t *client = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, client); + if (uv_accept(s, (uv_stream_t *)client) == 0) { + uv_write_t *write_req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_buf_t dummy_buf = uv_buf_init("a", 1); + // despatch to worker thread + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + uv_write2(write_req, (uv_stream_t *)&(pObj->pipe[pObj->workerIdx][0]), + &dummy_buf, 1, (uv_stream_t *)client, echo_write); + } else { + uv_close((uv_handle_t *)client, NULL); + } +} +void child_on_new_connection(uv_stream_t *q, ssize_t nread, + const uv_buf_t *buf) { + printf("x child_on_new_connection \n"); + if (nread < 0) { + if (nread != UV_EOF) + fprintf(stderr, "Read error %s\n", uv_err_name(nread)); + uv_close((uv_handle_t *)q, NULL); + return; + } + SThreadObj *pObj = (SThreadObj *)container_of(q, struct SThreadObj, pipe); + + uv_pipe_t *pipe = (uv_pipe_t *)q; + if (!uv_pipe_pending_count(pipe)) { + fprintf(stderr, "No pending count\n"); + return; + } + + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SConnCtx *pConn = malloc(sizeof(SConnCtx)); + + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pObj->loop, pConn->pTimer); + + pConn->pClient = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + pConn->pWorkerAsync = pObj->workerAsync; // thread safty + uv_tcp_init(pObj->loop, pConn->pClient); + + if (uv_accept(q, (uv_stream_t *)(pConn->pClient)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t *)pConn->pClient, &fd); + fprintf(stderr, "Worker Accepted fd %d\n", fd); + uv_timer_start(pConn->pTimer, timeOutCallBack, TIMEOUT, 0); + uv_read_start((uv_stream_t *)(pConn->pClient), alloc_buffer, echo_read); + } else { + uv_timer_stop(pConn->pTimer); + free(pConn->pTimer); + uv_close((uv_handle_t *)pConn->pClient, NULL); + free(pConn->pClient); + free(pConn); + } +} + +static void workerAsyncCallback(uv_async_t *handle) { + SThreadObj *pObj = container_of(handle, SThreadObj, workerAsync); + // do nothing +} +void *worker_thread(void *arg) { + SThreadObj *pObj = (SThreadObj *)arg; + int fd = pObj->fd; + pObj->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + uv_loop_init(pObj->loop); + + uv_pipe_init(pObj->loop, pObj->pipe, 1); + uv_pipe_open(pObj->pipe, fd); + + pObj->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCallback); + uv_read_start((uv_stream_t *)pObj->pipe, alloc_buffer, + child_on_new_connection); + + uv_run(pObj->loop, UV_RUN_DEFAULT); +} +int main() { + + SServerObj *server = calloc(1, sizeof(SServerObj)); + server->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + server->numOfThread = NUM_OF_THREAD; + server->workerIdx = 0; + server->pThreadObj = + (SThreadObj **)calloc(server->numOfThread, sizeof(SThreadObj *)); + server->pipe = (uv_pipe_t **)calloc(server->numOfThread, sizeof(uv_pipe_t *)); + + uv_loop_init(server->loop); + + for (int i = 0; i < server->numOfThread; i++) { + server->pThreadObj[i] = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + server->pipe[i] = (uv_pipe_t *)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, + UV_NONBLOCK_PIPE) != 0) { + return -1; + } + uv_pipe_init(server->loop, &(server->pipe[i][0]), 1); + uv_pipe_open(&(server->pipe[i][0]), fds[1]); // init write + + server->pThreadObj[i]->fd = fds[0]; + server->pThreadObj[i]->pipe = &(server->pipe[i][1]); // init read + int err = pthread_create(&(server->pThreadObj[i]->thread), NULL, + worker_thread, (void *)(server->pThreadObj[i])); + if (err == 0) { + printf("thread %d create\n", i); + } else { + printf("thread %d create failed", i); + } + + uv_tcp_init(server->loop, &server->server); + struct sockaddr_in bind_addr; + uv_ip4_addr("0.0.0.0", 7000, &bind_addr); + uv_tcp_bind(&server->server, (const struct sockaddr *)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t *)&server->server, 128, + on_new_connection)) != 0) { + fprintf(stderr, "Listen error %s\n", uv_err_name(err)); + return 2; + } + uv_run(server->loop, UV_RUN_DEFAULT); + return 0; + } +} From 8adf6d9bb1a901ad579f2941dd0b99493412c0d9 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Jan 2022 18:59:01 +0800 Subject: [PATCH 086/103] [modify] --- tests/test/c/create_table.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index d387bf483b..559367a275 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -26,6 +26,7 @@ char dbName[32] = "db"; char stbName[64] = "st"; int32_t numOfThreads = 1; int64_t numOfTables = 200000; +int64_t startOffset = 0; int32_t createTable = 1; int32_t insertData = 0; int32_t batchNumOfTbl = 100; @@ -84,7 +85,7 @@ void createDbAndStb() { } taos_free_result(pRes); - sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName); + sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j int)", stbName); pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { @@ -296,6 +297,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); printf("%s%s\n", indent, "-n"); printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-o"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset); printf("%s%s\n", indent, "-v"); printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups); printf("%s%s\n", indent, "-a"); @@ -329,6 +332,8 @@ void parseArgument(int32_t argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTables = atoll(argv[++i]); + } else if (strcmp(argv[i], "-o") == 0) { + startOffset = atoll(argv[++i]); } else if (strcmp(argv[i], "-v") == 0) { numOfVgroups = atoi(argv[++i]); } else if (strcmp(argv[i], "-a") == 0) { @@ -351,7 +356,8 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s dbName:%s %s", GREEN, dbName, NC); pPrint("%s stbName:%s %s", GREEN, stbName, NC); pPrint("%s configDir:%s %s", GREEN, configDir, NC); - pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC); + pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC); + pPrint("%s startOffset:%" PRId64 " %s", GREEN, startOffset, NC); pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); pPrint("%s createTable:%d %s", GREEN, createTable, NC); @@ -381,7 +387,7 @@ int32_t main(int32_t argc, char *argv[]) { createDbAndStb(); } - pPrint("%d threads are spawned to create %" PRId64 " tables", numOfThreads, numOfTables); + pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); pthread_attr_t thattr; pthread_attr_init(&thattr); @@ -406,8 +412,8 @@ int32_t main(int32_t argc, char *argv[]) { int64_t tableFrom = 0; for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = tableFrom; - pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + pInfo[i].tableBeginIndex = tableFrom + startOffset; + pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1) + startOffset; tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; pInfo[i].minDelay = INT64_MAX; From 37dcc3deb5827554740095e95ecb345a6a68cdf9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 19:19:00 +0800 Subject: [PATCH 087/103] refactor rpc --- source/libs/transport/test/rsclient.c | 196 ++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 source/libs/transport/test/rsclient.c diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c new file mode 100644 index 0000000000..65170d4abb --- /dev/null +++ b/source/libs/transport/test/rsclient.c @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#include "os.h" +#include "tutil.h" +#include "tglobal.h" +#include "rpcLog.h" +#include "trpc.h" +#include "taoserror.h" + +typedef struct { + int index; + SRpcEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t *pOverSem; + pthread_t thread; + void *pRpc; +} SInfo; + + +static int tcount = 0; +static int terror = 0; + +static void *sendRequest(void *param) { + SInfo *pInfo = (SInfo *)param; + SRpcMsg rpcMsg, rspMsg; + + tDebug("thread:%d, start to send request", pInfo->index); + + while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { + pInfo->num++; + rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); + rpcMsg.contLen = pInfo->msgSize; + rpcMsg.handle = pInfo; + rpcMsg.msgType = 1; + tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + + rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg); + + // handle response + if (rspMsg.code != 0) terror++; + + tDebug("thread:%d, rspLen:%d code:%d", pInfo->index, rspMsg.contLen, rspMsg.code); + + rpcFreeCont(rspMsg.pCont); + + if ( pInfo->num % 20000 == 0 ) + tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + } + + tDebug("thread:%d, it is over", pInfo->index); + tcount++; + + return NULL; +} + +int main(int argc, char *argv[]) { + SRpcInit rpcInit; + SRpcEpSet epSet; + int msgSize = 128; + int numOfReqs = 0; + int appThreads = 1; + char serverIp[40] = "127.0.0.1"; + char secret[TSDB_KEY_LEN] = "mypassword"; + struct timeval systemTime; + int64_t startTime, endTime; + pthread_attr_t thattr; + + // server info + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.port[0] = 7000; + epSet.port[1] = 7000; + strcpy(epSet.fqdn[0], serverIp); + strcpy(epSet.fqdn[1], "192.168.0.1"); + + // client info + memset(&rpcInit, 0, sizeof(rpcInit)); + //rpcInit.localIp = "0.0.0.0"; + rpcInit.localPort = 0; + rpcInit.label = "APP"; + rpcInit.numOfThreads = 1; + rpcInit.sessions = 100; + rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.user = "michael"; + rpcInit.secret = secret; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.connType = TAOS_CONN_CLIENT; + + for (int i=1; iindex = i; + pInfo->epSet = epSet; + pInfo->numOfReqs = numOfReqs; + pInfo->msgSize = msgSize; + tsem_init(&pInfo->rspSem, 0, 0); + pInfo->pRpc = pRpc; + pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); + pInfo++; + } + + do { + usleep(1); + } while ( tcount < appThreads); + + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime)/1000.0; // mseconds + + tInfo("it takes %.3f mseconds to send %d requests to server, error num:%d", usedTime, numOfReqs*appThreads, terror); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); + + taosCloseLog(); + + return 0; +} + + From 36bfc3763b16861a630c77a3f864a4f1e7ef9b13 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 20 Jan 2022 19:34:46 +0800 Subject: [PATCH 088/103] feature/qnode --- include/common/tmsg.h | 1 + include/libs/executor/executor.h | 7 + include/libs/qcom/query.h | 5 + include/util/taoserror.h | 1 + source/libs/executor/src/executorMain.c | 13 + source/libs/qworker/inc/qworkerInt.h | 15 +- source/libs/qworker/inc/qworkerMsg.h | 2 +- source/libs/qworker/src/qworker.c | 342 ++++++++++------ source/libs/qworker/src/qworkerMsg.c | 4 +- source/libs/qworker/test/CMakeLists.txt | 2 +- source/libs/qworker/test/qworkerTests.cpp | 464 ++++++++++++++++++---- source/libs/scheduler/src/scheduler.c | 2 + source/util/src/terror.c | 1 + 13 files changed, 637 insertions(+), 222 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 15daa97dbf..11920cef70 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -875,6 +875,7 @@ typedef struct SSubQueryMsg { uint64_t sId; uint64_t queryId; uint64_t taskId; + int8_t taskType; uint32_t contentLen; char msg[]; } SSubQueryMsg; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 0fc7fd679e..61970ff440 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -84,6 +84,13 @@ void* qGetResultRetrieveMsg(qTaskInfo_t qinfo); */ int32_t qKillTask(qTaskInfo_t qinfo); +/** + * kill the ongoing query asynchronously + * @param qinfo qhandle + * @return + */ +int32_t qAsyncKillTask(qTaskInfo_t qinfo); + /** * return whether query is completed or not * @param qinfo diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3d5c74d093..02207c4d1b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -38,6 +38,11 @@ enum { JOB_TASK_STATUS_FREEING, }; +enum { + TASK_TYPE_PERSISTENT = 1, + TASK_TYPE_TEMP, +}; + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e93577e620..570c1d8375 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -362,6 +362,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation") #define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error") #define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) //"Job freed") +#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B) //"Task status error") // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 56e2977753..09d0161e4c 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -317,6 +317,19 @@ int32_t qKillTask(qTaskInfo_t qinfo) { return TSDB_CODE_SUCCESS; } +int32_t qAsyncKillTask(qTaskInfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("QInfo:0x%"PRIx64" query async killed", pQInfo->qId); + setQueryKilled(pQInfo); + + return TSDB_CODE_SUCCESS; +} + int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 7d043a0e02..5f9b33f7e3 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -84,6 +84,7 @@ typedef struct SQWMsg { typedef struct SQWPhaseInput { int8_t status; + int8_t taskType; int32_t code; qTaskInfo_t taskHandle; DataSinkHandle sinkHandle; @@ -91,8 +92,7 @@ typedef struct SQWPhaseInput { typedef struct SQWPhaseOutput { int32_t rspCode; - bool needStop; - bool needRsp; + bool needStop; } SQWPhaseOutput; @@ -104,10 +104,15 @@ typedef struct SQWTaskStatus { typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; + int8_t taskType; + void *readyConnection; + void *dropConnection; + void *cancelConnection; + bool emptyRes; int8_t queryContinue; - int8_t inQueue; + int8_t queryInQueue; int32_t rspCode; int8_t events[QW_EVENT_MAX]; @@ -170,6 +175,10 @@ typedef struct SQWorkerMgmt { #define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG_E(param) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) + #define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index 3b5f3b1605..7ecc2b2b20 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -23,7 +23,7 @@ extern "C" { #include "qworkerInt.h" #include "dataSinkMgt.h" -int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); +int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg, int8_t taskType); int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index c6e2bad421..1cb719c667 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -141,7 +141,8 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, u } else if (QW_NOT_EXIST_RET_ERR == nOpt) { QW_RET(TSDB_CODE_QRY_SCH_NOT_EXIST); } else { - assert(0); + QW_TASK_ELOG("unknown notExistOpt:%d", nOpt); + QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } } @@ -260,7 +261,7 @@ int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tI *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { - QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); + QW_TASK_ELOG("ctx not in ctxHash, ctxHashSize:%d", taosHashGetSize(mgmt->ctxHash)); QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); } @@ -271,8 +272,6 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - printf("%"PRIx64", tid:%"PRIx64"\n", qId, tId); - SQWTaskCtx nctx = {0}; QW_LOCK(QW_WRITE, &mgmt->ctxLock); @@ -324,11 +323,11 @@ void qwReleaseTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->ctxLock); } -void qwFreeTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { +void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { // RC WARNING - qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); - if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { - qDestroyTask(taskHandle); + qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); + if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + qDestroyTask(otaskHandle); } } @@ -337,7 +336,7 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { // RC WARNING qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { - code = qKillTask(taskHandle); + code = qAsyncKillTask(taskHandle); atomic_store_ptr(&ctx->taskHandle, taskHandle); } @@ -346,7 +345,7 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { - qwFreeTaskHandle(QW_FPARAMS(), ctx); + qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); if (ctx->sinkHandle) { dsDestroyDataSinker(ctx->sinkHandle); @@ -369,7 +368,7 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t octx = *ctx; if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { - QW_TASK_ELOG("taosHashRemove from ctx hash failed, id:%s", id); + QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); } @@ -380,6 +379,8 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t if (octx.sinkHandle) { dsDestroyDataSinker(octx.sinkHandle); } + + QW_TASK_DLOG_E("task ctx dropped"); return TSDB_CODE_SUCCESS; } @@ -394,23 +395,23 @@ int32_t qwDropTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_SET_QTID(id, qId, tId); if (qwAcquireScheduler(QW_FPARAMS(), QW_WRITE, &sch)) { - QW_TASK_WLOG("scheduler does not exist, id:%s", id); + QW_TASK_WLOG_E("scheduler does not exist"); return TSDB_CODE_SUCCESS; } if (qwAcquireTaskStatus(QW_FPARAMS(), QW_WRITE, sch, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); - QW_TASK_WLOG("task does not exist, id:%s", id); + QW_TASK_WLOG_E("task does not exist"); return TSDB_CODE_SUCCESS; } if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { - QW_TASK_ELOG("taosHashRemove task from hash failed, task:%p", task); + QW_TASK_ELOG_E("taosHashRemove task from hash failed"); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - QW_TASK_DLOG("task dropped, id:%s", id); + QW_TASK_DLOG_E("task status dropped"); _return: @@ -444,7 +445,7 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx *ctx = NULL; bool locked = false; - QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_WRITE, &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -484,7 +485,7 @@ _return: QW_RET(code); } -int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHandle) { +int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; @@ -494,7 +495,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHa while (true) { QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); - code = qExecTask(taskHandle, &pRes, &useconds); + code = qExecTask(*taskHandle, &pRes, &useconds); if (code) { QW_TASK_ELOG("qExecTask failed, code:%x", code); QW_ERR_JRET(code); @@ -502,7 +503,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t taskHandle, DataSinkHandle sinkHa if (NULL == pRes) { QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); + dsEndPut(sinkHandle, useconds); + + if (TASK_TYPE_TEMP == taskType) { + qwFreeTaskHandle(QW_FPARAMS(), taskHandle); + } break; } @@ -608,11 +614,14 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void } -int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { +int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; int8_t status = 0; SQWTaskCtx *ctx = NULL; bool locked = false; + void *readyConnection = NULL; + void *dropConnection = NULL; + void *cancelConnection = NULL; QW_SCH_TASK_DLOG("handle event at phase %d", phase); @@ -620,17 +629,24 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S case QW_PHASE_PRE_QUERY: { QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->phase = phase; + atomic_store_32(&ctx->phase, phase); + atomic_store_8(&ctx->taskType, input->taskType); - assert(!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_ELOG("task already cancelled at wrong phase, phase:%d", phase); + + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + break; + } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - output->needStop = true; - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it locked = false; @@ -664,18 +680,17 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } if (input->code) { - QW_SET_RSP_CODE(ctx, input->code); + output->rspCode = input->code; } - assert(!QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)); - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { output->needStop = true; QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it locked = false; @@ -689,11 +704,9 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { - output->needRsp = true; + readyConnection = ctx->readyConnection; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - - output->rspCode = input->code; } if (!output->needStop) { @@ -701,84 +714,14 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } break; } - case QW_PHASE_PRE_CQUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - ctx->phase = phase; - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task is dropping, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task is cancelling, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } - case QW_PHASE_POST_CQUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (input->code) { - QW_SET_RSP_CODE(ctx, input->code); - } - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task is dropping, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task is cancelling, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } case QW_PHASE_PRE_FETCH: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); locked = true; - ctx->phase = phase; + atomic_store_32(&ctx->phase, phase); if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); @@ -788,13 +731,15 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task is dropping, phase:%d", phase); + QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task is cancelling, phase:%d", phase); + QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { @@ -827,7 +772,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S locked = true; if (input->code) { - QW_SET_RSP_CODE(ctx, input->code); + output->rspCode = input->code; } if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { @@ -838,13 +783,27 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task is dropping, phase:%d", phase); + QW_TASK_WLOG("start to drop task, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_DROPPING; + + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task is cancelling, phase:%d", phase); + QW_TASK_WLOG("start to cancel task, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLING; + + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + cancelConnection = ctx->cancelConnection; } if (ctx->rspCode) { @@ -855,21 +814,127 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int32_t phase, SQWPhaseInput *input, S } break; } + case QW_PHASE_PRE_CQUERY: { + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + atomic_store_32(&ctx->phase, phase); + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + break; + } + case QW_PHASE_POST_CQUERY: { + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + if (input->code) { + output->rspCode = input->code; + } + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("start to drop task, phase:%d", phase); + output->needStop = true; + + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("start to cancel task, phase:%d", phase); + output->needStop = true; + + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + cancelConnection = ctx->cancelConnection; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + break; + } } _return: + if (output->rspCode) { + QW_SET_RSP_CODE(ctx, output->rspCode); + } + if (locked) { - ctx->phase = phase; + atomic_store_32(&ctx->phase, phase); QW_UNLOCK(QW_WRITE, &ctx->lock); } + if (readyConnection) { + qwBuildAndSendReadyRsp(readyConnection, output->rspCode); + QW_TASK_DLOG("ready msg rsped, code:%x", output->rspCode); + } + + if (dropConnection) { + qwBuildAndSendDropRsp(dropConnection, output->rspCode); + QW_TASK_DLOG("drop msg rsped, code:%x", output->rspCode); + } + + if (cancelConnection) { + qwBuildAndSendCancelRsp(cancelConnection, output->rspCode); + QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); + } + + QW_RET(code); } -int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg, int8_t taskType) { int32_t code = 0; bool queryRsped = false; bool needStop = false; @@ -877,6 +942,10 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t rspCode = 0; SQWPhaseInput input = {0}; SQWPhaseOutput output = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + + input.taskType = taskType; QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); @@ -893,15 +962,19 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_TASK_ELOG("task string to subplan failed, code:%x", code); QW_ERR_JRET(code); } - - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); } + + if ((pTaskInfo && NULL == sinkHandle) || (NULL == pTaskInfo && sinkHandle)) { + QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + //TODO OPTIMIZE EMTYP RESULT QUERY RSP TO AVOID FURTHER FETCH QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, code)); QW_TASK_DLOG("query msg rsped, code:%d", code); @@ -909,8 +982,9 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t queryRsped = true; if (pTaskInfo && sinkHandle) { - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), pTaskInfo, sinkHandle)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &pTaskInfo, sinkHandle, taskType)); } + _return: if (code) { @@ -943,11 +1017,6 @@ _return: } QW_ERR_RET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); - - if (queryRsped && output.needRsp) { - qwBuildAndSendReadyRsp(qwMsg->connection, output.rspCode); - QW_TASK_DLOG("ready msg rsped, code:%x", output.rspCode); - } QW_RET(rspCode); } @@ -956,6 +1025,8 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t code = 0; SQWTaskCtx *ctx = NULL; int8_t phase = 0; + bool needRsp = false; + int32_t rspCode = 0; QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); @@ -968,16 +1039,19 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_TASK_DLOG("ready msg not rsped, phase:%d", phase); } else if (phase == QW_PHASE_POST_QUERY) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - QW_ERR_JRET(qwBuildAndSendReadyRsp(qwMsg->connection, ctx->rspCode)); - QW_TASK_DLOG("ready msg rsped, code:%x", ctx->rspCode); + needRsp = true; + rspCode = ctx->rspCode; } else { QW_TASK_ELOG("invalid phase when got ready msg, phase:%d", phase); - assert(0); + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); + needRsp = true; + rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); } _return: - if (code) { + if (code && ctx) { QW_SET_RSP_CODE(ctx, code); } @@ -985,6 +1059,11 @@ _return: QW_UNLOCK(QW_WRITE, &ctx->lock); } + if (needRsp) { + qwBuildAndSendReadyRsp(qwMsg->connection, rspCode); + QW_TASK_DLOG("ready msg rsped, code:%x", rspCode); + } + QW_RET(code); } @@ -1013,12 +1092,11 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - atomic_store_8(&ctx->inQueue, 0); + atomic_store_8(&ctx->queryInQueue, 0); - qTaskInfo_t taskHandle = ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), taskHandle, sinkHandle)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx->taskHandle, sinkHandle, ctx->taskType)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; @@ -1110,10 +1188,10 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t // RC WARNING if (QW_IN_EXECUTOR(ctx)) { atomic_store_8(&ctx->queryContinue, 1); - } else if (0 == atomic_load_8(&ctx->inQueue)) { + } else if (0 == atomic_load_8(&ctx->queryInQueue)) { QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); - atomic_store_8(&ctx->inQueue, 1); + atomic_store_8(&ctx->queryInQueue, 1); QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); } @@ -1197,10 +1275,10 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == mgmt->ctxHash) { + qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); taosHashCleanup(mgmt->schHash); mgmt->schHash = NULL; tfree(mgmt); - qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index dad3a5d73b..feb8fd645e 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -290,7 +290,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_SCH_TASK_DLOG("processQuery start, node:%p", node); - QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg)); + QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); @@ -374,7 +374,9 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; msg->sId = htobe64(msg->sId); + uint64_t sId = msg->sId; SSchedulerStatusRsp *sStatus = NULL; diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt index 6d755ad487..a464486546 100644 --- a/source/libs/qworker/test/CMakeLists.txt +++ b/source/libs/qworker/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( qworkerTest - PUBLIC os util common transport gtest qcom planner qworker + PUBLIC os util common transport gtest qcom planner qworker executor ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 4962eab460..46d80bbd26 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -32,11 +32,75 @@ #include "qworker.h" #include "stub.h" #include "addr_any.h" +#include "executor.h" +#include "dataSinkMgt.h" namespace { -bool testStop = false; +bool qwtTestEnableSleep = true; +bool qwtTestStop = false; +bool qwtTestDeadLoop = true; +int32_t qwtTestMTRunSec = 10; +int32_t qwtTestPrintNum = 100000; +int32_t qwtTestCaseIdx = 0; +int32_t qwtTestCaseNum = 4; + +void qwtInitLogFile() { + const char *defaultLogFileNamePrefix = "taosdlog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } + +} + +void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { + SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); + queryMsg->queryId = htobe64(1); + queryMsg->sId = htobe64(1); + queryMsg->taskId = htobe64(1); + queryMsg->contentLen = htonl(100); + queryRpc->pCont = queryMsg; + queryRpc->contLen = sizeof(SSubQueryMsg) + 100; +} + +void qwtBuildReadyReqMsg(SResReadyReq *readyMsg, SRpcMsg *readyRpc) { + readyMsg->sId = htobe64(1); + readyMsg->queryId = htobe64(1); + readyMsg->taskId = htobe64(1); + readyRpc->pCont = readyMsg; + readyRpc->contLen = sizeof(SResReadyReq); +} + +void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { + fetchMsg->sId = htobe64(1); + fetchMsg->queryId = htobe64(1); + fetchMsg->taskId = htobe64(1); + fetchRpc->pCont = fetchMsg; + fetchRpc->contLen = sizeof(SResFetchReq); +} + +void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { + dropMsg->sId = htobe64(1); + dropMsg->queryId = htobe64(1); + dropMsg->taskId = htobe64(1); + dropRpc->pCont = dropMsg; + dropRpc->contLen = sizeof(STaskDropReq); +} + +void qwtBuildStatusReqMsg(SSchTasksStatusReq *statusMsg, SRpcMsg *statusRpc) { + statusMsg->sId = htobe64(1); + statusRpc->pCont = statusMsg; + statusRpc->contLen = sizeof(SSchTasksStatusReq); + statusRpc->msgType = TDMT_VND_TASKS_STATUS; +} int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { return 0; @@ -48,6 +112,7 @@ int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { void qwtRpcSendResponse(const SRpcMsg *pRsp) { +/* if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; printf("task num:%d\n", rsp->num); @@ -56,9 +121,63 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) { printf("qId:%"PRIx64",tId:%"PRIx64",status:%d\n", task->queryId, task->taskId, task->status); } } +*/ return; } +int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { + int32_t idx = qwtTestCaseIdx % qwtTestCaseNum; + + if (0 == idx) { + *pTaskInfo = qwtTestCaseIdx; + *handle = qwtTestCaseIdx+1; + } else if (1 == idx) { + *pTaskInfo = NULL; + *handle = NULL; + } else if (2 == idx) { + *pTaskInfo = qwtTestCaseIdx; + *handle = NULL; + } else if (3 == idx) { + *pTaskInfo = NULL; + *handle = qwtTestCaseIdx; + } + + ++qwtTestCaseIdx; + + return 0; +} + +int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { + return 0; +} + +int32_t qwtKillTask(qTaskInfo_t qinfo) { + return 0; +} + +void qwtDestroyTask(qTaskInfo_t qHandle) { + +} + + +int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { + return 0; +} + +void qwtEndPut(DataSinkHandle handle, uint64_t useconds) { +} + +void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { +} + +int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { + return 0; +} + +void qwtDestroyDataSinker(DataSinkHandle handle) { + +} + void stubSetStringToPlan() { @@ -74,11 +193,118 @@ void stubSetStringToPlan() { } } +void stubSetExecTask() { + static Stub stub; + stub.set(qExecTask, qwtExecTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qExecTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtExecTask); + } + } +} + + + +void stubSetCreateExecTask() { + static Stub stub; + stub.set(qCreateExecTask, qwtCreateExecTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qCreateExecTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtCreateExecTask); + } + } +} + +void stubSetAsyncKillTask() { + static Stub stub; + stub.set(qAsyncKillTask, qwtKillTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qAsyncKillTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtKillTask); + } + } +} + +void stubSetDestroyTask() { + static Stub stub; + stub.set(qDestroyTask, qwtDestroyTask); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^qDestroyTask$", result); + for (const auto& f : result) { + stub.set(f.second, qwtDestroyTask); + } + } +} + + +void stubSetDestroyDataSinker() { + static Stub stub; + stub.set(dsDestroyDataSinker, qwtDestroyDataSinker); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsDestroyDataSinker$", result); + for (const auto& f : result) { + stub.set(f.second, qwtDestroyDataSinker); + } + } +} + +void stubSetGetDataLength() { + static Stub stub; + stub.set(dsGetDataLength, qwtGetDataLength); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsGetDataLength$", result); + for (const auto& f : result) { + stub.set(f.second, qwtGetDataLength); + } + } +} + +void stubSetEndPut() { + static Stub stub; + stub.set(dsEndPut, qwtEndPut); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsEndPut$", result); + for (const auto& f : result) { + stub.set(f.second, qwtEndPut); + } + } +} + +void stubSetPutDataBlock() { + static Stub stub; + stub.set(dsPutDataBlock, qwtPutDataBlock); + { + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsPutDataBlock$", result); + for (const auto& f : result) { + stub.set(f.second, qwtPutDataBlock); + } + } +} + void stubSetRpcSendResponse() { static Stub stub; stub.set(rpcSendResponse, qwtRpcSendResponse); { - AddrAny any("libplanner.so"); + AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendResponse$", result); for (const auto& f : result) { @@ -87,24 +313,35 @@ void stubSetRpcSendResponse() { } } +void stubSetGetDataBlock() { + static Stub stub; + stub.set(dsGetDataBlock, qwtGetDataBlock); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^dsGetDataBlock$", result); + for (const auto& f : result) { + stub.set(f.second, qwtGetDataBlock); + } + } +} + + void *queryThread(void *param) { SRpcMsg queryRpc = {0}; int32_t code = 0; uint32_t n = 0; void *mockPointer = (void *)0x1; void *mgmt = param; - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - while (!testStop) { - qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + while (!qwtTestStop) { + qwtBuildQueryReqMsg(&queryRpc); + qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + free(queryRpc.pCont); + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("query:%d\n", n); } } @@ -119,16 +356,14 @@ void *readyThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; SResReadyReq readyMsg = {0}; - readyMsg.sId = htobe64(1); - readyMsg.queryId = htobe64(1); - readyMsg.taskId = htobe64(1); - readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyReq); - while (!testStop) { + while (!qwtTestStop) { + qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("ready:%d\n", n); } } @@ -143,16 +378,14 @@ void *fetchThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; SResFetchReq fetchMsg = {0}; - fetchMsg.sId = htobe64(1); - fetchMsg.queryId = htobe64(1); - fetchMsg.taskId = htobe64(1); - fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchReq); - while (!testStop) { + while (!qwtTestStop) { + qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("fetch:%d\n", n); } } @@ -167,16 +400,14 @@ void *dropThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - while (!testStop) { + while (!qwtTestStop) { + qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("drop:%d\n", n); } } @@ -191,16 +422,14 @@ void *statusThread(void *param) { void *mockPointer = (void *)0x1; void *mgmt = param; SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; - while (!testStop) { - statusMsg.sId = htobe64(1); + while (!qwtTestStop) { + qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - usleep(rand()%5); - if (++n % 50000 == 0) { + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { printf("status:%d\n", n); } } @@ -209,6 +438,27 @@ void *statusThread(void *param) { } +void *controlThread(void *param) { + SRpcMsg queryRpc = {0}; + int32_t code = 0; + uint32_t n = 0; + void *mockPointer = (void *)0x1; + void *mgmt = param; + + while (!qwtTestStop) { + qwtBuildQueryReqMsg(&queryRpc); + qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + free(queryRpc.pCont); + if (qwtTestEnableSleep) { + usleep(rand()%5); + } + if (++n % qwtTestPrintNum == 0) { + printf("query:%d\n", n); + } + } + + return NULL; +} @@ -224,6 +474,8 @@ TEST(seqTest, normalCase) { SRpcMsg fetchRpc = {0}; SRpcMsg dropRpc = {0}; SRpcMsg statusRpc = {0}; + + qwtInitLogFile(); SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); queryMsg->queryId = htobe64(1); @@ -262,6 +514,15 @@ TEST(seqTest, normalCase) { stubSetStringToPlan(); stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); @@ -308,6 +569,8 @@ TEST(seqTest, cancelFirst) { SRpcMsg queryRpc = {0}; SRpcMsg dropRpc = {0}; SRpcMsg statusRpc = {0}; + + qwtInitLogFile(); SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); queryMsg->queryId = htobe64(1); @@ -348,7 +611,7 @@ TEST(seqTest, cancelFirst) { ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - ASSERT_EQ(code, 0); + ASSERT_EQ(code, TSDB_CODE_QRY_TASK_DROPPED); statusMsg.sId = htobe64(1); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); @@ -366,44 +629,16 @@ TEST(seqTest, randCase) { SRpcMsg fetchRpc = {0}; SRpcMsg dropRpc = {0}; SRpcMsg statusRpc = {0}; - - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - SResReadyReq readyMsg = {0}; - readyMsg.sId = htobe64(1); - readyMsg.queryId = htobe64(1); - readyMsg.taskId = htobe64(1); - readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyReq); - SResFetchReq fetchMsg = {0}; - fetchMsg.sId = htobe64(1); - fetchMsg.queryId = htobe64(1); - fetchMsg.taskId = htobe64(1); - fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchReq); - STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; + + qwtInitLogFile(); stubSetStringToPlan(); stubSetRpcSendResponse(); + stubSetCreateExecTask(); srand(time(NULL)); @@ -416,20 +651,25 @@ TEST(seqTest, randCase) { int32_t r = rand() % maxr; if (r >= 0 && r < maxr/5) { - printf("Query,%d\n", t++); + printf("Query,%d\n", t++); + qwtBuildQueryReqMsg(&queryRpc); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); + free(queryRpc.pCont); } else if (r >= maxr/5 && r < maxr * 2/5) { printf("Ready,%d\n", t++); + qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); + qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); } else if (r >= maxr * 3/5 && r < maxr * 4/5) { printf("Drop,%d\n", t++); + qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); } else if (r >= maxr * 4/5 && r < maxr-1) { printf("Status,%d\n", t++); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); } else { @@ -445,6 +685,8 @@ TEST(seqTest, multithreadRand) { void *mgmt = NULL; int32_t code = 0; void *mockPointer = (void *)0x1; + + qwtInitLogFile(); stubSetStringToPlan(); stubSetRpcSendResponse(); @@ -464,15 +706,69 @@ TEST(seqTest, multithreadRand) { pthread_create(&(t4), &thattr, dropThread, NULL); pthread_create(&(t5), &thattr, statusThread, NULL); - int32_t t = 0; - int32_t maxr = 10001; - sleep(300); - testStop = true; - sleep(1); + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + sleep(3); qWorkerDestroy(&mgmt); } +TEST(rcTest, multithread) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, controlThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, NULL); + pthread_create(&(t3), &thattr, fetchQueueThread, NULL); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + sleep(3); + + qWorkerDestroy(&mgmt); +} + + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 45c6936b62..161293ce99 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1102,6 +1102,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskType = TASK_TYPE_TEMP; pMsg->contentLen = htonl(pTask->msgLen); memcpy(pMsg->msg, pTask->msg, pTask->msgLen); break; @@ -1487,6 +1488,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(plan->id.queryId); pMsg->taskId = htobe64(schGenUUID()); + pMsg->taskType = TASK_TYPE_PERSISTENT; pMsg->contentLen = htonl(msgLen); memcpy(pMsg->msg, msg, msgLen); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a5dd1483ec..a67d66efb0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -361,6 +361,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPING, "Task dropping") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUPLICATTED_OPERATION, "Duplicatted operation") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED, "Job already freed") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_STATUS_ERROR, "Task status error") From 5e8cb50a20f296fb3f631c13dc4e2f93558faeb4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 20:01:27 +0800 Subject: [PATCH 089/103] refactor rpc --- source/libs/transport/inc/transComm.h | 2 ++ source/libs/transport/src/transCli.c | 24 +++++++++++++++++++----- source/libs/transport/src/transComm.c | 5 +++++ source/libs/transport/src/transSrv.c | 12 ++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 506b085ecd..4d3a1b70cb 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -199,4 +199,6 @@ void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); bool transCompressMsg(char* msg, int32_t len, int32_t* flen); bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); +void transConnCtxDestroy(STransConnCtx* ctx); + #endif diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8622db9b3f..c96a0f81e0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -63,6 +63,8 @@ static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); static void clientConnDestroy(SCliConn* pConn); +static void clientMsgDestroy(SCliMsg* pMsg); + static void* clientThread(void* arg); static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); @@ -235,9 +237,22 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, } return cli; } +static void clientMsgDestroy(SCliMsg* pMsg) { + // impl later + free(pMsg); +} void taosCloseClient(void* arg) { // impl later SClientObj* cli = arg; + for (int i = 0; i < cli->numOfThreads; i++) { + SCliThrdObj* pThrd = cli->pThreadObj[i]; + pthread_join(pThrd->thread, NULL); + pthread_mutex_destroy(&pThrd->msgMtx); + free(pThrd->loop); + free(pThrd); + } + free(cli->pThreadObj); + free(cli); } void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { @@ -247,19 +262,18 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* SRpcInfo* pRpc = (SRpcInfo*)shandle; - int len = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); + int32_t flen = 0; + if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { + // imp later + } STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); pCtx->pRpc = (SRpcInfo*)shandle; pCtx->ahandle = pMsg->ahandle; - // pContext->contLen = len; - // pContext->pCont = pMsg->pCont; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); pCtx->port = port; - // pContext->epSet = *pEpSet; - // pContext->oldInUse = pEpSet->inUse; assert(pRpc->connType == TAOS_CONN_CLIENT); // atomic or not diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 27cff80586..617abeea39 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -107,6 +107,7 @@ int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { } bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { + return false; // SRpcHead* pHead = rpcHeadFromCont(pCont); bool succ = false; int overhead = sizeof(STransCompMsg); @@ -186,4 +187,8 @@ SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { return pHead; } +void transConnCtxDestroy(STransConnCtx* ctx) { + free(ctx->ip); + free(ctx); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 46caffd93c..f4625dc0b3 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -496,6 +496,7 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { @@ -530,6 +531,17 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void taosCloseServer(void* arg) { // impl later SServerObj* srv = arg; + for (int i = 0; i < srv->numOfThreads; i++) { + SWorkThrdObj* pThrd = srv->pThreadObj[i]; + pthread_join(pThrd->thread, NULL); + free(srv->pipe[i]); + free(pThrd->loop); + free(pThrd); + } + free(srv->loop); + free(srv->pipe); + free(srv->pThreadObj); + pthread_join(srv->thread, NULL); } void rpcSendResponse(const SRpcMsg* pMsg) { From f2f218402f5f9d1f1744344ff0ac13a6a9dd03e7 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Jan 2022 20:34:51 +0800 Subject: [PATCH 090/103] [modify] --- tests/test/c/create_table.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 559367a275..ab4ef32854 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -181,9 +181,20 @@ void *threadFunc(void *param) { pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); exit(1); } + + pError("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", + pInfo->threadIndex, + pInfo->tableBeginIndex, + pInfo->tableEndIndex); - // printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - // pInfo->tableEndIndex); + pInfo->tableBeginIndex += startOffset; + pInfo->tableEndIndex += startOffset; + + pError("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", + pInfo->threadIndex, + pInfo->tableBeginIndex, + pInfo->tableEndIndex); + sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); taos_free_result(pRes); @@ -211,7 +222,7 @@ void *threadFunc(void *param) { TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); + pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); } taos_free_result(pRes); int64_t endTs = taosGetTimestampUs(); @@ -297,7 +308,7 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); printf("%s%s\n", indent, "-n"); printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables); - printf("%s%s\n", indent, "-o"); + printf("%s%s\n", indent, "-g"); printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset); printf("%s%s\n", indent, "-v"); printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups); @@ -332,7 +343,7 @@ void parseArgument(int32_t argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTables = atoll(argv[++i]); - } else if (strcmp(argv[i], "-o") == 0) { + } else if (strcmp(argv[i], "-g") == 0) { startOffset = atoll(argv[++i]); } else if (strcmp(argv[i], "-v") == 0) { numOfVgroups = atoi(argv[++i]); @@ -412,8 +423,8 @@ int32_t main(int32_t argc, char *argv[]) { int64_t tableFrom = 0; for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = tableFrom + startOffset; - pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1) + startOffset; + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1); tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; pInfo[i].minDelay = INT64_MAX; From 906e0e785fe2a3cd64e54ab3321e93f0e8f3bc21 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 20:54:08 +0800 Subject: [PATCH 091/103] refactor rpc --- source/libs/transport/inc/transComm.h | 7 +++ source/libs/transport/src/trans.c | 10 +--- source/libs/transport/src/transCli.c | 68 ++++++++++++++++++++++++++- source/libs/transport/src/transSrv.c | 18 +++---- 4 files changed, 81 insertions(+), 22 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 4d3a1b70cb..c760acd52e 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -201,4 +201,11 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); void transConnCtxDestroy(STransConnCtx* ctx); +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; +} SConnBuffer; + #endif diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48398316f1..cf1e153965 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -17,13 +17,6 @@ #include "transComm.h" -typedef struct SConnBuffer { - char* buf; - int len; - int cap; - int left; -} SConnBuffer; - void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = { taosInitServer, taosInitClient}; void (*taosCloseHandle[])(void* arg) = {taosCloseServer, taosCloseClient}; @@ -46,10 +39,11 @@ void* rpcOpen(const SRpcInit* pInit) { void rpcClose(void* arg) { SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + free(pRpc); return; } void* rpcMallocCont(int contLen) { - int size = contLen + RPC_MSG_OVERHEAD; + int size = contLen + TRANS_MSG_OVERHEAD; char* start = (char*)calloc(1, (size_t)size); if (start == NULL) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c96a0f81e0..86e9c05ccb 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -21,6 +21,7 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; uv_write_t* writeReq; + SConnBuffer readBuf; void* data; queue conn; char spi; @@ -55,9 +56,17 @@ typedef struct SClientObj { static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); +// process data read from server, auth/decompress etc +static void clientProcessData(SCliConn* conn); +// check whether already read complete packet from server +static bool clientReadComplete(SConnBuffer* pBuf); +// alloc buf for read static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +// callback after read nbytes from socket static void clientReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +// callback after write data to socket static void clientWriteCb(uv_write_t* req, int status); +// callback after conn to server static void clientConnCb(uv_connect_t* req, int status); static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); @@ -67,18 +76,72 @@ static void clientMsgDestroy(SCliMsg* pMsg); static void* clientThread(void* arg); +static void clientProcessData(SCliConn* conn) { + // impl +} static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static bool clientReadComplete(SConnBuffer* data) { + STransMsgHead head; + int32_t headLen = sizeof(head); + if (data->len >= headLen) { + memcpy((char*)&head, data->buf, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); + if (msgLen > data->len) { + data->left = msgLen - data->len; + return false; + } else { + return true; + } + } else { + return false; + } +} static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { // impl later + static const int CAPACITY = 512; + + SCliConn* conn = handle->data; + SConnBuffer* pBuf = &conn->readBuf; + if (pBuf->cap == 0) { + pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); + pBuf->len = 0; + pBuf->cap = CAPACITY; + pBuf->left = -1; + buf->base = pBuf->buf; + buf->len = CAPACITY; + } else { + if (pBuf->len >= pBuf->cap) { + if (pBuf->left == -1) { + pBuf->cap *= 2; + pBuf->buf = realloc(pBuf->buf, pBuf->cap); + } else if (pBuf->len + pBuf->left > pBuf->cap) { + pBuf->cap = pBuf->len + pBuf->left; + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); + } + } + buf->base = pBuf->buf + pBuf->len; + buf->len = pBuf->cap - pBuf->len; + } } static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // impl later - SCliConn* conn = handle->data; + SCliConn* conn = handle->data; + SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { + pBuf->len += nread; + if (clientReadComplete(pBuf)) { + tDebug("alread read complete pack"); + clientProcessData(conn); + } else { + tDebug("read halp packet, continue to read"); + } return; } + if (nread != UV_EOF) { + tDebug("Read error %s\n", uv_err_name(nread)); + } // uv_close((uv_handle_t*)handle, clientDestroy); } @@ -166,7 +229,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn->writeReq->data = conn; clientWrite(conn); } else { - SCliConn* conn = malloc(sizeof(SCliConn)); + SCliConn* conn = calloc(1, sizeof(SCliConn)); conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); @@ -248,6 +311,7 @@ void taosCloseClient(void* arg) { SCliThrdObj* pThrd = cli->pThreadObj[i]; pthread_join(pThrd->thread, NULL); pthread_mutex_destroy(&pThrd->msgMtx); + free(pThrd->cliAsync); free(pThrd->loop); free(pThrd); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index f4625dc0b3..d096ab7813 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -16,13 +16,6 @@ #ifdef USE_UV #include "transComm.h" -typedef struct SConnBuffer { - char* buf; - int len; - int cap; - int left; -} SConnBuffer; - typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -100,7 +93,8 @@ static void* acceptThread(void* arg); void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { /* * formate of data buffer: - * |<-------SRpcReqContext------->|<------------data read from socket----------->| + * |<--------------------------data from socket------------------------------->| + * |<------STransMsgHead------->|<-------------------other data--------------->| */ static const int CAPACITY = 1024; @@ -133,7 +127,6 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b // static bool readComplete(SConnBuffer* data) { // TODO(yihao): handle pipeline later - // SRpcHead rpcHead; STransMsgHead head; int32_t headLen = sizeof(head); if (data->len >= headLen) { @@ -270,13 +263,13 @@ static void uvProcessData(SConn* pConn) { void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // opt - SConn* ctx = cli->data; - SConnBuffer* pBuf = &ctx->connBuf; + SConn* conn = cli->data; + SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; if (readComplete(pBuf)) { tDebug("alread read complete packet"); - uvProcessData(ctx); + uvProcessData(conn); } else { tDebug("read half packet, continue to read"); } @@ -542,6 +535,7 @@ void taosCloseServer(void* arg) { free(srv->pipe); free(srv->pThreadObj); pthread_join(srv->thread, NULL); + free(srv); } void rpcSendResponse(const SRpcMsg* pMsg) { From bcd6df15ef20a80829f0e3a26f66799191bdf960 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 22:16:07 +0800 Subject: [PATCH 092/103] refactor rpc --- source/libs/transport/src/transCli.c | 63 ++++++++++++++++++++++++---- source/libs/transport/src/transSrv.c | 18 ++++---- source/libs/transport/test/rclient.c | 2 + 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 86e9c05ccb..2b2fcb557f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -52,7 +52,14 @@ typedef struct SClientObj { SCliThrdObj** pThreadObj; } SClientObj; +typedef struct SConnList { + queue conn; +} SConnList; + // conn pool +// add expire timeout and capacity limit +static void* connCacheCreate(int size); +static void* connCacheDestroy(void* cache); static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); @@ -81,6 +88,53 @@ static void clientProcessData(SCliConn* conn) { } static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void* connCacheCreate(int size) { + SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + return false; +} +static void* connCacheDestroy(void* cache) { + SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); + while (!QUEUE_IS_EMPTY(&connList->conn)) { + queue* h = QUEUE_HEAD(&connList->conn); + QUEUE_REMOVE(h); + SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + clientConnDestroy(c); + } + taosHashClear(cache); +} + +static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { + char key[128] = {0}; + tstrncpy(key, ip, strlen(ip)); + tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + + SHashObj* pCache = cache; + SConnList* plist = taosHashGet(pCache, key, strlen(key)); + if (plist == NULL) { + SConnList list; + plist = &list; + QUEUE_INIT(&plist->conn); + taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist)); + } + + if (QUEUE_IS_EMPTY(&plist->conn)) { + return NULL; + } + queue* h = QUEUE_HEAD(&plist->conn); + QUEUE_REMOVE(h); + return QUEUE_DATA(h, SCliConn, conn); +} +static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { + char key[128] = {0}; + tstrncpy(key, ip, strlen(ip)); + tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + + SHashObj* pCache = cache; + SConnList* plist = taosHashGet(pCache, key, strlen(key)); + // list already create before + assert(plist != NULL); + QUEUE_PUSH(&plist->conn, &conn->conn); +} static bool clientReadComplete(SConnBuffer* data) { STransMsgHead head; int32_t headLen = sizeof(head); @@ -206,15 +260,6 @@ static void clientConnCb(uv_connect_t* req, int status) { clientWrite(pConn); } -static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { - // impl later - - return NULL; -} -static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { - // impl later -} - static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { uint64_t et = taosGetTimestampUs(); uint64_t el = et - pMsg->st; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index d096ab7813..4542541043 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -210,8 +210,8 @@ static int uvAuthMsg(SConn* pConn, char* msg, int len) { // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { - // impl later SConn* conn = handle->data; + tDebug("%p timeout since no activity", conn); } static void uvProcessData(SConn* pConn) { @@ -232,12 +232,13 @@ static void uvProcessData(SConn* pConn) { SRpcInfo* pRpc = (SRpcInfo*)p->shandle; // auth here + // auth should not do in rpc thread - int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); - if (code != 0) { - terrno = code; - return; - } + // int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); + // if (code != 0) { + // terrno = code; + // return; + //} pHead->code = htonl(pHead->code); int32_t dlen = 0; @@ -248,7 +249,7 @@ static void uvProcessData(SConn* pConn) { } else { // impl later } - rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); + rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; rpcMsg.code = pHead->code; @@ -318,6 +319,9 @@ void uvWorkerAsyncCb(uv_async_t* handle) { return; } uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); + + uv_timer_stop(conn->pTimer); + uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 6339e58560..4ccbb60cc2 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -40,6 +40,7 @@ static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); + // tsem_post(&pInfo->rspSem); tsem_post(&pInfo->rspSem); } @@ -60,6 +61,7 @@ static void *sendRequest(void *param) { // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); } From 5a3a0f3d6c68b829313ea414eb5fc554e29d9275 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Jan 2022 22:57:29 +0800 Subject: [PATCH 093/103] [td-11818]fix bug in creating table. --- include/libs/planner/plannerOp.h | 1 + source/client/test/clientTests.cpp | 8 +++- source/libs/executor/inc/executorimpl.h | 11 +++-- source/libs/executor/src/executorimpl.c | 62 +++++++++++++++++++++++-- source/libs/parser/src/dCDAstProcess.c | 3 +- source/libs/parser/src/parserUtil.c | 2 +- tests/test/c/create_table.c | 2 +- 7 files changed, 76 insertions(+), 13 deletions(-) diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 5cc896f1c2..2793e72635 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -28,6 +28,7 @@ OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) OP_ENUM_MACRO(SystemTableScan) +OP_ENUM_MACRO(StreamBlockScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) // OP_ENUM_MACRO(Groupby) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 6f9715a09f..440ef0d728 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -650,7 +650,13 @@ TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); + TAOS_RES* pRes = taos_query(pConn, "use dbv"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tx using st tags(111111111111111)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(pRes)); + } taos_free_result(pRes); pRes = taos_query(pConn, "select count(*) from tu"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index dd633002db..2fe3392b25 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -376,7 +376,7 @@ typedef struct STaskParam { typedef struct SExchangeInfo { SArray *pSources; - int32_t bytes; // total load bytes from remote + uint64_t bytes; // total load bytes from remote tsem_t ready; void *pTransporter; SRetrieveTableRsp *pRsp; @@ -385,7 +385,7 @@ typedef struct SExchangeInfo { typedef struct STableScanInfo { void *pTsdbReadHandle; - int32_t numOfBlocks; + int32_t numOfBlocks; // extract basic running information. int32_t numOfSkipped; int32_t numOfBlockStatis; int64_t numOfRows; @@ -415,7 +415,11 @@ typedef struct STagScanInfo { } STagScanInfo; typedef struct SStreamBlockScanInfo { - + SSDataBlock *pRes; // result SSDataBlock + SColumnInfo *pCols; // the output column info + uint64_t numOfRows; // total scanned rows + uint64_t numOfExec; // execution times + void *readerHandle;// stream block reader handle } SStreamBlockScanInfo; typedef struct SOptrBasicInfo { @@ -423,7 +427,6 @@ typedef struct SOptrBasicInfo { int32_t *rowCellInfoOffset; // offset value for each row result cell info SQLFunctionCtx *pCtx; SSDataBlock *pRes; - void *keyBuf; } SOptrBasicInfo; typedef struct SOptrBasicInfo STableIntervalOperatorInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2b5faba29d..dc4d9c7238 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,7 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include +#include "parser.h" +#include "tq.h" #include "exception.h" #include "os.h" #include "tglobal.h" @@ -3576,7 +3577,7 @@ void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t sta SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; int64_t tid = 0; - pInfo->keyBuf = realloc(pInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); + pAggInfo->keyBuf = realloc(pAggInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid, pTaskInfo, false, pAggInfo); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { @@ -5061,6 +5062,42 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { #endif } +static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*)param; + + // NOTE: this operator never check if current status is done or not + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamBlockScanInfo* pInfo = pOperator->info; + + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + while (tqNextDataBlock(pInfo->readerHandle)) { + pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + terrno = pTaskInfo->code; + return NULL; + } + + if (pBlockInfo->rows == 0) { + return NULL; + } + + pInfo->pRes->pDataBlock = tqRetrieveDataBlock(pInfo->readerHandle); + if (pInfo->pRes->pDataBlock == NULL) { + // TODO add log + pTaskInfo->code = terrno; + return NULL; + } + + break; + } + + // record the scan action. + pInfo->numOfExec++; + pInfo->numOfRows += pBlockInfo->rows; + + return (pBlockInfo->rows == 0)? NULL:pInfo->pRes; +} + int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) { SExchangeInfo* pEx = (SExchangeInfo*) param; pEx->pRsp = pMsg->pData; @@ -5263,7 +5300,6 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { tfree(pInfo); tfree(pOperator); @@ -5371,8 +5407,26 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } -SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + pInfo->readerHandle = pStreamBlockHandle; + + pOperator->name = "StreamBlockScanOperator"; + pOperator->operatorType = OP_StreamBlockScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->exec = doStreamBlockScan; + pOperator->pTaskInfo = pTaskInfo; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 662e550655..5852678880 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -680,10 +680,9 @@ int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseConte serializeVgroupTablesBatchImpl(&tbatch, pBufArray); destroyCreateTbReqBatch(&tbatch); - } else { // it is a child table, created according to a super table code = doCheckAndBuildCreateCTableReq(pCreateTable, pCtx, pMsgBuf, &pBufArray); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { return code; } } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 1793c85e30..b8545b7486 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1656,7 +1656,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha } // Remove quotation marks - if (TK_STRING == type) { + if (TSDB_DATA_TYPE_BINARY == type) { if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); } diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index d387bf483b..2dd5f04a58 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -84,7 +84,7 @@ void createDbAndStb() { } taos_free_result(pRes); - sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName); + sprintf(qstr, "create table %s (ts timestamp, i int) tags (j bigint)", stbName); pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { From 930ffd3eb833395f1db1f483f132bec7de1ebb51 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 23:21:41 +0800 Subject: [PATCH 094/103] refactor rpc --- source/libs/transport/src/transCli.c | 69 ++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2b2fcb557f..f2d844f73d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -26,6 +26,7 @@ typedef struct SCliConn { queue conn; char spi; char secured; + uint64_t expireTime; } SCliConn; typedef struct SCliMsg { @@ -39,10 +40,13 @@ typedef struct SCliThrdObj { pthread_t thread; uv_loop_t* loop; uv_async_t* cliAsync; // - void* cache; // conn pool + uv_timer_t* pTimer; + void* cache; // conn pool queue msg; pthread_mutex_t msgMtx; - void* shandle; + uint64_t nextTimeout; // next timeout + void* shandle; // + } SCliThrdObj; typedef struct SClientObj { @@ -63,6 +67,8 @@ static void* connCacheDestroy(void* cache); static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); +// register timer in each thread to clear expire conn +static void clientTimeoutCb(uv_timer_t* handle); // process data read from server, auth/decompress etc static void clientProcessData(SCliConn* conn); // check whether already read complete packet from server @@ -84,21 +90,55 @@ static void clientMsgDestroy(SCliMsg* pMsg); static void* clientThread(void* arg); static void clientProcessData(SCliConn* conn) { + STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; + SRpcInfo* pRpc = pCtx->ahandle; + SRpcMsg rpcMsg; + + rpcMsg.pCont = conn->readBuf.buf; + rpcMsg.contLen = conn->readBuf.len; + rpcMsg.ahandle = pCtx->ahandle; + (pRpc->cfp)(NULL, &rpcMsg, NULL); // impl } static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void clientTimeoutCb(uv_timer_t* handle) { + SCliThrdObj* pThrd = handle->data; + SRpcInfo* pRpc = pThrd->shandle; + int64_t currentTime = pThrd->nextTimeout; + + SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); + while (p != NULL) { + while (!QUEUE_IS_EMPTY(&p->conn)) { + queue* h = QUEUE_HEAD(&p->conn); + SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + if (c->expireTime < currentTime) { + QUEUE_REMOVE(h); + clientConnDestroy(c); + } else { + break; + } + } + p = taosHashIterate((SHashObj*)pThrd->cache, p); + } + + pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + uv_timer_start(handle, clientTimeoutCb, pRpc->idleTime * 10, 0); +} static void* connCacheCreate(int size) { SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); return false; } static void* connCacheDestroy(void* cache) { SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); - while (!QUEUE_IS_EMPTY(&connList->conn)) { - queue* h = QUEUE_HEAD(&connList->conn); - QUEUE_REMOVE(h); - SCliConn* c = QUEUE_DATA(h, SCliConn, conn); - clientConnDestroy(c); + while (connList != NULL) { + while (!QUEUE_IS_EMPTY(&connList->conn)) { + queue* h = QUEUE_HEAD(&connList->conn); + QUEUE_REMOVE(h); + SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + clientConnDestroy(c); + } + connList = taosHashIterate((SHashObj*)cache, connList); } taosHashClear(cache); } @@ -129,8 +169,10 @@ static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); - SHashObj* pCache = cache; - SConnList* plist = taosHashGet(pCache, key, strlen(key)); + STransConnCtx* ctx = ((SCliMsg*)conn->data)->ctx; + SRpcInfo* pRpc = ctx->pRpc; + conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -206,6 +248,7 @@ static void clientConnDestroy(SCliConn* conn) { } static void clientDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; + QUEUE_REMOVE(&conn->conn); clientConnDestroy(conn); } @@ -279,6 +322,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->writeReq = malloc(sizeof(uv_write_t)); + QUEUE_INIT(&conn->conn); conn->connReq.data = conn; conn->data = pMsg; @@ -315,6 +359,9 @@ static void clientAsyncCb(uv_async_t* handle) { static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; + SRpcInfo* pRpc = pThrd->shandle; + pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; + uv_timer_start(pThrd->pTimer, clientTimeoutCb, pRpc->idleTime * 10, 0); uv_run(pThrd->loop, UV_RUN_DEFAULT); } @@ -336,7 +383,11 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); pThrd->cliAsync->data = pThrd; + pThrd->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pThrd->pTimer); + pThrd->shandle = shandle; + int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); if (err == 0) { tDebug("sucess to create tranport-client thread %d", i); From 6c0842fb05fefde4a9096c6ae413841466be96d5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 20 Jan 2022 23:57:21 +0800 Subject: [PATCH 095/103] add index test --- source/libs/index/test/indexTests.cc | 114 ++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 12 deletions(-) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 4f3330b7b3..bbcc654ae2 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -82,7 +82,9 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { return false; } + if (nRead <= 0) { + return false; + } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -108,7 +110,9 @@ class FstReadMemory { StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; - while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { result.push_back((uint64_t)(rt->out.out)); } + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + result.push_back((uint64_t)(rt->out.out)); + } return true; } bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { @@ -184,7 +188,9 @@ void checkFstPerf() { delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); - if (m->init()) { printf("success to init fst read"); } + if (m->init()) { + printf("success to init fst read"); + } Performance_fstReadRecords(m); delete m; } @@ -348,7 +354,9 @@ class TFileObj { tfileReaderDestroy(reader_); reader_ = NULL; } - if (writer_ == NULL) { InitWriter(); } + if (writer_ == NULL) { + InitWriter(); + } return tfileWriterPut(writer_, tv, false); } bool InitWriter() { @@ -388,8 +396,12 @@ class TFileObj { return tfileReaderSearch(reader_, query, result); } ~TFileObj() { - if (writer_) { tfileWriterDestroy(writer_); } - if (reader_) { tfileReaderDestroy(reader_); } + if (writer_) { + tfileWriterDestroy(writer_); + } + if (reader_) { + tfileReaderDestroy(reader_); + } } private: @@ -912,7 +924,8 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { std::string path = "/tmp/cache_and_tfile"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } std::thread threads[NUM_OF_THREAD]; for (int i = 0; i < NUM_OF_THREAD; i++) { @@ -927,14 +940,24 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { TEST_F(IndexEnv2, testIndex_restart) { std::string path = "/tmp/cache_and_tfile"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } + index->SearchOneTarget("tag1", "Hello", 10); + index->SearchOneTarget("tag2", "Test", 10); +} +TEST_F(IndexEnv2, testIndex_restart1) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->ReadMultiMillonData("tag1", "coding"); index->SearchOneTarget("tag1", "Hello", 10); index->SearchOneTarget("tag2", "Test", 10); } TEST_F(IndexEnv2, testIndex_read_performance) { std::string path = "/tmp/cache_and_tfile"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } index->PutOneTarge("tag1", "Hello", 12); index->PutOneTarge("tag1", "Hello", 15); index->ReadMultiMillonData("tag1", "Hello"); @@ -943,17 +966,84 @@ TEST_F(IndexEnv2, testIndex_read_performance) { } TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/multi_tag"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } int64_t st = taosGetTimestampUs(); int32_t num = 1000 * 10000; index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num); std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl; // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); } -TEST_F(IndexEnv2, testLongComVal) { +TEST_F(IndexEnv2, testLongComVal1) { std::string path = "/tmp/long_colVal"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + } // gen colVal by randstr std::string randstr = "xxxxxxxxxxxxxxxxx"; index->WriteMultiMillonData("tag1", randstr, 100 * 10000); } + +TEST_F(IndexEnv2, testLongComVal2) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) { + } + // gen colVal by randstr + std::string randstr = "abcccc fdadfafdafda"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); +} +TEST_F(IndexEnv2, testLongComVal3) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) { + } + // gen colVal by randstr + std::string randstr = "Yes, coding and coding and coding"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); +} +TEST_F(IndexEnv2, testLongComVal4) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) { + } + // gen colVal by randstr + std::string randstr = "111111 bac fdadfa"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); +} +TEST_F(IndexEnv2, testIndex_read_performance1) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); + index->ReadMultiMillonData("tag1", "Hello", 1000); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); +} +TEST_F(IndexEnv2, testIndex_read_performance2) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); + index->ReadMultiMillonData("tag1", "Hello", 1000 * 10); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); +} +TEST_F(IndexEnv2, testIndex_read_performance3) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); + index->ReadMultiMillonData("tag1", "Hello", 1000 * 100); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); +} +TEST_F(IndexEnv2, testIndex_read_performance4) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + } + index->PutOneTarge("tag10", "Hello", 12); + index->PutOneTarge("tag12", "Hello", 15); + index->ReadMultiMillonData("tag10", "Hello", 1000 * 100); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag10", "Hello")); +} From 616a02f5601b35951f66377a8f0b67c514d4db21 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 21 Jan 2022 00:01:23 +0800 Subject: [PATCH 096/103] add test --- source/libs/transport/src/trans.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index cf1e153965..4b490936cc 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -70,6 +70,7 @@ int32_t rpcInit(void) { void rpcCleanup(void) { // impl later + // return; } #endif From 50bbb8742230ba4ea77d00936aeca9c25022a482 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 21 Jan 2022 09:17:58 +0800 Subject: [PATCH 097/103] feature/qnode --- source/libs/qworker/src/qworker.c | 106 ++++++++++++---------- source/libs/qworker/test/qworkerTests.cpp | 16 +++- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 1cb719c667..566356e255 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -439,52 +439,6 @@ _return: QW_RET(code); } - -int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needRsp) { - int32_t code = 0; - SQWTaskCtx *ctx = NULL; - bool locked = false; - - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_WRITE, &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("task already dropping, phase:%d", ctx->phase); - QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); - } - - if (QW_IN_EXECUTOR(ctx)) { - QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); - } else if (ctx->phase > 0) { - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - - locked = false; - *needRsp = true; - } - - if (locked) { - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); - } - -_return: - - if (code) { - QW_SET_RSP_CODE(ctx, code); - } - - if (locked) { - QW_UNLOCK(QW_WRITE, &ctx->lock); - } - - QW_RET(code); -} - int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType) { int32_t code = 0; bool qcontinue = true; @@ -619,11 +573,12 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ int8_t status = 0; SQWTaskCtx *ctx = NULL; bool locked = false; + bool ctxAcquired = false; void *readyConnection = NULL; void *dropConnection = NULL; void *cancelConnection = NULL; - QW_SCH_TASK_DLOG("handle event at phase %d", phase); + QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); switch (phase) { case QW_PHASE_PRE_QUERY: { @@ -716,6 +671,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } case QW_PHASE_PRE_FETCH: { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + ctxAcquired = true; QW_LOCK(QW_WRITE, &ctx->lock); @@ -816,6 +772,7 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } case QW_PHASE_PRE_CQUERY: { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); + ctxAcquired = true; QW_LOCK(QW_WRITE, &ctx->lock); @@ -914,6 +871,10 @@ _return: QW_UNLOCK(QW_WRITE, &ctx->lock); } + if (ctxAcquired && ctx) { + qwReleaseTaskCtx(QW_READ, mgmt); + } + if (readyConnection) { qwBuildAndSendReadyRsp(readyConnection, output->rspCode); QW_TASK_DLOG("ready msg rsped, code:%x", output->rspCode); @@ -929,6 +890,7 @@ _return: QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); } + QW_SCH_TASK_DLOG("end to handle event at phase %d", phase); QW_RET(code); } @@ -1036,6 +998,7 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t if (phase == QW_PHASE_PRE_QUERY) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); + ctx->readyConnection = qwMsg->connection; QW_TASK_DLOG("ready msg not rsped, phase:%d", phase); } else if (phase == QW_PHASE_POST_QUERY) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); @@ -1225,16 +1188,59 @@ _return: int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { int32_t code = 0; bool needRsp = false; - - QW_ERR_JRET(qwDropTask(QW_FPARAMS(), &needRsp)); + SQWTaskCtx *ctx = NULL; + bool locked = false; + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_WRITE, &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropping, phase:%d", ctx->phase); + QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); + } + + if (QW_IN_EXECUTOR(ctx)) { + QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); + + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); + + ctx->dropConnection = qwMsg->connection; + } else if (ctx->phase > 0) { + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + + locked = false; + needRsp = true; + } + + if (!needRsp) { + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + } + _return: + if (code) { + QW_SET_RSP_CODE(ctx, code); + } + + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + if (ctx) { + qwReleaseTaskCtx(QW_WRITE, mgmt); + } + if (TSDB_CODE_SUCCESS != code || needRsp) { QW_ERR_RET(qwBuildAndSendDropRsp(qwMsg->connection, code)); + + QW_TASK_DLOG("drop msg rsped, code:%x", code); } - return TSDB_CODE_SUCCESS; + QW_RET(code); } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) { diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 46d80bbd26..d1cc9f03d1 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -129,17 +129,17 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTas int32_t idx = qwtTestCaseIdx % qwtTestCaseNum; if (0 == idx) { - *pTaskInfo = qwtTestCaseIdx; - *handle = qwtTestCaseIdx+1; + *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; + *handle = (DataSinkHandle)qwtTestCaseIdx+1; } else if (1 == idx) { *pTaskInfo = NULL; *handle = NULL; } else if (2 == idx) { - *pTaskInfo = qwtTestCaseIdx; + *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; *handle = NULL; } else if (3 == idx) { *pTaskInfo = NULL; - *handle = qwtTestCaseIdx; + *handle = (DataSinkHandle)qwtTestCaseIdx; } ++qwtTestCaseIdx; @@ -460,6 +460,14 @@ void *controlThread(void *param) { return NULL; } +void *queryQueueThread(void *param) { + +} + +void *fetchQueueThread(void *param) { + +} + } From d8d67b93b1ce7c59d3a25a09305eba3c8f7edde2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 20 Jan 2022 17:46:41 -0800 Subject: [PATCH 098/103] minor changes --- tests/test/c/create_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 080f1551c2..b8d8123380 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -85,7 +85,7 @@ void createDbAndStb() { } taos_free_result(pRes); - sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j int)", stbName); + sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j bigint)", stbName); pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { From 93f950c740c6472d6637f0266a51bac1c867aa14 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 10:28:41 +0800 Subject: [PATCH 099/103] refine query interface --- include/common/tmsg.h | 19 ++++++- include/libs/qcom/query.h | 61 +++++++++++++++++++++- include/libs/transport/trpc.h | 50 ------------------ source/dnode/mnode/impl/inc/mndDef.h | 15 ++++-- source/dnode/mnode/impl/src/mndSubscribe.c | 18 ++++++- source/dnode/vnode/src/tq/tq.c | 6 +-- 6 files changed, 105 insertions(+), 64 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 01b142e333..b468456cb7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1525,9 +1525,23 @@ typedef struct SMqSetCVgReq { char* sql; char* logicalPlan; char* physicalPlan; - SArray* tasks; // SArray + SSubQueryMsg msg; } SMqSetCVgReq; +static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { + int32_t tlen = sizeof(SSubQueryMsg) + pMsg->contentLen; + if (buf == NULL) return tlen; + memcpy(*buf, pMsg, tlen); + *buf = POINTER_SHIFT(*buf, tlen); + return tlen; +} + +static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { + int32_t tlen = sizeof(SSubQueryMsg) + ((SSubQueryMsg*)buf)->contentLen; + memcpy(pMsg, buf, tlen); + return POINTER_SHIFT(buf, tlen); +} + static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->vgId); @@ -1537,6 +1551,7 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); + tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } @@ -1548,7 +1563,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - pReq->tasks = NULL; + buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 02207c4d1b..1925f0e3bd 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -109,12 +109,71 @@ typedef struct STableMetaOutput { STableMeta *tbMeta; } STableMetaOutput; -const SSchema* tGetTbnameColumnSchema(); +typedef struct SDataBuf { + void *pData; + uint32_t len; +} SDataBuf; + +typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code); +typedef int32_t (*__async_exec_fn_t)(void* param); + +typedef struct SMsgSendInfo { + __async_send_cb_fn_t fp; //async callback function + void *param; + uint64_t requestId; + uint64_t requestObjRefId; + int32_t msgType; + SDataBuf msgInfo; +} SMsgSendInfo; + +typedef struct SQueryNodeAddr { + int32_t nodeId; // vgId or qnodeId + int8_t inUse; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; +} SQueryNodeAddr; + +static FORCE_INLINE void tConvertQueryAddrToEpSet(SEpSet* pEpSet, const SQueryNodeAddr* pAddr) { + pEpSet->inUse = pAddr->inUse; + pEpSet->numOfEps = pAddr->numOfEps; + for (int j = 0; j < TSDB_MAX_REPLICA; j++) { + pEpSet->port[j] = pAddr->epAddr[j].port; + memcpy(pEpSet->fqdn[j], pAddr->epAddr[j].fqdn, TSDB_FQDN_LEN); + } +} + +int32_t initTaskQueue(); +int32_t cleanupTaskQueue(); + +/** + * + * @param execFn The asynchronously execution function + * @param execParam The parameters of the execFn + * @param code The response code during execution the execFn + * @return + */ +int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); + +/** + * Asynchronously send message to server, after the response received, the callback will be incured. + * + * @param pTransporter + * @param epSet + * @param pTransporterId + * @param pInfo + * @return + */ +int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); + void initQueryModuleMsgHandle(); +const SSchema* tGetTbnameColumnSchema(); +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); + extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); + #define SET_META_TYPE_NONE(t) (t) = META_TYPE_NON_TABLE #define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 25e295f980..5afafa08a3 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -83,56 +83,6 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(int64_t rid); - -typedef struct SDataBuf { - void *pData; - uint32_t len; -} SDataBuf; - -typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code); -typedef int32_t (*__async_exec_fn_t)(void* param); - -typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; //async callback function - void *param; - uint64_t requestId; - uint64_t requestObjRefId; - int32_t msgType; - SDataBuf msgInfo; -} SMsgSendInfo; - -typedef struct SQueryNodeAddr { - int32_t nodeId; // vgId or qnodeId - int8_t inUse; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; -} SQueryNodeAddr; - -bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); - -int32_t initTaskQueue(); -int32_t cleanupTaskQueue(); - -/** - * - * @param execFn The asynchronously execution function - * @param execParam The parameters of the execFn - * @param code The response code during execution the execFn - * @return - */ -int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); - -/** - * Asynchronously send message to server, after the response received, the callback will be incured. - * - * @param pTransporter - * @param epSet - * @param pTransporterId - * @param pInfo - * @return - */ -int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 772b9bf079..aaedf280b5 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -327,11 +327,13 @@ typedef struct SMqTopicConsumer { #endif typedef struct SMqConsumerEp { - int32_t vgId; // -1 for unassigned - SEpSet epset; - int64_t consumerId; // -1 for unassigned - int64_t lastConsumerHbTs; - int64_t lastVgHbTs; + int32_t vgId; // -1 for unassigned + SEpSet epset; + int64_t consumerId; // -1 for unassigned + int64_t lastConsumerHbTs; + int64_t lastVgHbTs; + int32_t execLen; + SSubQueryMsg qExec; } SMqConsumerEp; static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { @@ -339,6 +341,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epset); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); + tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); return tlen; } @@ -346,6 +349,8 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); buf = taosDecodeSEpSet(buf, &pConsumerEp->epset); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); + buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); + pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen; return buf; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 4cf7505f74..a6634a9f01 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -105,6 +105,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { strcpy(req.sql, pTopic->sql); strcpy(req.logicalPlan, pTopic->logicalPlan); strcpy(req.physicalPlan, pTopic->physicalPlan); + memcpy(&req.msg, &pCEp->qExec, pCEp->execLen); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *reqStr = malloc(tlen); if (reqStr == NULL) { @@ -143,7 +144,21 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { SMqConsumerEp CEp; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; - int32_t sz; + //convert phyplan to dag + SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); + SArray *pArray; + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + + } + int32_t sz = taosArrayGetSize(pArray); + //convert dag to msg + for (int32_t i = 0; i < sz; i++) { + STaskInfo* pTaskInfo = taosArrayGet(pArray, i); + int32_t vgId = pTaskInfo->addr.nodeId; + SEpSet epSet; + tConvertQueryAddrToEpSet(&epSet, &pTaskInfo->addr); + } + /*pTopic->physicalPlan;*/ SVgObj *pVgroup = NULL; SSdb *pSdb = pMnode->pSdb; void *pIter = sdbFetch(pSdb, SDB_VGROUP, NULL, (void **)&pVgroup); @@ -156,6 +171,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); } return 0; + qDestroyQueryDag(pDag); } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ead856a06b..cbc948b95e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -681,7 +681,6 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { // TODO: handle error } - ASSERT(taosArrayGetSize(pArray) == 0); STaskInfo *pInfo = taosArrayGet(pArray, 0); SArray* pTasks; schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); @@ -733,6 +732,7 @@ int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) // TODO: filter out unused column return 0; } + SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t sversion = pHandle->pBlock->sversion; SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true); @@ -762,7 +762,3 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { taosArrayPush(pArray, &colInfo); return pArray; } -/*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t - * status) {*/ -/*return 0;*/ -/*}*/ From 09ba6a611d8d0d0fedf7fdcfb0909b6000e7227e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 11:19:24 +0800 Subject: [PATCH 100/103] [td-11818] refactor. --- include/libs/function/function.h | 1 + include/libs/planner/plannerOp.h | 1 + source/client/src/clientImpl.c | 5 + source/client/test/clientTests.cpp | 114 ++++++++++----------- source/dnode/vnode/src/tq/tq.c | 30 ++++++ source/libs/executor/inc/executorimpl.h | 6 +- source/libs/executor/src/executorMain.c | 12 +-- source/libs/executor/src/executorimpl.c | 20 ++-- source/libs/planner/inc/plannerInt.h | 27 ++--- source/libs/planner/src/logicPlan.c | 11 +- source/libs/planner/src/physicalPlan.c | 4 +- source/libs/planner/src/physicalPlanJson.c | 1 + source/libs/planner/src/planner.c | 6 +- 13 files changed, 141 insertions(+), 97 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 39d8ec3420..b15d5ad33a 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -254,6 +254,7 @@ typedef struct SMultiFunctionsDesc { bool interpQuery; bool distinct; bool join; + bool continueQuery; } SMultiFunctionsDesc; int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 2793e72635..42d3307ac8 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -23,6 +23,7 @@ #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME #endif +OP_ENUM_MACRO(StreamScan) OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 34ab1fb05a..159a92b0ab 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -394,6 +394,9 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo* ) pQueryNode; + pQueryStmtInfo->info.continueQuery = true; + // todo check for invalid sql statement and return with error code CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return); @@ -403,6 +406,8 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, goto _return; } + printf("%s\n", pStr); + // The topic should be related to a database that the queried table is belonged to. SName name = {0}; char dbName[TSDB_DB_FNAME_LEN] = {0}; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 440ef0d728..736a47273f 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -525,30 +525,30 @@ TEST(testCase, driverInit_Test) { // taosHashCleanup(phash); //} // -//TEST(testCase, create_topic_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == nullptr); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// char* sql = "select * from tu"; -// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); -// taos_free_result(pRes); -// taos_close(pConn); -//} -// +TEST(testCase, create_topic_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from tu"; + pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -646,36 +646,36 @@ TEST(testCase, driverInit_Test) { // taos_close(pConn); //} -TEST(testCase, agg_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use dbv"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tx using st tags(111111111111111)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "select count(*) from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} \ No newline at end of file +//TEST(testCase, agg_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use dbv"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tx using st tags(111111111111111)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create table, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select count(*) from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ead856a06b..e5f84a8d8d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "../../../../../include/libs/executor/executor.h" #include "tqInt.h" #include "tqMetaStore.h" @@ -766,3 +767,32 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { * status) {*/ /*return 0;*/ /*}*/ + +static qTaskInfo_t createExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle) { + if (pMsg == NULL || pStreamBlockReadHandle == NULL) { + return NULL; + } + + // print those info into log + pMsg->sId = be64toh(pMsg->sId); + pMsg->queryId = be64toh(pMsg->queryId); + pMsg->taskId = be64toh(pMsg->taskId); + pMsg->contentLen = ntohl(pMsg->contentLen); + + struct SSubplan *plan = NULL; + int32_t code = qStringToSubplan(pMsg->msg, &plan); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(pStreamBlockReadHandle, 0, plan, &pTaskInfo, NULL); + if (code != TSDB_CODE_SUCCESS) { + // TODO: destroy SSubplan & pTaskInfo + terrno = code; + return NULL; + } + + return pTaskInfo; +} \ No newline at end of file diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2fe3392b25..73a30a62f5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -253,12 +253,8 @@ typedef struct SExecTaskInfo { STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure pthread_mutex_t lock; // used to synchronize the rsp/query threads -// tsem_t ready; -// int32_t dataReady; // denote if query result is ready or not -// void* rspContext; // response context char *sql; // query sql string jmp_buf env; // - DataSinkHandle dsHandle; struct SOperatorInfo *pRoot; } SExecTaskInfo; @@ -666,6 +662,6 @@ int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); -int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index f39df4d4ae..f5c13ef782 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -69,11 +69,11 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { - assert(tsdb != NULL && pSubplan != NULL); +int32_t qCreateExecTask(void* readHandle, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { + assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -84,11 +84,9 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ goto _error; } - code = dsCreateDataSinker(pSubplan->pDataSink, &(*pTask)->dsHandle); + code = dsCreateDataSinker(pSubplan->pDataSink, handle); - *handle = (*pTask)->dsHandle; - -_error: + _error: // if failed to add ref for all tables in this query, abort current query return code; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index dc4d9c7238..e2beb43fa8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7778,22 +7778,26 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, return NULL; } -int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { - tsdbReadHandleT tReaderHandle = NULL; - - int32_t code = 0; +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { uint64_t queryId = pPlan->id.queryId; - SPhyNode* pPhyNode = pPlan->pNode; - + int32_t code = TSDB_CODE_SUCCESS; *pTaskInfo = createExecTaskInfo(queryId); + if (*pTaskInfo == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _complete; + } (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId); if ((*pTaskInfo)->pRoot == NULL) { - return terrno; + code = TSDB_CODE_QRY_OUT_OF_MEMORY; } - return TSDB_CODE_SUCCESS; +_complete: + tfree(*pTaskInfo); + + terrno = code; + return code; } /** diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 4ff8364198..41f50607cb 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -28,19 +28,20 @@ extern "C" { #define QNODE_TAGSCAN 1 #define QNODE_TABLESCAN 2 -#define QNODE_PROJECT 3 -#define QNODE_AGGREGATE 4 -#define QNODE_GROUPBY 5 -#define QNODE_LIMIT 6 -#define QNODE_JOIN 7 -#define QNODE_DISTINCT 8 -#define QNODE_SORT 9 -#define QNODE_UNION 10 -#define QNODE_TIMEWINDOW 11 -#define QNODE_SESSIONWINDOW 12 -#define QNODE_STATEWINDOW 13 -#define QNODE_FILL 14 -#define QNODE_MODIFY 15 +#define QNODE_STREAMSCAN 3 +#define QNODE_PROJECT 4 +#define QNODE_AGGREGATE 5 +#define QNODE_GROUPBY 6 +#define QNODE_LIMIT 7 +#define QNODE_JOIN 8 +#define QNODE_DISTINCT 9 +#define QNODE_SORT 10 +#define QNODE_UNION 11 +#define QNODE_TIMEWINDOW 12 +#define QNODE_SESSIONWINDOW 13 +#define QNODE_STATEWINDOW 14 +#define QNODE_FILL 15 +#define QNODE_MODIFY 16 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 9c9ff8fe2b..e817b764d5 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -121,6 +121,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla switch(type) { case QNODE_TAGSCAN: + case QNODE_STREAMSCAN: case QNODE_TABLESCAN: { SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); @@ -195,7 +196,12 @@ static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, SQ return pNode; } - SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); + SQueryPlanNode* pNode = NULL; + if (pQueryInfo->info.continueQuery) { + pNode = createQueryNode(QNODE_STREAMSCAN, "StreamScan", NULL, 0, NULL, 0, info); + } else { + pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); + } if (!pQueryInfo->info.projectionQuery) { SArray* p = pQueryInfo->exprList[0]; @@ -261,7 +267,6 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* pNode->numOfExpr = num; pNode->pExpr = taosArrayInit(num, POINTER_BYTES); taosArrayAddAll(pNode->pExpr, p); -// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); } } @@ -433,6 +438,7 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t len = len1 + totalLen; switch(pQueryNode->info.type) { + case QNODE_STREAMSCAN: case QNODE_TABLESCAN: { SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; len1 = sprintf(buf + len, "%s #%" PRIu64, pInfo->tableName, pInfo->uid); @@ -643,7 +649,6 @@ int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t lev int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { assert(pQueryNode); - *str = calloc(1, 4096); int32_t len = sprintf(*str, "===== logic plan =====\n"); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 1e1a97df1f..dd869d87b3 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -290,7 +290,8 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) { static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, SSubplan* subplan) { vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[0]), &subplan->execNode); - return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; + return createUserTableScanNode(pPlanNode, pTable, type); } static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { @@ -326,6 +327,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TAGSCAN: node = createTagScanNode(pPlanNode); break; + case QNODE_STREAMSCAN: case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 2abb90993b..cf54fdec85 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -829,6 +829,7 @@ static bool exchangeNodeFromJson(const cJSON* json, void* obj) { static bool specificPhyNodeToJson(const void* obj, cJSON* json) { const SPhyNode* phyNode = (const SPhyNode*)obj; switch (phyNode->info.type) { + case OP_StreamScan: case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 9b32213ad7..6b3f37741e 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From 1e7c23cd3cf9c22ece49ea4b10184f03403dd0dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 11:30:03 +0800 Subject: [PATCH 101/103] [td-11818] comment the print out. --- source/libs/planner/src/planner.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 6b3f37741e..9b32213ad7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From 03ee26ed90878280d748d811e8b3ece2af7634ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Jan 2022 11:55:13 +0800 Subject: [PATCH 102/103] [td-11818] fix error. --- source/libs/executor/src/executorimpl.c | 3 +++ source/libs/qworker/src/qworker.c | 6 +++--- source/libs/scheduler/src/scheduler.c | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e2beb43fa8..0ed480ed15 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7791,8 +7791,11 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId); if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _complete; } + return code; + _complete: tfree(*pTaskInfo); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 566356e255..07f1b0f858 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -921,17 +921,17 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task string to subplan failed, code:%x", code); + QW_TASK_ELOG("task string to subplan failed, code:%s", tstrerror(code)); QW_ERR_JRET(code); } code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); if (code) { - QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); + QW_TASK_ELOG("qCreateExecTask failed, code:%s", tstrerror(code)); QW_ERR_JRET(code); } - if ((pTaskInfo && NULL == sinkHandle) || (NULL == pTaskInfo && sinkHandle)) { + if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d8904cdfa9..ddfa73f0a5 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -666,12 +666,12 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) int32_t taskDone = 0; int32_t code = 0; - SCH_TASK_DLOG("taskOnFailure, code:%x", errCode); + SCH_TASK_DLOG("taskOnFailure, code:%s", tstrerror(errCode)); SCH_ERR_JRET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry)); if (!needRetry) { - SCH_TASK_ELOG("task failed and no more retry, code:%x", errCode); + SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode)); if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { code = schMoveTaskToFailList(pJob, pTask, &moved); From 7361019e94c01d5302a74d4c0d8595f7df66c070 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 21 Jan 2022 13:45:24 +0800 Subject: [PATCH 103/103] put createStreamExecTaskInfo into right place --- include/libs/executor/executor.h | 9 +++++- source/dnode/mnode/impl/inc/mndDef.h | 6 ++-- source/dnode/mnode/impl/src/mndSubscribe.c | 29 ++++++------------ source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/inc/tq.h | 3 +- source/dnode/vnode/src/tq/tq.c | 34 ++-------------------- source/libs/executor/src/executor.c | 34 +++++++++++++++++++++- 7 files changed, 58 insertions(+), 58 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 61970ff440..a98bb8f51a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -26,6 +26,13 @@ typedef void* qTaskInfo_t; typedef void* DataSinkHandle; struct SSubplan; + /** + * Create the exec task for streaming mode + * @param pMsg + * @param pStreamBlockReadHandle + * @return + */ +qTaskInfo_t createStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle); /** * Create the exec task object according to task json @@ -203,4 +210,4 @@ void** qDeregisterQInfo(void* pMgmt, void* pQInfo); } #endif -#endif /*_TD_EXECUTOR_H_*/ \ No newline at end of file +#endif /*_TD_EXECUTOR_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index aaedf280b5..1507e2a30d 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -328,7 +328,7 @@ typedef struct SMqTopicConsumer { typedef struct SMqConsumerEp { int32_t vgId; // -1 for unassigned - SEpSet epset; + SEpSet epSet; int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; @@ -339,7 +339,7 @@ typedef struct SMqConsumerEp { static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); - tlen += taosEncodeSEpSet(buf, &pConsumerEp->epset); + tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); return tlen; @@ -347,7 +347,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); - buf = taosDecodeSEpSet(buf, &pConsumerEp->epset); + buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a6634a9f01..817ca4f4be 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -117,7 +117,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // persist msg STransAction action = {0}; - action.epSet = pCEp->epset; + action.epSet = pCEp->epSet; action.pCont = reqStr; action.contLen = tlen; action.msgType = TDMT_VND_MQ_SET_CONN; @@ -142,36 +142,25 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { - SMqConsumerEp CEp; - CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; //convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); SArray *pArray; if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - + return -1; } int32_t sz = taosArrayGetSize(pArray); //convert dag to msg for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp CEp; + CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; STaskInfo* pTaskInfo = taosArrayGet(pArray, i); - int32_t vgId = pTaskInfo->addr.nodeId; - SEpSet epSet; - tConvertQueryAddrToEpSet(&epSet, &pTaskInfo->addr); + tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); + CEp.vgId = pTaskInfo->addr.nodeId; + taosArrayPush(unassignedVg, &CEp); } - /*pTopic->physicalPlan;*/ - SVgObj *pVgroup = NULL; - SSdb *pSdb = pMnode->pSdb; - void *pIter = sdbFetch(pSdb, SDB_VGROUP, NULL, (void **)&pVgroup); - while (pIter != NULL) { - if (pVgroup->dbUid == pTopic->dbUid) { - CEp.epset = mndGetVgroupEpset(pMnode, pVgroup); - CEp.vgId = pVgroup->vgId; - taosArrayPush(unassignedVg, &CEp); - } - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - } - return 0; + qDestroyQueryDag(pDag); + return 0; } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 75734d2d29..e625c56db1 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries( PUBLIC tfs PUBLIC wal PUBLIC scheduler + PUBLIC executor PUBLIC qworker ) diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index ec71777882..8c4effa221 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -21,6 +21,7 @@ #include "meta.h" #include "os.h" #include "scheduler.h" +#include "executor.h" #include "taoserror.h" #include "tlist.h" #include "tmsg.h" @@ -165,7 +166,7 @@ typedef struct STqTaskItem { int8_t status; int64_t offset; void* dst; - SSubQueryMsg* pMsg; + qTaskInfo_t task; } STqTaskItem; // new version diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c34c64214d..89d4af48fd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "../../../../../include/libs/executor/executor.h" #include "tqInt.h" #include "tqMetaStore.h" @@ -635,7 +634,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { } SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; - SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; + /*SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg;*/ // TODO: launch query and get output data void* outputData; @@ -689,8 +688,8 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { pTopic->buffer.lastOffset = -1; for (int i = 0; i < TQ_BUFFER_SIZE; i++) { SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); - pTopic->buffer.output[i].pMsg = pMsg; pTopic->buffer.output[i].status = 0; + pTopic->buffer.output[i].task = createStreamExecTaskInfo(pMsg, NULL); } pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); // write mq meta @@ -763,32 +762,3 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { taosArrayPush(pArray, &colInfo); return pArray; } - -static qTaskInfo_t createExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle) { - if (pMsg == NULL || pStreamBlockReadHandle == NULL) { - return NULL; - } - - // print those info into log - pMsg->sId = be64toh(pMsg->sId); - pMsg->queryId = be64toh(pMsg->queryId); - pMsg->taskId = be64toh(pMsg->taskId); - pMsg->contentLen = ntohl(pMsg->contentLen); - - struct SSubplan *plan = NULL; - int32_t code = qStringToSubplan(pMsg->msg, &plan); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } - - qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(pStreamBlockReadHandle, 0, plan, &pTaskInfo, NULL); - if (code != TSDB_CODE_SUCCESS) { - // TODO: destroy SSubplan & pTaskInfo - terrno = code; - return NULL; - } - - return pTaskInfo; -} \ No newline at end of file diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6dea4a4e57..a933402296 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -11,4 +11,36 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "planner.h" +#include "executor.h" + +qTaskInfo_t createStreamExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle) { + if (pMsg == NULL || pStreamBlockReadHandle == NULL) { + return NULL; + } + + // print those info into log + pMsg->sId = be64toh(pMsg->sId); + pMsg->queryId = be64toh(pMsg->queryId); + pMsg->taskId = be64toh(pMsg->taskId); + pMsg->contentLen = ntohl(pMsg->contentLen); + + struct SSubplan *plan = NULL; + int32_t code = qStringToSubplan(pMsg->msg, &plan); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(pStreamBlockReadHandle, 0, plan, &pTaskInfo, NULL); + if (code != TSDB_CODE_SUCCESS) { + // TODO: destroy SSubplan & pTaskInfo + terrno = code; + return NULL; + } + + return pTaskInfo; +}