xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

more format

This commit is contained in:
Hongze Cheng 2022-10-13 13:41:36 +08:00
parent 3950c8fc24
commit d5255f1eb7
30 changed files with 1233 additions and 1196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

699
source/libs/catalog/inc/catalogInt.h
View File

@ -21,18 +21,18 @@ extern "C" {
#endif
#include "catalog.h"
#include "tcommon.h"
#include "query.h"
#include "tcommon.h"
#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6
#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
#define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
#define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000
#define CTG_DEFAULT_RENT_SECOND 10
#define CTG_DEFAULT_RENT_SLOT_SIZE 10
#define CTG_DEFAULT_MAX_RETRY_TIMES 3
#define CTG_DEFAULT_BATCH_NUM 64
#define CTG_DEFAULT_FETCH_NUM 8
#define CTG_DEFAULT_RENT_SECOND 10
#define CTG_DEFAULT_RENT_SLOT_SIZE 10
#define CTG_DEFAULT_MAX_RETRY_TIMES 3
#define CTG_DEFAULT_BATCH_NUM 64
#define CTG_DEFAULT_FETCH_NUM 8
#define CTG_RENT_SLOT_SECOND 1.5
@ -90,7 +90,6 @@ typedef enum {
CTG_TASK_DONE,
} CTG_TASK_STATUS;
typedef struct SCtgDebug {
bool lockEnable;
bool cacheEnable;
@ -108,19 +107,19 @@ typedef struct SCtgTbCacheInfo {
typedef struct SCtgTbMetaCtx {
SCtgTbCacheInfo tbInfo;
int32_t vgId;
SName* pName;
int32_t flag;
int32_t vgId;
SName* pName;
int32_t flag;
} SCtgTbMetaCtx;
typedef struct SCtgFetch {
int32_t dbIdx;
int32_t tbIdx;
int32_t fetchIdx;
int32_t resIdx;
int32_t flag;
int32_t dbIdx;
int32_t tbIdx;
int32_t fetchIdx;
int32_t resIdx;
int32_t flag;
SCtgTbCacheInfo tbInfo;
int32_t vgId;
int32_t vgId;
} SCtgFetch;
typedef struct SCtgTbMetasCtx {
@ -153,7 +152,7 @@ typedef struct SCtgDbInfoCtx {
} SCtgDbInfoCtx;
typedef struct SCtgTbHashCtx {
char dbFName[TSDB_DB_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
SName* pName;
} SCtgTbHashCtx;
@ -164,7 +163,6 @@ typedef struct SCtgTbHashsCtx {
SArray* pFetchs;
} SCtgTbHashsCtx;
typedef struct SCtgIndexCtx {
char indexFName[TSDB_INDEX_FNAME_LEN];
} SCtgIndexCtx;
@ -181,118 +179,116 @@ typedef STableIndexRsp STableIndex;
typedef struct SCtgTbCache {
SRWLatch metaLock;
STableMeta *pMeta;
STableMeta* pMeta;
SRWLatch indexLock;
STableIndex *pIndex;
STableIndex* pIndex;
} SCtgTbCache;
typedef struct SCtgVgCache {
SRWLatch vgLock;
SDBVgInfo *vgInfo;
SRWLatch vgLock;
SDBVgInfo* vgInfo;
} SCtgVgCache;
typedef struct SCtgDBCache {
SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads
uint64_t dbId;
int8_t deleted;
SCtgVgCache vgCache;
SHashObj *tbCache; // key:tbname, value:SCtgTbCache
SHashObj *stbCache; // key:suid, value:char*
SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads
uint64_t dbId;
int8_t deleted;
SCtgVgCache vgCache;
SHashObj* tbCache; // key:tbname, value:SCtgTbCache
SHashObj* stbCache; // key:suid, value:char*
} SCtgDBCache;
typedef struct SCtgRentSlot {
SRWLatch lock;
bool needSort;
SArray *meta; // element is SDbVgVersion or SSTableVersion
SArray* meta; // element is SDbVgVersion or SSTableVersion
} SCtgRentSlot;
typedef struct SCtgRentMgmt {
int8_t type;
uint16_t slotNum;
uint16_t slotRIdx;
int64_t lastReadMsec;
SCtgRentSlot *slots;
int8_t type;
uint16_t slotNum;
uint16_t slotRIdx;
int64_t lastReadMsec;
SCtgRentSlot* slots;
} SCtgRentMgmt;
typedef struct SCtgUserAuth {
int32_t version;
SRWLatch lock;
bool superUser;
SHashObj *createdDbs;
SHashObj *readDbs;
SHashObj *writeDbs;
SHashObj* createdDbs;
SHashObj* readDbs;
SHashObj* writeDbs;
} SCtgUserAuth;
typedef struct SCatalog {
uint64_t clusterId;
SHashObj *userCache; //key:user, value:SCtgUserAuth
SHashObj *dbCache; //key:dbname, value:SCtgDBCache
SCtgRentMgmt dbRent;
SCtgRentMgmt stbRent;
uint64_t clusterId;
SHashObj* userCache; // key:user, value:SCtgUserAuth
SHashObj* dbCache; // key:dbname, value:SCtgDBCache
SCtgRentMgmt dbRent;
SCtgRentMgmt stbRent;
} SCatalog;
typedef struct SCtgBatch {
int32_t batchId;
int32_t msgType;
int32_t msgSize;
SArray* pMsgs;
int32_t batchId;
int32_t msgType;
int32_t msgSize;
SArray* pMsgs;
SRequestConnInfo conn;
char dbFName[TSDB_DB_FNAME_LEN];
SArray* pTaskIds;
SArray* pMsgIdxs;
char dbFName[TSDB_DB_FNAME_LEN];
SArray* pTaskIds;
SArray* pMsgIdxs;
} SCtgBatch;
typedef struct SCtgJob {
int64_t refId;
int32_t batchId;
SHashObj* pBatchs;
SArray* pTasks;
int32_t subTaskNum;
int32_t taskDone;
SMetaData jobRes;
int32_t jobResCode;
int32_t taskIdx;
SRWLatch taskLock;
int64_t refId;
int32_t batchId;
SHashObj* pBatchs;
SArray* pTasks;
int32_t subTaskNum;
int32_t taskDone;
SMetaData jobRes;
int32_t jobResCode;
int32_t taskIdx;
SRWLatch taskLock;
uint64_t queryId;
SCatalog* pCtg;
SRequestConnInfo conn;
void* userParam;
catalogCallback userFp;
int32_t tbMetaNum;
int32_t tbHashNum;
int32_t dbVgNum;
int32_t udfNum;
int32_t qnodeNum;
int32_t dnodeNum;
int32_t dbCfgNum;
int32_t indexNum;
int32_t userNum;
int32_t dbInfoNum;
int32_t tbIndexNum;
int32_t tbCfgNum;
int32_t svrVerNum;
uint64_t queryId;
SCatalog* pCtg;
SRequestConnInfo conn;
void* userParam;
catalogCallback userFp;
int32_t tbMetaNum;
int32_t tbHashNum;
int32_t dbVgNum;
int32_t udfNum;
int32_t qnodeNum;
int32_t dnodeNum;
int32_t dbCfgNum;
int32_t indexNum;
int32_t userNum;
int32_t dbInfoNum;
int32_t tbIndexNum;
int32_t tbCfgNum;
int32_t svrVerNum;
} SCtgJob;
typedef struct SCtgMsgCtx {
int32_t reqType;
void* lastOut;
void* out;
char* target;
int32_t reqType;
void* lastOut;
void* out;
char* target;
SHashObj* pBatchs;
} SCtgMsgCtx;
typedef struct SCtgTaskCallbackParam {
uint64_t queryId;
int64_t refId;
SArray* taskId;
int32_t reqType;
int32_t batchId;
SArray* msgIdx;
uint64_t queryId;
int64_t refId;
SArray* taskId;
int32_t reqType;
int32_t batchId;
SArray* msgIdx;
} SCtgTaskCallbackParam;
typedef struct SCtgTask SCtgTask;
typedef int32_t (*ctgSubTaskCbFp)(SCtgTask*);
@ -325,7 +321,7 @@ typedef struct SCtgTaskReq {
typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*);
typedef int32_t (*ctgLanchTaskFp)(SCtgTask*);
typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTaskReq*, int32_t, const SDataBuf *, int32_t);
typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTaskReq*, int32_t, const SDataBuf*, int32_t);
typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*);
typedef int32_t (*ctgCloneTaskResFp)(SCtgTask*, void**);
typedef int32_t (*ctgCompTaskFp)(SCtgTask*, void*, bool*);
@ -340,7 +336,6 @@ typedef struct SCtgAsyncFps {
} SCtgAsyncFps;
typedef struct SCtgApiStat {
#if defined(WINDOWS) || defined(_TD_DARWIN_64)
size_t avoidCompilationErrors;
#endif
@ -371,9 +366,9 @@ typedef struct SCtgCacheStat {
} SCtgCacheStat;
typedef struct SCatalogStat {
SCtgApiStat api;
SCtgRuntimeStat runtime;
SCtgCacheStat cache;
SCtgApiStat api;
SCtgRuntimeStat runtime;
SCtgCacheStat cache;
} SCatalogStat;
typedef struct SCtgUpdateMsgHeader {
@ -381,9 +376,9 @@ typedef struct SCtgUpdateMsgHeader {
} SCtgUpdateMsgHeader;
typedef struct SCtgUpdateVgMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
SDBVgInfo* dbInfo;
} SCtgUpdateVgMsg;
@ -394,33 +389,32 @@ typedef struct SCtgUpdateTbMetaMsg {
typedef struct SCtgDropDBMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
} SCtgDropDBMsg;
typedef struct SCtgDropDbVgroupMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
} SCtgDropDbVgroupMsg;
typedef struct SCtgDropStbMetaMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t suid;
char dbFName[TSDB_DB_FNAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t suid;
} SCtgDropStbMetaMsg;
typedef struct SCtgDropTblMetaMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
} SCtgDropTblMetaMsg;
typedef struct SCtgUpdateUserMsg {
SCatalog* pCtg;
SCatalog* pCtg;
SGetUserAuthRsp userAuth;
} SCtgUpdateUserMsg;
@ -430,63 +424,63 @@ typedef struct SCtgUpdateTbIndexMsg {
} SCtgUpdateTbIndexMsg;
typedef struct SCtgDropTbIndexMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
} SCtgDropTbIndexMsg;
typedef struct SCtgClearCacheMsg {
SCatalog* pCtg;
bool freeCtg;
SCatalog* pCtg;
bool freeCtg;
} SCtgClearCacheMsg;
typedef struct SCtgUpdateEpsetMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
int32_t vgId;
SEpSet epSet;
char dbFName[TSDB_DB_FNAME_LEN];
int32_t vgId;
SEpSet epSet;
} SCtgUpdateEpsetMsg;
typedef struct SCtgCacheOperation {
int32_t opId;
void *data;
bool syncOp;
tsem_t rspSem;
bool stopQueue;
bool unLocked;
int32_t opId;
void* data;
bool syncOp;
tsem_t rspSem;
bool stopQueue;
bool unLocked;
} SCtgCacheOperation;
typedef struct SCtgQNode {
SCtgCacheOperation *op;
struct SCtgQNode *next;
SCtgCacheOperation* op;
struct SCtgQNode* next;
} SCtgQNode;
typedef struct SCtgQueue {
SRWLatch qlock;
bool stopQueue;
SCtgQNode *head;
SCtgQNode *tail;
tsem_t reqSem;
uint64_t qRemainNum;
SRWLatch qlock;
bool stopQueue;
SCtgQNode* head;
SCtgQNode* tail;
tsem_t reqSem;
uint64_t qRemainNum;
} SCtgQueue;
typedef struct SCatalogMgmt {
bool exit;
int32_t jobPool;
SRWLatch lock;
SCtgQueue queue;
TdThread updateThread;
SHashObj *pCluster; //key: clusterId, value: SCatalog*
SCatalogStat stat;
SCatalogCfg cfg;
bool exit;
int32_t jobPool;
SRWLatch lock;
SCtgQueue queue;
TdThread updateThread;
SHashObj* pCluster; // key: clusterId, value: SCatalog*
SCatalogStat stat;
SCatalogCfg cfg;
} SCatalogMgmt;
typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
typedef int32_t (*ctgOpFunc)(SCtgCacheOperation *);
typedef uint32_t (*tableNameHashFp)(const char*, uint32_t);
typedef int32_t (*ctgOpFunc)(SCtgCacheOperation*);
typedef struct SCtgOperation {
int32_t opId;
char name[32];
int32_t opId;
char name[32];
ctgOpFunc func;
} SCtgOperation;
@ -495,16 +489,16 @@ typedef struct SCtgOperation {
#define CTG_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n)
#define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n)
#define CTG_STAT_GET(_item) atomic_load_64(&(_item))
#define CTG_STAT_GET(_item) atomic_load_64(&(_item))
#define CTG_RT_STAT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.stat.runtime.item, n))
#define CTG_RT_STAT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.stat.runtime.item, n))
#define CTG_CACHE_STAT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.stat.cache.item, n))
#define CTG_CACHE_STAT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.stat.cache.item, n))
#define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
#define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
#define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
#define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE)
#define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE)
#define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE)
#define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE)
#define CTG_FLAG_STB 0x1
#define CTG_FLAG_NOT_STB 0x2
@ -514,199 +508,263 @@ typedef struct SCtgOperation {
#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v))
#define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB)
#define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB)
#define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB)
#define CTG_FLAG_IS_SYS_DB(_flag) ((_flag) & CTG_FLAG_SYS_DB)
#define CTG_FLAG_IS_FORCE_UPDATE(_flag) ((_flag) & CTG_FLAG_FORCE_UPDATE)
#define CTG_FLAG_SET_SYS_DB(_flag) ((_flag) |= CTG_FLAG_SYS_DB)
#define CTG_FLAG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0)
#define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB))
#define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_FLAG_IS_STB(_flag) ((_flag)&CTG_FLAG_STB)
#define CTG_FLAG_IS_NOT_STB(_flag) ((_flag)&CTG_FLAG_NOT_STB)
#define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag)&CTG_FLAG_UNKNOWN_STB)
#define CTG_FLAG_IS_SYS_DB(_flag) ((_flag)&CTG_FLAG_SYS_DB)
#define CTG_FLAG_IS_FORCE_UPDATE(_flag) ((_flag)&CTG_FLAG_FORCE_UPDATE)
#define CTG_FLAG_SET_SYS_DB(_flag) ((_flag) |= CTG_FLAG_SYS_DB)
#define CTG_FLAG_SET_STB(_flag, tbType) \
do { \
(_flag) |= ((tbType) == TSDB_SUPER_TABLE) \
? CTG_FLAG_STB \
: ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); \
} while (0)
#define CTG_FLAG_MAKE_STB(_isStb) \
(((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB))
#define CTG_FLAG_MATCH_STB(_flag, tbType) \
(CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || \
(CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_GET_TASK_MSGCTX(_task, _id) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx)
#define CTG_GET_TASK_MSGCTX(_task, _id) \
(((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) \
? taosArrayGet((_task)->msgCtxs, (_id)) \
: &(_task)->msgCtx)
#define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema))
#define CTG_META_SIZE(pMeta) \
(sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema))
#define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST)
#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST)
#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST)
#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgWarn(param, ...) qWarn("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgInfo(param, ...) qInfo("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__)
#define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__)
#define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0)
#define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheEnable) { qDebug(__VA_ARGS__); } } while (0)
#define CTG_API_DEBUG(...) do { if (gCTGDebug.apiEnable) { qDebug(__VA_ARGS__); } } while (0)
#define CTG_LOCK_DEBUG(...) \
do { \
if (gCTGDebug.lockEnable) { \
qDebug(__VA_ARGS__); \
} \
} while (0)
#define CTG_CACHE_DEBUG(...) \
do { \
if (gCTGDebug.cacheEnable) { \
qDebug(__VA_ARGS__); \
} \
} while (0)
#define CTG_API_DEBUG(...) \
do { \
if (gCTGDebug.apiEnable) { \
qDebug(__VA_ARGS__); \
} \
} while (0)
#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
#define CTG_LOCK(type, _lock) do { \
if (CTG_READ == (type)) { \
assert(atomic_load_32((_lock)) >= 0); \
CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) > 0); \
} else { \
assert(atomic_load_32((_lock)) >= 0); \
CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
} \
} while (0)
#define CTG_UNLOCK(type, _lock) do { \
if (CTG_READ == (type)) { \
assert(atomic_load_32((_lock)) > 0); \
CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRUnLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} else { \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWUnLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} \
} while (0)
#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define CTG_API_LEAVE(c) do { \
int32_t __code = c; \
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); \
CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \
CTG_RET(__code); \
} while (0)
#define CTG_API_ENTER() do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
#define CTG_API_JENTER() do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_ERR_JRET(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
#define CTG_API_LEAVE_NOLOCK(c) do { \
int32_t __code = c; \
CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \
CTG_RET(__code); \
#define CTG_LOCK(type, _lock) \
do { \
if (CTG_READ == (type)) { \
assert(atomic_load_32((_lock)) >= 0); \
CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) > 0); \
} else { \
assert(atomic_load_32((_lock)) >= 0); \
CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
} \
} while (0)
#define CTG_API_ENTER_NOLOCK() do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_API_LEAVE_NOLOCK(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
#define CTG_UNLOCK(type, _lock) \
do { \
if (CTG_READ == (type)) { \
assert(atomic_load_32((_lock)) > 0); \
CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosRUnLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} else { \
assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \
CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
taosWUnLockLatch(_lock); \
CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
assert(atomic_load_32((_lock)) >= 0); \
} \
} while (0)
#define CTG_ERR_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define CTG_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
#define CTG_ERR_JRET(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
#define CTG_API_LEAVE(c) \
do { \
int32_t __code = c; \
CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); \
CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \
CTG_RET(__code); \
} while (0)
#define CTG_API_ENTER() \
do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
#define CTG_API_JENTER() \
do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_ERR_JRET(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
#define CTG_API_LEAVE_NOLOCK(c) \
do { \
int32_t __code = c; \
CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); \
CTG_RET(__code); \
} while (0)
#define CTG_API_ENTER_NOLOCK() \
do { \
CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \
if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \
CTG_API_LEAVE_NOLOCK(TSDB_CODE_CTG_OUT_OF_SERVICE); \
} \
} while (0)
void ctgdShowTableMeta(SCatalog* pCtg, const char* tbName, STableMeta* p);
void ctgdShowClusterCache(SCatalog* pCtg);
int32_t ctgdShowCacheInfo(void);
int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq);
int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList);
int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx,
int32_t* fetchIdx, int32_t baseResIdx, SArray* pList);
int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action);
int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action);
int32_t ctgOpDropDbCache(SCtgCacheOperation *action);
int32_t ctgOpDropDbVgroup(SCtgCacheOperation *action);
int32_t ctgOpDropStbMeta(SCtgCacheOperation *action);
int32_t ctgOpDropTbMeta(SCtgCacheOperation *action);
int32_t ctgOpUpdateUser(SCtgCacheOperation *action);
int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache);
void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgRUnlockVgInfo(SCtgDBCache *dbCache);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist);
int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action);
int32_t ctgOpUpdateTbMeta(SCtgCacheOperation* action);
int32_t ctgOpDropDbCache(SCtgCacheOperation* action);
int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action);
int32_t ctgOpDropStbMeta(SCtgCacheOperation* action);
int32_t ctgOpDropTbMeta(SCtgCacheOperation* action);
int32_t ctgOpUpdateUser(SCtgCacheOperation* action);
int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache);
void ctgReleaseDBCache(SCatalog* pCtg, SCtgDBCache* dbCache);
void ctgRUnlockVgInfo(SCtgDBCache* dbCache);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32_t* exist);
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName);
int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass);
int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId);
int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq);
int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq);
int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq);
int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq);
int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq);
int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq);
int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet);
int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp);
int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType,
uint64_t* suid, char* stbName);
int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool* inCache, bool* pass);
int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId);
int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char* dbFName, bool syncReq);
int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* stbName, uint64_t suid,
bool syncReq);
int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* tbName, bool syncReq);
int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq);
int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput* output, bool syncReq);
int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp* pAuth, bool syncReq);
int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char* dbFName, int32_t vgId, SEpSet* pEpSet);
int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex** pIndex, bool syncOp);
int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool freeCtg, bool stopQueue, bool syncOp);
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type);
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size);
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size);
int32_t ctgMetaRentInit(SCtgRentMgmt* mgmt, uint32_t rentSec, int8_t type);
int32_t ctgMetaRentAdd(SCtgRentMgmt* mgmt, void* meta, int64_t id, int32_t size);
int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t size);
int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq);
int32_t ctgStartUpdateThread();
int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask);
void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache);
int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask);
void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache);
int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes);
int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp);
int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation);
int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation);
int32_t ctgOpClearCache(SCtgCacheOperation *operation);
int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType);
int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVgroupInfo **pVgroup);
int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation);
int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation);
int32_t ctgOpClearCache(SCtgCacheOperation* operation);
int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType);
int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup);
int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target);
int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTaskReq* tReq);
int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask);
int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray **out, SCtgTask* pTask);
int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask);
int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *indexName, SIndexInfo *out, SCtgTask* pTask);
int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask);
int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask);
int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask);
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask);
int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask);
int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask);
int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob *pJob, SHashObj* pBatchs);
int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out,
SCtgTaskReq* tReq);
int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* out, SCtgTask* pTask);
int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray** out, SCtgTask* pTask);
int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* out,
SCtgTask* pTask);
int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* out,
SCtgTask* pTask);
int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* name, STableIndex* out, SCtgTask* pTask);
int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* out,
SCtgTask* pTask);
int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, SGetUserAuthRsp* out,
SCtgTask* pTask);
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName,
STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out,
SCtgTaskReq* tReq);
int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* vgroupInfo,
STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* vgroupInfo, STableCfg** out, SCtgTask* pTask);
int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** out,
SCtgTask* pTask);
int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** out, SCtgTask* pTask);
int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs);
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param);
int32_t ctgLaunchJob(SCtgJob *pJob);
int32_t ctgMakeAsyncRes(SCtgJob *pJob);
int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param);
int32_t ctgGetTbCfgCb(SCtgTask *pTask);
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp,
void* param);
int32_t ctgLaunchJob(SCtgJob* pJob);
int32_t ctgMakeAsyncRes(SCtgJob* pJob);
int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param);
int32_t ctgGetTbCfgCb(SCtgTask* pTask);
void ctgFreeHandle(SCatalog* pCatalog);
void ctgFreeMsgSendParam(void* param);
void ctgFreeBatch(SCtgBatch *pBatch);
void ctgFreeBatchs(SHashObj *pBatchs);
int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst);
int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput);
int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList);
void ctgFreeBatch(SCtgBatch* pBatch);
void ctgFreeBatchs(SHashObj* pBatchs);
int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst);
int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput);
int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList);
void ctgFreeJob(void* job);
void ctgFreeHandleImpl(SCatalog* pCtg);
void ctgFreeVgInfo(SDBVgInfo *vgInfo);
int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup);
int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update);
void ctgFreeVgInfo(SDBVgInfo* vgInfo);
int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup);
int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx,
char* dbFName, SArray* pNames, bool update);
void ctgResetTbMetaTask(SCtgTask* pTask);
void ctgFreeDbCache(SCtgDBCache *dbCache);
void ctgFreeDbCache(SCtgDBCache* dbCache);
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2);
int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2);
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2);
@ -714,23 +772,22 @@ int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2);
void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput);
int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target);
int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target);
char * ctgTaskTypeStr(CTG_TASK_TYPE type);
int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId);
int32_t ctgGetTablesReqNum(SArray *pList);
int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag);
char* ctgTaskTypeStr(CTG_TASK_TYPE type);
int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId);
int32_t ctgGetTablesReqNum(SArray* pList);
int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag);
int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes);
void ctgFreeSTableIndex(void *info);
void ctgClearSubTaskRes(SCtgSubRes *pRes);
void ctgFreeQNode(SCtgQNode *node);
void ctgFreeSTableIndex(void* info);
void ctgClearSubTaskRes(SCtgSubRes* pRes);
void ctgFreeQNode(SCtgQNode* node);
void ctgClearHandle(SCatalog* pCtg);
void ctgFreeTbCacheImpl(SCtgTbCache *pCache);
void ctgFreeTbCacheImpl(SCtgTbCache* pCache);
int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName);
int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup);
int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup);
SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch);
extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug;
extern SCtgDebug gCTGDebug;
extern SCtgAsyncFps gCtgAsyncFps[];
#ifdef __cplusplus

2
source/libs/catalog/inc/ctgRemote.h
View File

@ -20,8 +20,6 @@
extern "C" {
#endif
#ifdef __cplusplus
}
#endif

171
source/libs/catalog/src/catalog.c
View File

@ -22,7 +22,8 @@
SCatalogMgmt gCtgMgmt = {0};
int32_t ctgGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) {
int32_t ctgGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SCtgDBCache** dbCache,
SDBVgInfo** pInfo) {
int32_t code = 0;
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache));
@ -53,8 +54,8 @@ _return:
CTG_RET(code);
}
int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName) {
int32_t code = 0;
int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) {
int32_t code = 0;
SCtgDBCache* dbCache = NULL;
CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache));
@ -87,7 +88,8 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char*
return TSDB_CODE_SUCCESS;
}
int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMetaOutput **pOutput, bool syncReq) {
int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMetaOutput** pOutput,
bool syncReq) {
SVgroupInfo vgroupInfo = {0};
int32_t code = 0;
@ -105,7 +107,8 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx*
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName));
CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)ctx->pName->dbname, (char *)ctx->pName->tname, output, NULL));
CTG_ERR_JRET(
ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL));
} else if (CTG_FLAG_IS_STB(ctx->flag)) {
ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName));
@ -126,7 +129,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx*
ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName));
taosMemoryFreeClear(output->tbMeta);
CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, output->dbFName, output->tbName, output, NULL));
} else if (CTG_IS_META_BOTH(output->metaType)) {
int32_t exist = 0;
@ -182,9 +185,9 @@ _return:
CTG_RET(code);
}
int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) {
int32_t code = 0;
STableMetaOutput *output = NULL;
int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) {
int32_t code = 0;
STableMetaOutput* output = NULL;
CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, ctx, pTableMeta));
if (*pTableMeta) {
@ -299,12 +302,13 @@ _return:
CTG_RET(code);
}
int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) {
bool inCache = false;
int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass) {
bool inCache = false;
int32_t code = 0;
*pass = false;
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass));
if (inCache) {
@ -313,7 +317,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, co
SGetUserAuthRsp authRsp = {0};
CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, user, &authRsp, NULL));
if (authRsp.superAuth) {
*pass = true;
goto _return;
@ -337,7 +341,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, int32_t *tbType) {
int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType));
@ -345,7 +349,7 @@ int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName,
return TSDB_CODE_SUCCESS;
}
STableMeta* pMeta = NULL;
STableMeta* pMeta = NULL;
SCtgTbMetaCtx ctx = {0};
ctx.pName = (SName*)pTableName;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
@ -357,24 +361,24 @@ int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName,
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pRes) {
int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) {
CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes));
if (*pRes) {
return TSDB_CODE_SUCCESS;
}
STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex));
STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex));
if (NULL == pIndex) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t code = ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL);
if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
SArray* pInfo = NULL;
CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo));
@ -395,7 +399,7 @@ _return:
CTG_RET(code);
}
int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, STableCfg** pCfg) {
int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, STableCfg** pCfg) {
int32_t tbType = 0;
CTG_ERR_RET(ctgGetTbType(pCtg, pConn, pTableName, &tbType));
@ -410,25 +414,25 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName,
CTG_RET(TSDB_CODE_SUCCESS);
}
int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pVgList) {
STableMeta *tbMeta = NULL;
int32_t code = 0;
SVgroupInfo vgroupInfo = {0};
SCtgDBCache* dbCache = NULL;
SArray *vgList = NULL;
SDBVgInfo *vgInfo = NULL;
int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) {
STableMeta* tbMeta = NULL;
int32_t code = 0;
SVgroupInfo vgroupInfo = {0};
SCtgDBCache* dbCache = NULL;
SArray* vgList = NULL;
SDBVgInfo* vgInfo = NULL;
SCtgTbMetaCtx ctx = {0};
ctx.pName = pTableName;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
*pVgList = NULL;
CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &tbMeta));
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
SHashObj *vgHash = NULL;
SHashObj* vgHash = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo));
if (dbCache) {
@ -489,8 +493,7 @@ _return:
CTG_RET(code);
}
int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup) {
int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup) {
if (IS_SYS_DBNAME(pTableName->dbname)) {
ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname);
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
@ -501,7 +504,7 @@ int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
SDBVgInfo *vgInfo = NULL;
SDBVgInfo* vgInfo = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo));
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup));
@ -539,7 +542,6 @@ _return:
CTG_RET(code);
}
int32_t catalogInit(SCatalogCfg* cfg) {
if (gCtgMgmt.pCluster) {
qError("catalog already initialized");
@ -648,7 +650,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
clusterCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
clusterCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum,
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (NULL == clusterCtg->userCache) {
qError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
@ -714,7 +717,7 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName, SArray** vgroupList) {
int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SArray** vgroupList) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == dbFName || NULL == pConn || NULL == vgroupList) {
@ -722,10 +725,10 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char*
}
SCtgDBCache* dbCache = NULL;
int32_t code = 0;
SArray *vgList = NULL;
SHashObj *vgHash = NULL;
SDBVgInfo *vgInfo = NULL;
int32_t code = 0;
SArray* vgList = NULL;
SHashObj* vgHash = NULL;
SDBVgInfo* vgInfo = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo));
if (dbCache) {
vgHash = dbCache->vgCache.vgInfo->vgHash;
@ -808,11 +811,11 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp) {
int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp) {
CTG_API_ENTER();
int32_t code = 0;
if (NULL == pCtg || NULL == pRsp) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
@ -827,11 +830,10 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp) {
CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) {
CTG_API_ENTER();
@ -860,17 +862,18 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogGetTableMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMeta** pTableMeta) {
int32_t catalogGetTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta) {
CTG_API_ENTER();
SCtgTbMetaCtx ctx = {0};
ctx.pName = (SName*)pTableName;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
}
int32_t catalogGetSTableMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMeta** pTableMeta) {
int32_t catalogGetSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta) {
CTG_API_ENTER();
SCtgTbMetaCtx ctx = {0};
@ -895,7 +898,7 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo *pConn, SArray* pTables) {
int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* pTables) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTables) {
@ -944,7 +947,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo *pConn, SArray*
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName) {
int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == dbFName) {
@ -954,7 +957,7 @@ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const ch
CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pConn, dbFName));
}
int32_t catalogRefreshTableMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, int32_t isSTable) {
int32_t catalogRefreshTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, int32_t isSTable) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName) {
@ -968,7 +971,8 @@ int32_t catalogRefreshTableMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const S
CTG_API_LEAVE(ctgRefreshTbMeta(pCtg, pConn, &ctx, NULL, true));
}
int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) {
int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
STableMeta** pTableMeta, int32_t isSTable) {
CTG_API_ENTER();
SCtgTbMetaCtx ctx = {0};
@ -978,7 +982,7 @@ int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, SRequestConnInfo *pConn, cons
CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
}
int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pVgList) {
int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pVgList) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pVgList) {
@ -993,13 +997,14 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const
CTG_API_LEAVE(ctgGetTbDistVgInfo(pCtg, pConn, (SName*)pTableName, pVgList));
}
int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup) {
int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
SVgroupInfo* pVgroup) {
CTG_API_ENTER();
CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup));
}
int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SCatalogReq* pReq, SMetaData* pRsp) {
int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalogReq* pReq, SMetaData* pRsp) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pReq || NULL == pRsp) {
@ -1028,7 +1033,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SCatalo
SCtgTbMetaCtx ctx = {0};
ctx.pName = name;
ctx.flag = CTG_FLAG_UNKNOWN_STB;
CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &pTableMeta));
if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) {
@ -1062,15 +1067,16 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const SCatalogReq* pReq, catalogCallback fp, void* param, int64_t* jobId) {
int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalogReq* pReq, catalogCallback fp,
void* param, int64_t* jobId) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pReq || NULL == fp || NULL == param) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
SCtgJob *pJob = NULL;
int32_t code = 0;
SCtgJob* pJob = NULL;
CTG_ERR_JRET(ctgInitJob(pCtg, pConn, &pJob, pReq, fp, param));
CTG_ERR_JRET(ctgLaunchJob(pJob));
@ -1091,7 +1097,7 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogGetQnodeList(SCatalog* pCtg, SRequestConnInfo *pConn, SArray* pQnodeList) {
int32_t catalogGetQnodeList(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* pQnodeList) {
CTG_API_ENTER();
int32_t code = 0;
@ -1121,14 +1127,14 @@ _return:
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion **stables, uint32_t *num) {
int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion** stables, uint32_t* num) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == stables || NULL == num) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableVersion)));
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableVersion)));
}
int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion** dbs, uint32_t* num) {
@ -1152,7 +1158,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
if (*num <= 0) {
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
*users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion));
if (NULL == *users) {
ctgError("calloc %d userAuthVersion failed", *num);
@ -1172,16 +1178,16 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
taosHashCancelIterate(pCtg->userCache, pAuth);
break;
}
pAuth = taosHashIterate(pCtg->userCache, pAuth);
}
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName, SDbCfgInfo* pDbCfg) {
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == dbFName || NULL == pDbCfg) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
@ -1189,9 +1195,9 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbF
CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL));
}
int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const char* indexName, SIndexInfo* pInfo) {
int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == indexName || NULL == pInfo) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
@ -1199,24 +1205,24 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo *pConn, const char*
CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pConn, indexName, pInfo, NULL));
}
int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) {
int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetTbIndex(pCtg, pConn, (SName*)pTableName, pRes));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg** pCfg) {
int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pCfg) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
@ -1231,52 +1237,52 @@ _return:
CTG_API_LEAVE(code);
}
int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* funcName, SFuncInfo* pInfo) {
int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == funcName || NULL == pInfo) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pConn, funcName, pInfo, NULL));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) {
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
bool* pass) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo *pConn, char** pVersion) {
int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pVersion) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetSvrVerFromMnode(pCtg, pConn, pVersion, NULL));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) {
CTG_API_ENTER();
@ -1299,11 +1305,10 @@ int32_t catalogClearCache(void) {
int32_t code = ctgClearCacheEnqueue(NULL, false, false, true);
qInfo("clear catalog cache end, code: %s", tstrerror(code));
CTG_API_LEAVE_NOLOCK(code);
}
void catalogDestroy(void) {
qInfo("start to destroy catalog");

2
source/libs/catalog/src/ctgAsync.c
View File

@ -636,7 +636,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
taosAcquireRef(gCtgMgmt.jobPool, pJob->refId);
double el = (taosGetTimestampUs() - st)/1000.0;
double el = (taosGetTimestampUs() - st) / 1000.0;
qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms",
pJob->queryId, pJob->refId, taskNum, pReq->forceUpdate, el);
return TSDB_CODE_SUCCESS;

126
source/libs/catalog/src/ctgDbg.c
View File

@ -13,16 +13,16 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "catalogInt.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
#include "trpc.h"
extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {0};
SCtgDebug gCTGDebug = {0};
void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
ASSERT(*(int32_t*)param == 1);
void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) {
ASSERT(*(int32_t *)param == 1);
taosMemoryFree(param);
qDebug("async call result: %s", tstrerror(code));
@ -36,16 +36,19 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
if (pResult->pTableMeta && taosArrayGetSize(pResult->pTableMeta) > 0) {
num = taosArrayGetSize(pResult->pTableMeta);
for (int32_t i = 0; i < num; ++i) {
STableMeta *p = *(STableMeta **)taosArrayGet(pResult->pTableMeta, i);
STableMeta *p = *(STableMeta **)taosArrayGet(pResult->pTableMeta, i);
STableComInfo *c = &p->tableInfo;
if (TSDB_CHILD_TABLE == p->tableType) {
qDebug("table meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64, p->tableType, p->vgId, p->uid, p->suid);
qDebug("table meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64, p->tableType, p->vgId, p->uid,
p->suid);
} else {
qDebug("table meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
qDebug("table meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64
",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision,
c->numOfColumns, c->rowSize);
}
int32_t colNum = c->numOfColumns + c->numOfTags;
for (int32_t j = 0; j < colNum; ++j) {
SSchema *s = &p->schema[j];
@ -59,11 +62,11 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
if (pResult->pDbVgroup && taosArrayGetSize(pResult->pDbVgroup) > 0) {
num = taosArrayGetSize(pResult->pDbVgroup);
for (int32_t i = 0; i < num; ++i) {
SArray *pDb = *(SArray**)taosArrayGet(pResult->pDbVgroup, i);
SArray *pDb = *(SArray **)taosArrayGet(pResult->pDbVgroup, i);
int32_t vgNum = taosArrayGetSize(pDb);
qDebug("db %d vgInfo:", i);
for (int32_t j = 0; j < vgNum; ++j) {
SVgroupInfo* pInfo = taosArrayGet(pDb, j);
SVgroupInfo *pInfo = taosArrayGet(pDb, j);
qDebug("vg :%d info: vgId:%d", j, pInfo->vgId);
}
}
@ -84,7 +87,7 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
if (pResult->pTableHash && taosArrayGetSize(pResult->pTableHash) > 0) {
num = taosArrayGetSize(pResult->pTableHash);
for (int32_t i = 0; i < num; ++i) {
SVgroupInfo* pInfo = taosArrayGet(pResult->pTableHash, i);
SVgroupInfo *pInfo = taosArrayGet(pResult->pTableHash, i);
qDebug("table %d vg info: vgId:%d", i, pInfo->vgId);
}
} else {
@ -94,7 +97,7 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
if (pResult->pUdfList && taosArrayGetSize(pResult->pUdfList) > 0) {
num = taosArrayGetSize(pResult->pUdfList);
for (int32_t i = 0; i < num; ++i) {
SFuncInfo* pInfo = taosArrayGet(pResult->pUdfList, i);
SFuncInfo *pInfo = taosArrayGet(pResult->pUdfList, i);
qDebug("udf %d info: name:%s, funcType:%d", i, pInfo->name, pInfo->funcType);
}
} else {
@ -104,35 +107,34 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
if (pResult->pDbCfg && taosArrayGetSize(pResult->pDbCfg) > 0) {
num = taosArrayGetSize(pResult->pDbCfg);
for (int32_t i = 0; i < num; ++i) {
SDbCfgInfo* pInfo = taosArrayGet(pResult->pDbCfg, i);
SDbCfgInfo *pInfo = taosArrayGet(pResult->pDbCfg, i);
qDebug("db %d info: numOFVgroups:%d, numOfStables:%d", i, pInfo->numOfVgroups, pInfo->numOfStables);
}
} else {
qDebug("empty db cfg info");
}
}
if (pResult->pUser && taosArrayGetSize(pResult->pUser) > 0) {
num = taosArrayGetSize(pResult->pUser);
for (int32_t i = 0; i < num; ++i) {
bool* auth = taosArrayGet(pResult->pUser, i);
bool *auth = taosArrayGet(pResult->pUser, i);
qDebug("user auth %d info: %d", i, *auth);
}
} else {
qDebug("empty user auth info");
}
}
if (pResult->pQnodeList && taosArrayGetSize(pResult->pQnodeList) > 0) {
num = taosArrayGetSize(pResult->pQnodeList);
for (int32_t i = 0; i < num; ++i) {
SQueryNodeAddr* qaddr = taosArrayGet(pResult->pQnodeList, i);
SQueryNodeAddr *qaddr = taosArrayGet(pResult->pQnodeList, i);
qDebug("qnode %d info: id:%d", i, qaddr->nodeId);
}
} else {
qDebug("empty qnode info");
}
}
}
/*
prepare SQL:
create database db1;
@ -147,8 +149,8 @@ grant write on db2.* to user1;
create function udf1 as '/tmp/libudf1.so' outputtype int;
create aggregate function udf2 as '/tmp/libudf2.so' outputtype int;
*/
int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t reqId, bool forceUpdate) {
int32_t code = 0;
int32_t ctgdLaunchAsyncCall(SCatalog *pCtg, SRequestConnInfo *pConn, uint64_t reqId, bool forceUpdate) {
int32_t code = 0;
SCatalogReq req = {0};
req.pTableMeta = taosArrayInit(2, sizeof(SName));
req.pDbVgroup = taosArrayInit(2, TSDB_DB_FNAME_LEN);
@ -156,16 +158,16 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t re
req.pTableHash = taosArrayInit(2, sizeof(SName));
req.pUdf = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
req.pDbCfg = taosArrayInit(2, TSDB_DB_FNAME_LEN);
req.pIndex = NULL;//taosArrayInit(2, TSDB_INDEX_FNAME_LEN);
req.pIndex = NULL; // taosArrayInit(2, TSDB_INDEX_FNAME_LEN);
req.pUser = taosArrayInit(2, sizeof(SUserAuthInfo));
req.qNodeRequired = true;
req.forceUpdate = forceUpdate;
SName name = {0};
char dbFName[TSDB_DB_FNAME_LEN] = {0};
char funcName[TSDB_FUNC_NAME_LEN] = {0};
SName name = {0};
char dbFName[TSDB_DB_FNAME_LEN] = {0};
char funcName[TSDB_FUNC_NAME_LEN] = {0};
SUserAuthInfo user = {0};
tNameFromString(&name, "1.db1.tb1", T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
taosArrayPush(req.pTableMeta, &name);
taosArrayPush(req.pTableHash, &name);
@ -207,7 +209,7 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t re
int32_t *param = taosMemoryCalloc(1, sizeof(int32_t));
*param = 1;
int64_t jobId = 0;
CTG_ERR_JRET(catalogAsyncGetAllMeta(pCtg, pConn, &req, ctgdUserCallback, param, &jobId));
@ -221,7 +223,7 @@ _return:
taosArrayDestroy(req.pDbCfg);
taosArrayDestroy(req.pUser);
CTG_RET(code);
CTG_RET(code);
}
int32_t ctgdEnableDebug(char *option) {
@ -250,7 +252,7 @@ int32_t ctgdEnableDebug(char *option) {
}
qError("invalid debug option:%s", option);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
@ -261,7 +263,7 @@ int32_t ctgdGetStatNum(char *option, void *res) {
}
qError("invalid stat option:%s", option);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
@ -287,7 +289,7 @@ int32_t ctgdGetRentNum(SCtgRentMgmt *rent) {
return num;
}
int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) {
if (NULL == pCtg || NULL == pCtg->dbCache) {
return 0;
}
@ -304,8 +306,8 @@ int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
}
SCtgDBCache *dbCache = NULL;
int32_t num = 0;
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
int32_t num = 0;
void *pIter = taosHashIterate(pCtg->dbCache, NULL);
while (pIter) {
dbCache = (SCtgDBCache *)pIter;
switch (type) {
@ -325,7 +327,7 @@ int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
return num;
}
void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
void ctgdShowTableMeta(SCatalog *pCtg, const char *tbName, STableMeta *p) {
if (!gCTGDebug.metaEnable) {
return;
}
@ -333,11 +335,14 @@ void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
STableComInfo *c = &p->tableInfo;
if (TSDB_CHILD_TABLE == p->tableType) {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64, tbName, p->tableType, p->vgId,
p->uid, p->suid);
return;
} else {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64
",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision,
c->numOfColumns, c->rowSize);
}
int32_t colNum = c->numOfColumns + c->numOfTags;
@ -347,18 +352,18 @@ void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
}
}
void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
if (NULL == dbHash || !gCTGDebug.cacheEnable) {
return;
}
int32_t i = 0;
int32_t i = 0;
SCtgDBCache *dbCache = NULL;
void *pIter = taosHashIterate(dbHash, NULL);
void *pIter = taosHashIterate(dbHash, NULL);
while (pIter) {
char *dbFName = NULL;
char *dbFName = NULL;
size_t len = 0;
dbCache = (SCtgDBCache *)pIter;
dbFName = taosHashGetKey(pIter, &len);
@ -380,29 +385,29 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash);
}
}
ctgDebug("[%d] db [%.*s][0x%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, hashPrefix, hashSuffix, vgNum);
ctgDebug("[%d] db [%.*s][0x%" PRIx64
"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion,
hashMethod, hashPrefix, hashSuffix, vgNum);
pIter = taosHashIterate(dbHash, pIter);
}
}
void ctgdShowClusterCache(SCatalog* pCtg) {
void ctgdShowClusterCache(SCatalog *pCtg) {
if (!gCTGDebug.cacheEnable || NULL == pCtg) {
return;
}
ctgDebug("## cluster 0x%"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgDebug("## cluster 0x%" PRIx64 " %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgdShowDBCache(pCtg, pCtg->dbCache);
ctgDebug("## cluster 0x%"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg);
ctgDebug("## cluster 0x%" PRIx64 " %p cache Info END ##", pCtg->clusterId, pCtg);
}
int32_t ctgdShowCacheInfo(void) {
@ -413,19 +418,18 @@ int32_t ctgdShowCacheInfo(void) {
CTG_API_ENTER();
qDebug("# total catalog cluster number %d #", taosHashGetSize(gCtgMgmt.pCluster));
SCatalog *pCtg = NULL;
void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
while (pIter) {
pCtg = *(SCatalog **)pIter;
if (pCtg) {
ctgdShowClusterCache(pCtg);
}
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
}
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}

400
source/libs/catalog/src/ctgUtil.c
View File

@ -13,11 +13,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "trpc.h"
#include "query.h"
#include "tname.h"
#include "catalogInt.h"
#include "query.h"
#include "systable.h"
#include "tname.h"
#include "trpc.h"
void ctgFreeMsgSendParam(void* param) {
if (NULL == param) {
@ -39,16 +39,16 @@ void ctgFreeBatchMsg(void* msg) {
taosMemoryFree(pMsg->msg);
}
void ctgFreeBatch(SCtgBatch *pBatch) {
void ctgFreeBatch(SCtgBatch* pBatch) {
if (NULL == pBatch) {
return;
}
taosArrayDestroyEx(pBatch->pMsgs, ctgFreeBatchMsg);
taosArrayDestroy(pBatch->pTaskIds);
}
void ctgFreeBatchs(SHashObj *pBatchs) {
void ctgFreeBatchs(SHashObj* pBatchs) {
void* p = taosHashIterate(pBatchs, NULL);
while (NULL != p) {
SCtgBatch* pBatch = (SCtgBatch*)p;
@ -61,7 +61,7 @@ void ctgFreeBatchs(SHashObj *pBatchs) {
taosHashCleanup(pBatchs);
}
char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
char* ctgTaskTypeStr(CTG_TASK_TYPE type) {
switch (type) {
case CTG_TASK_GET_QNODE:
return "[get qnode list]";
@ -98,7 +98,7 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
}
}
void ctgFreeQNode(SCtgQNode *node) {
void ctgFreeQNode(SCtgQNode* node) {
if (NULL == node) {
return;
}
@ -107,16 +107,16 @@ void ctgFreeQNode(SCtgQNode *node) {
taosMemoryFree(node->op->data);
taosMemoryFree(node->op);
}
taosMemoryFree(node);
}
void ctgFreeSTableIndex(void *info) {
void ctgFreeSTableIndex(void* info) {
if (NULL == info) {
return;
}
STableIndex *pInfo = (STableIndex *)info;
STableIndex* pInfo = (STableIndex*)info;
taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo);
}
@ -125,42 +125,42 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pTableMeta);
pData->pTableMeta = NULL;
/*
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
taosArrayDestroy(*pArray);
}
*/
/*
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
taosArrayDestroy(*pArray);
}
*/
taosArrayDestroy(pData->pDbVgroup);
pData->pDbVgroup = NULL;
taosArrayDestroy(pData->pTableHash);
pData->pTableHash = NULL;
taosArrayDestroy(pData->pTableIndex);
pData->pTableIndex = NULL;
taosArrayDestroy(pData->pUdfList);
pData->pUdfList = NULL;
/*
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
taosArrayDestroy(pInfo->pRetensions);
}
*/
/*
for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
taosArrayDestroy(pInfo->pRetensions);
}
*/
taosArrayDestroy(pData->pDbCfg);
pData->pDbCfg = NULL;
taosArrayDestroy(pData->pDbInfo);
pData->pDbInfo = NULL;
taosArrayDestroy(pData->pIndex);
pData->pIndex = NULL;
taosArrayDestroy(pData->pUser);
pData->pUser = NULL;
taosArrayDestroy(pData->pQnodeList);
pData->pQnodeList = NULL;
@ -173,19 +173,19 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosMemoryFreeClear(pData->pSvrVer);
}
void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) {
void ctgFreeSCtgUserAuth(SCtgUserAuth* userCache) {
taosHashCleanup(userCache->createdDbs);
taosHashCleanup(userCache->readDbs);
taosHashCleanup(userCache->writeDbs);
}
void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
void ctgFreeMetaRent(SCtgRentMgmt* mgmt) {
if (NULL == mgmt->slots) {
return;
}
for (int32_t i = 0; i < mgmt->slotNum; ++i) {
SCtgRentSlot *slot = &mgmt->slots[i];
SCtgRentSlot* slot = &mgmt->slots[i];
if (slot->meta) {
taosArrayDestroy(slot->meta);
slot->meta = NULL;
@ -195,18 +195,18 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
taosMemoryFreeClear(mgmt->slots);
}
void ctgFreeStbMetaCache(SCtgDBCache *dbCache) {
void ctgFreeStbMetaCache(SCtgDBCache* dbCache) {
if (NULL == dbCache->stbCache) {
return;
}
int32_t stbNum = taosHashGetSize(dbCache->stbCache);
int32_t stbNum = taosHashGetSize(dbCache->stbCache);
taosHashCleanup(dbCache->stbCache);
dbCache->stbCache = NULL;
CTG_CACHE_STAT_DEC(numOfStb, stbNum);
}
void ctgFreeTbCacheImpl(SCtgTbCache *pCache) {
void ctgFreeTbCacheImpl(SCtgTbCache* pCache) {
qDebug("tbMeta freed, p:%p", pCache->pMeta);
taosMemoryFreeClear(pCache->pMeta);
if (pCache->pIndex) {
@ -215,13 +215,13 @@ void ctgFreeTbCacheImpl(SCtgTbCache *pCache) {
}
}
void ctgFreeTbCache(SCtgDBCache *dbCache) {
void ctgFreeTbCache(SCtgDBCache* dbCache) {
if (NULL == dbCache->tbCache) {
return;
}
int32_t tblNum = taosHashGetSize(dbCache->tbCache);
SCtgTbCache *pCache = taosHashIterate(dbCache->tbCache, NULL);
int32_t tblNum = taosHashGetSize(dbCache->tbCache);
SCtgTbCache* pCache = taosHashIterate(dbCache->tbCache, NULL);
while (NULL != pCache) {
ctgFreeTbCacheImpl(pCache);
pCache = taosHashIterate(dbCache->tbCache, pCache);
@ -231,7 +231,7 @@ void ctgFreeTbCache(SCtgDBCache *dbCache) {
CTG_CACHE_STAT_DEC(numOfTbl, tblNum);
}
void ctgFreeVgInfo(SDBVgInfo *vgInfo) {
void ctgFreeVgInfo(SDBVgInfo* vgInfo) {
if (NULL == vgInfo) {
return;
}
@ -240,15 +240,13 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) {
taosHashCleanup(vgInfo->vgHash);
vgInfo->vgHash = NULL;
}
taosMemoryFreeClear(vgInfo);
}
void ctgFreeVgInfoCache(SCtgDBCache *dbCache) {
ctgFreeVgInfo(dbCache->vgCache.vgInfo);
}
void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { ctgFreeVgInfo(dbCache->vgCache.vgInfo); }
void ctgFreeDbCache(SCtgDBCache *dbCache) {
void ctgFreeDbCache(SCtgDBCache* dbCache) {
if (NULL == dbCache) {
return;
}
@ -262,20 +260,20 @@ void ctgFreeInstDbCache(SHashObj* pDbCache) {
if (NULL == pDbCache) {
return;
}
int32_t dbNum = taosHashGetSize(pDbCache);
void *pIter = taosHashIterate(pDbCache, NULL);
void* pIter = taosHashIterate(pDbCache, NULL);
while (pIter) {
SCtgDBCache *dbCache = pIter;
SCtgDBCache* dbCache = pIter;
atomic_store_8(&dbCache->deleted, 1);
ctgFreeDbCache(dbCache);
pIter = taosHashIterate(pDbCache, pIter);
}
}
taosHashCleanup(pDbCache);
CTG_CACHE_STAT_DEC(numOfDb, dbNum);
}
@ -283,19 +281,19 @@ void ctgFreeInstUserCache(SHashObj* pUserCache) {
if (NULL == pUserCache) {
return;
}
int32_t userNum = taosHashGetSize(pUserCache);
void *pIter = taosHashIterate(pUserCache, NULL);
void* pIter = taosHashIterate(pUserCache, NULL);
while (pIter) {
SCtgUserAuth *userCache = pIter;
SCtgUserAuth* userCache = pIter;
ctgFreeSCtgUserAuth(userCache);
pIter = taosHashIterate(pUserCache, pIter);
}
}
taosHashCleanup(pUserCache);
CTG_CACHE_STAT_DEC(numOfUser, userNum);
}
@ -309,7 +307,6 @@ void ctgFreeHandleImpl(SCatalog* pCtg) {
taosMemoryFree(pCtg);
}
void ctgFreeHandle(SCatalog* pCtg) {
if (NULL == pCtg) {
return;
@ -345,13 +342,15 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB);
ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE);
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false,
HASH_ENTRY_LOCK);
if (NULL == pCtg->dbCache) {
qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
}
pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false,
HASH_ENTRY_LOCK);
if (NULL == pCtg->userCache) {
ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
}
@ -370,7 +369,7 @@ void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) {
taosHashCleanup(pOutput->dbVgroup->vgHash);
taosMemoryFreeClear(pOutput->dbVgroup);
}
taosMemoryFree(pOutput);
}
@ -379,7 +378,7 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
if (NULL == pCtx->out) {
return;
}
switch (pCtx->reqType) {
case TDMT_MND_GET_DB_CFG: {
SDbCfgInfo* pOut = (SDbCfgInfo*)pCtx->out;
@ -387,7 +386,7 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
taosMemoryFreeClear(pCtx->out);
break;
}
case TDMT_MND_USE_DB:{
case TDMT_MND_USE_DB: {
SUseDbOutput* pOut = (SUseDbOutput*)pCtx->out;
ctgFreeSUseDbOutput(pOut);
pCtx->out = NULL;
@ -459,17 +458,16 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) {
if (NULL == pOutput) {
return;
}
taosMemoryFree(pOutput->tbMeta);
taosMemoryFree(pOutput);
}
void ctgResetTbMetaTask(SCtgTask* pTask) {
SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx;
memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo));
taskCtx->flag = CTG_FLAG_UNKNOWN_STB;
if (pTask->msgCtx.lastOut) {
ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut);
pTask->msgCtx.lastOut = NULL;
@ -486,7 +484,7 @@ void ctgFreeBatchMeta(void* meta) {
if (NULL == meta) {
return;
}
SMetaRes* pRes = (SMetaRes*)meta;
taosMemoryFreeClear(pRes->pRes);
}
@ -495,13 +493,12 @@ void ctgFreeBatchHash(void* hash) {
if (NULL == hash) {
return;
}
SMetaRes* pRes = (SMetaRes*)hash;
taosMemoryFreeClear(pRes->pRes);
}
void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) {
void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
switch (type) {
case CTG_TASK_GET_QNODE:
case CTG_TASK_GET_DNODE:
@ -534,8 +531,8 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) {
case CTG_TASK_GET_TB_HASH:
case CTG_TASK_GET_DB_INFO:
case CTG_TASK_GET_INDEX:
case CTG_TASK_GET_UDF:
case CTG_TASK_GET_USER:
case CTG_TASK_GET_UDF:
case CTG_TASK_GET_USER:
case CTG_TASK_GET_SVR_VER:
case CTG_TASK_GET_TB_META: {
taosMemoryFreeClear(*pRes);
@ -547,7 +544,7 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) {
for (int32_t i = 0; i < num; ++i) {
ctgFreeBatchMeta(taosArrayGet(pArray, i));
}
*pRes = NULL; // no need to free it
*pRes = NULL; // no need to free it
break;
}
case CTG_TASK_GET_TB_HASH_BATCH: {
@ -556,17 +553,16 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) {
for (int32_t i = 0; i < num; ++i) {
ctgFreeBatchHash(taosArrayGet(pArray, i));
}
*pRes = NULL; // no need to free it
*pRes = NULL; // no need to free it
break;
}
}
default:
qError("invalid task type %d", type);
break;
}
}
void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) {
void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void** pRes) {
switch (type) {
case CTG_TASK_GET_QNODE:
case CTG_TASK_GET_DNODE: {
@ -603,12 +599,12 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) {
}
break;
}
case CTG_TASK_GET_TB_META:
case CTG_TASK_GET_TB_META:
case CTG_TASK_GET_DB_INFO:
case CTG_TASK_GET_TB_HASH:
case CTG_TASK_GET_INDEX:
case CTG_TASK_GET_UDF:
case CTG_TASK_GET_SVR_VER:
case CTG_TASK_GET_INDEX:
case CTG_TASK_GET_UDF:
case CTG_TASK_GET_SVR_VER:
case CTG_TASK_GET_USER: {
taosMemoryFreeClear(*pRes);
break;
@ -629,8 +625,7 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) {
}
}
void ctgClearSubTaskRes(SCtgSubRes *pRes) {
void ctgClearSubTaskRes(SCtgSubRes* pRes) {
pRes->code = 0;
if (NULL == pRes->res) {
@ -659,7 +654,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
// NO NEED TO FREE pNames
taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeTbMetasMsgCtx);
if (pTask->msgCtx.lastOut) {
ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut);
pTask->msgCtx.lastOut = NULL;
@ -670,7 +665,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
case CTG_TASK_GET_TB_HASH: {
SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx;
taosMemoryFreeClear(taskCtx->pName);
taosMemoryFreeClear(pTask->taskCtx);
taosMemoryFreeClear(pTask->taskCtx);
break;
}
case CTG_TASK_GET_TB_HASH_BATCH: {
@ -680,10 +675,10 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
// NO NEED TO FREE pNames
taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx);
taosMemoryFreeClear(pTask->taskCtx);
break;
}
}
case CTG_TASK_GET_TB_INDEX: {
SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx;
taosMemoryFreeClear(taskCtx->pName);
@ -699,10 +694,10 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
}
case CTG_TASK_GET_DB_VGROUP:
case CTG_TASK_GET_DB_CFG:
case CTG_TASK_GET_DB_INFO:
case CTG_TASK_GET_DB_INFO:
case CTG_TASK_GET_INDEX:
case CTG_TASK_GET_UDF:
case CTG_TASK_GET_QNODE:
case CTG_TASK_GET_QNODE:
case CTG_TASK_GET_USER: {
taosMemoryFreeClear(pTask->taskCtx);
break;
@ -713,7 +708,6 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
}
}
void ctgFreeTask(SCtgTask* pTask) {
ctgFreeMsgCtx(&pTask->msgCtx);
ctgFreeTaskRes(pTask->type, &pTask->res);
@ -741,10 +735,10 @@ void ctgFreeJob(void* job) {
if (NULL == job) {
return;
}
SCtgJob* pJob = (SCtgJob*)job;
int64_t rid = pJob->refId;
int64_t rid = pJob->refId;
uint64_t qid = pJob->queryId;
ctgFreeTasks(pJob->pTasks);
@ -776,7 +770,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ
int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) {
SCtgMsgCtx ctx = {0};
ctx.reqType = reqType;
ctx.out = out;
if (target) {
@ -791,8 +785,7 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp* fp) {
switch (hashMethod) {
default:
*fp = MurmurHash3_32;
@ -802,29 +795,29 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) {
SHashObj *vgroupHash = NULL;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
int32_t code = 0;
int32_t vgNum = taosHashGetSize(vgHash);
int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList) {
SHashObj* vgroupHash = NULL;
SVgroupInfo* vgInfo = NULL;
SArray* vgList = NULL;
int32_t code = 0;
int32_t vgNum = taosHashGetSize(vgHash);
vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo));
if (NULL == vgList) {
ctgError("taosArrayInit failed, num:%d", vgNum);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
void *pIter = taosHashIterate(vgHash, NULL);
void* pIter = taosHashIterate(vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (NULL == taosArrayPush(vgList, vgInfo)) {
ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId);
taosHashCancelIterate(vgHash, pIter);
taosHashCancelIterate(vgHash, pIter);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
pIter = taosHashIterate(vgHash, pIter);
vgInfo = NULL;
}
@ -844,12 +837,11 @@ _return:
CTG_RET(code);
}
int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) {
int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) {
int32_t code = 0;
int32_t vgNum = taosHashGetSize(dbInfo->vgHash);
char db[TSDB_DB_FNAME_LEN] = {0};
char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db);
if (vgNum <= 0) {
@ -857,40 +849,43 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
SVgroupInfo *vgInfo = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN];
SVgroupInfo* vgInfo = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix);
void *pIter = taosHashIterate(dbInfo->vgHash, NULL);
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
taosHashCancelIterate(dbInfo->vgHash, pIter);
break;
}
pIter = taosHashIterate(dbInfo->vgHash, pIter);
vgInfo = NULL;
}
if (NULL == vgInfo) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash));
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db,
taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
*pVgroup = *vgInfo;
ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps,
vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId,
vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn,
vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
CTG_RET(code);
}
int32_t ctgHashValueComp(void const *lp, void const *rp) {
uint32_t *key = (uint32_t *)lp;
SVgroupInfo *pVg = *(SVgroupInfo **)rp;
int32_t ctgHashValueComp(void const* lp, void const* rp) {
uint32_t* key = (uint32_t*)lp;
SVgroupInfo* pVg = *(SVgroupInfo**)rp;
if (*key < pVg->hashBegin) {
return -1;
@ -902,8 +897,8 @@ int32_t ctgHashValueComp(void const *lp, void const *rp) {
}
int ctgVgInfoComp(const void* lp, const void* rp) {
SVgroupInfo *pLeft = *(SVgroupInfo **)lp;
SVgroupInfo *pRight = *(SVgroupInfo **)rp;
SVgroupInfo* pLeft = *(SVgroupInfo**)lp;
SVgroupInfo* pRight = *(SVgroupInfo**)rp;
if (pLeft->hashBegin < pRight->hashBegin) {
return -1;
} else if (pLeft->hashBegin > pRight->hashBegin) {
@ -913,22 +908,22 @@ int ctgVgInfoComp(const void* lp, const void* rp) {
return 0;
}
int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) {
int32_t code = 0;
int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx,
char* dbFName, SArray* pNames, bool update) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SMetaRes res = {0};
int32_t vgNum = taosHashGetSize(dbInfo->vgHash);
SMetaRes res = {0};
int32_t vgNum = taosHashGetSize(dbInfo->vgHash);
if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SVgroupInfo *vgInfo = NULL;
int32_t tbNum = taosArrayGetSize(pNames);
SVgroupInfo* vgInfo = NULL;
int32_t tbNum = taosArrayGetSize(pNames);
if (1 == vgNum) {
void *pIter = taosHashIterate(dbInfo->vgHash, NULL);
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
for (int32_t i = 0; i < tbNum; ++i) {
vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo));
if (NULL == vgInfo) {
@ -939,11 +934,11 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
*vgInfo = *(SVgroupInfo*)pIter;
ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps,
vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
if (update) {
SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx);
SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
pRes->pRes = vgInfo;
} else {
res.pRes = vgInfo;
@ -956,7 +951,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
}
SArray* pVgList = taosArrayInit(vgNum, POINTER_BYTES);
void *pIter = taosHashIterate(dbInfo->vgHash, NULL);
void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) {
taosArrayPush(pVgList, &pIter);
pIter = taosHashIterate(dbInfo->vgHash, pIter);
@ -967,21 +962,23 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
char tbFullName[TSDB_TABLE_FNAME_LEN];
sprintf(tbFullName, "%s.", dbFName);
int32_t offset = strlen(tbFullName);
SName* pName = NULL;
SName* pName = NULL;
int32_t tbNameLen = 0;
for (int32_t i = 0; i < tbNum; ++i) {
pName = taosArrayGet(pNames, i);
tbNameLen = offset + strlen(pName->tname);
strcpy(tbFullName + offset, pName->tname);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix);
SVgroupInfo **p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ);
SVgroupInfo** p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ);
if (NULL == p) {
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash));
ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName,
taosHashGetSize(dbInfo->vgHash));
taosArrayDestroy(pVgList);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
@ -996,17 +993,18 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
*pNewVg = *vgInfo;
ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps,
vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId,
vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn,
vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
if (update) {
SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx);
SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
pRes->pRes = pNewVg;
} else {
res.pRes = pNewVg;
taosArrayPush(pCtx->pResList, &res);
}
}
}
taosArrayDestroy(pVgList);
@ -1014,11 +1012,10 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
CTG_RET(code);
}
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) {
if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) {
return -1;
} else if (*(uint64_t *)key1 > ((SSTableVersion*)key2)->suid) {
} else if (*(uint64_t*)key1 > ((SSTableVersion*)key2)->suid) {
return 1;
} else {
return 0;
@ -1026,9 +1023,9 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) {
}
int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) {
if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) {
if (*(int64_t*)key1 < ((SDbVgVersion*)key2)->dbId) {
return -1;
} else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) {
} else if (*(int64_t*)key1 > ((SDbVgVersion*)key2)->dbId) {
return 1;
} else {
return 0;
@ -1055,10 +1052,7 @@ int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) {
}
}
int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
*dst = taosMemoryMalloc(sizeof(SDBVgInfo));
if (NULL == *dst) {
qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo));
@ -1075,29 +1069,26 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
int32_t *vgId = NULL;
void *pIter = taosHashIterate(src->vgHash, NULL);
int32_t* vgId = NULL;
void* pIter = taosHashIterate(src->vgHash, NULL);
while (pIter) {
vgId = taosHashGetKey(pIter, NULL);
if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
if (taosHashPut((*dst)->vgHash, (void*)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize);
taosHashCancelIterate(src->vgHash, pIter);
taosHashCleanup((*dst)->vgHash);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pIter = taosHashIterate(src->vgHash, pIter);
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) {
int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) {
*pOutput = taosMemoryMalloc(sizeof(STableMetaOutput));
if (NULL == *pOutput) {
qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput));
@ -1135,7 +1126,7 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) {
}
for (int32_t i = 0; i < num; ++i) {
STableIndexInfo *pInfo = taosArrayGet(pIndex, i);
STableIndexInfo* pInfo = taosArrayGet(pIndex, i);
pInfo = taosArrayPush(*pRes, pInfo);
pInfo->expr = strdup(pInfo->expr);
}
@ -1143,8 +1134,7 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId) {
int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId) {
if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META) {
pMsgSendInfo->target.type = TARGET_TYPE_VNODE;
pMsgSendInfo->target.vgId = vgId;
@ -1156,7 +1146,7 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, cha
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTablesReqNum(SArray *pList) {
int32_t ctgGetTablesReqNum(SArray* pList) {
if (NULL == pList) {
return 0;
}
@ -1164,25 +1154,25 @@ int32_t ctgGetTablesReqNum(SArray *pList) {
int32_t total = 0;
int32_t n = taosArrayGetSize(pList);
for (int32_t i = 0; i < n; ++i) {
STablesReq *pReq = taosArrayGet(pList, i);
STablesReq* pReq = taosArrayGet(pList, i);
total += taosArrayGetSize(pReq->pTables);
}
return total;
}
int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag) {
int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag) {
if (NULL == (*pFetchs)) {
*pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch));
}
SCtgFetch fetch = {0};
fetch.dbIdx = dbIdx;
fetch.tbIdx = tbIdx;
fetch.fetchIdx = (*fetchIdx)++;
fetch.resIdx = resIdx;
fetch.flag = flag;
taosArrayPush(*pFetchs, &fetch);
return TSDB_CODE_SUCCESS;
@ -1193,13 +1183,9 @@ SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) {
return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx);
}
static void* ctgCloneDbVgroup(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)pSrc); }
static void ctgFreeDbVgroup(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneDbCfgInfo(void* pSrc) {
SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo));
@ -1210,9 +1196,7 @@ static void* ctgCloneDbCfgInfo(void* pSrc) {
return pDst;
}
static void ctgFreeDbCfgInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeDbCfgInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneDbInfo(void* pSrc) {
SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo));
@ -1223,9 +1207,7 @@ static void* ctgCloneDbInfo(void* pSrc) {
return pDst;
}
static void ctgFreeDbInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneTableMeta(void* pSrc) {
STableMeta* pMeta = pSrc;
@ -1238,9 +1220,7 @@ static void* ctgCloneTableMeta(void* pSrc) {
return pDst;
}
static void ctgFreeTableMeta(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeTableMeta(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneVgroupInfo(void* pSrc) {
SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo));
@ -1251,17 +1231,11 @@ static void* ctgCloneVgroupInfo(void* pSrc) {
return pDst;
}
static void ctgFreeVgroupInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneTableIndices(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void* ctgCloneTableIndices(void* pSrc) { return taosArrayDup((const SArray*)pSrc); }
static void ctgFreeTableIndices(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void ctgFreeTableIndices(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneFuncInfo(void* pSrc) {
SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo));
@ -1272,9 +1246,7 @@ static void* ctgCloneFuncInfo(void* pSrc) {
return pDst;
}
static void ctgFreeFuncInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeFuncInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneIndexInfo(void* pSrc) {
SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo));
@ -1285,9 +1257,7 @@ static void* ctgCloneIndexInfo(void* pSrc) {
return pDst;
}
static void ctgFreeIndexInfo(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeIndexInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneUserAuth(void* pSrc) {
bool* pDst = taosMemoryMalloc(sizeof(bool));
@ -1298,17 +1268,11 @@ static void* ctgCloneUserAuth(void* pSrc) {
return pDst;
}
static void ctgFreeUserAuth(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeUserAuth(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneQnodeList(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void* ctgCloneQnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc); }
static void ctgFreeQnodeList(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneTableCfg(void* pSrc) {
STableCfg* pDst = taosMemoryMalloc(sizeof(STableCfg));
@ -1319,17 +1283,11 @@ static void* ctgCloneTableCfg(void* pSrc) {
return pDst;
}
static void ctgFreeTableCfg(void* p) {
taosMemoryFree(((SMetaRes*)p)->pRes);
}
static void ctgFreeTableCfg(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneDnodeList(void* pSrc) {
return taosArrayDup((const SArray*)pSrc);
}
static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc); }
static void ctgFreeDnodeList(void* p) {
taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes);
}
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static int32_t ctgCloneMetaDataArray(SArray* pSrc, FCopy copyFunc, SArray** pDst) {
if (NULL == pSrc) {
@ -1343,7 +1301,7 @@ static int32_t ctgCloneMetaDataArray(SArray* pSrc, FCopy copyFunc, SArray** pDst
}
for (int32_t i = 0; i < size; ++i) {
SMetaRes* pRes = taosArrayGet(pSrc, i);
SMetaRes res = {.code = pRes->code, .pRes = copyFunc(pRes->pRes)};
SMetaRes res = {.code = pRes->code, .pRes = copyFunc(pRes->pRes)};
if (NULL == res.pRes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -1412,8 +1370,8 @@ void catalogFreeMetaData(SMetaData* pData) {
taosArrayDestroyEx(pData->pDbInfo, ctgFreeDbInfo);
taosArrayDestroyEx(pData->pTableMeta, ctgFreeTableMeta);
taosArrayDestroyEx(pData->pTableHash, ctgFreeVgroupInfo);
taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndices);
taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo);
taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndices);
taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo);
taosArrayDestroyEx(pData->pIndex, ctgFreeIndexInfo);
taosArrayDestroyEx(pData->pUser, ctgFreeUserAuth);
taosArrayDestroyEx(pData->pQnodeList, ctgFreeQnodeList);

315
source/libs/catalog/test/catalogTests.cpp
View File

@ -27,20 +27,20 @@
#ifdef WINDOWS
#define TD_USE_WINSOCK
#endif
#include "os.h"
#include "tglobal.h"
#include "catalog.h"
#include "catalogInt.h"
#include "os.h"
#include "stub.h"
#include "taos.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tglobal.h"
#include "trpc.h"
#include "tvariant.h"
namespace {
extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type);
extern "C" int32_t ctgdEnableDebug(char *option);
extern "C" int32_t ctgdGetStatNum(char *option, void *res);
@ -49,7 +49,7 @@ void ctgTestSetRspCTableMeta();
void ctgTestSetRspSTableMeta();
void ctgTestSetRspMultiSTableMeta();
//extern "C" SCatalogMgmt gCtgMgmt;
// extern "C" SCatalogMgmt gCtgMgmt;
enum {
CTGT_RSP_VGINFO = 1,
@ -131,7 +131,7 @@ void ctgTestInitLogFile() {
if (!ctgTestEnableLog) {
return;
}
const char *defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
@ -142,7 +142,7 @@ void ctgTestInitLogFile() {
ctgdEnableDebug("api");
ctgdEnableDebug("meta");
ctgdEnableDebug("cache");
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
@ -153,11 +153,11 @@ int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) {
}
void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) {
SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName cn = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(cn.dbname, "db1");
strcpy(cn.tname, ctgTestCTablename);
SName sn = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName sn = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(sn.dbname, "db1");
strcpy(sn.tname, ctgTestSTablename);
@ -175,7 +175,8 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) {
output->ctbMeta.uid = 3;
output->ctbMeta.suid = 2;
output->tbMeta = (STableMeta *)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum));
output->tbMeta =
(STableMeta *)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum));
output->tbMeta->vgId = 9;
output->tbMeta->tableType = TSDB_SUPER_TABLE;
output->tbMeta->uid = 2;
@ -211,7 +212,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) {
static int32_t vgVersion = ctgTestVgVersion + 1;
int32_t vgNum = 0;
SVgroupInfo vgInfo = {0};
SDBVgInfo *dbVgroup = (SDBVgInfo *)taosMemoryCalloc(1, sizeof(SDBVgInfo));
SDBVgInfo *dbVgroup = (SDBVgInfo *)taosMemoryCalloc(1, sizeof(SDBVgInfo));
dbVgroup->vgVersion = vgVersion++;
@ -256,7 +257,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
rspMsg->suid = ctgTestSuid + 1;
rspMsg->tuid = ctgTestSuid + 1;
rspMsg->vgId = 1;
rspMsg->pSchemas = (SSchema *)taosMemoryCalloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema));
SSchema *s = NULL;
@ -364,7 +365,6 @@ void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, S
pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST;
}
void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
STableMetaRsp metaRsp = {0};
strcpy(metaRsp.dbFName, ctgTestDbname);
@ -573,8 +573,8 @@ void ctgTestSetRspDbVgroups() {
stub.set(rpcSendRecv, ctgTestRspDbVgroups);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -593,8 +593,8 @@ void ctgTestSetRspTableMeta() {
stub.set(rpcSendRecv, ctgTestRspTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -613,8 +613,8 @@ void ctgTestSetRspCTableMeta() {
stub.set(rpcSendRecv, ctgTestRspCTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -633,8 +633,8 @@ void ctgTestSetRspSTableMeta() {
stub.set(rpcSendRecv, ctgTestRspSTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -653,8 +653,8 @@ void ctgTestSetRspMultiSTableMeta() {
stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -673,8 +673,8 @@ void ctgTestSetRspByIdx() {
stub.set(rpcSendRecv, ctgTestRspByIdx);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -688,14 +688,13 @@ void ctgTestSetRspByIdx() {
}
}
void ctgTestSetRspDbVgroupsAndNormalMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -714,8 +713,8 @@ void ctgTestSetRspDbVgroupsAndChildMeta() {
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -734,8 +733,8 @@ void ctgTestSetRspDbVgroupsAndSuperMeta() {
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -754,8 +753,8 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() {
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
AddrAny any;
std::map<std::string, void *> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
@ -772,11 +771,11 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() {
} // namespace
void *ctgTestGetDbVgroupThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SArray *vgList = NULL;
int32_t n = 0;
SArray *vgList = NULL;
int32_t n = 0;
while (!ctgTestStop) {
code = catalogGetDBVgInfo(pCtg, mockPointer, ctgTestDbname, &vgList);
@ -826,7 +825,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) {
void *ctgTestSetDiffDbVgroupThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SDBVgInfo *dbVgroup = NULL;
SDBVgInfo *dbVgroup = NULL;
int32_t n = 0;
while (!ctgTestStop) {
@ -854,7 +853,7 @@ void *ctgTestGetCtableMetaThread(void *param) {
STableMeta *tbMeta = NULL;
bool inCache = false;
SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName cn = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(cn.dbname, "db1");
strcpy(cn.tname, ctgTestCTablename);
@ -883,14 +882,14 @@ void *ctgTestGetCtableMetaThread(void *param) {
}
void *ctgTestSetCtableMetaThread(void *param) {
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SDBVgInfo dbVgroup = {0};
int32_t n = 0;
struct SCatalog *pCtg = (struct SCatalog *)param;
int32_t code = 0;
SDBVgInfo dbVgroup = {0};
int32_t n = 0;
STableMetaOutput *output = NULL;
SCtgCacheOperation operation = {0};
operation.opId = CTG_OP_UPDATE_TB_META;
while (!ctgTestStop) {
@ -920,11 +919,10 @@ void *ctgTestSetCtableMetaThread(void *param) {
#if 1
TEST(tableMeta, normalTable) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -940,7 +938,7 @@ TEST(tableMeta, normalTable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -952,7 +950,7 @@ TEST(tableMeta, normalTable) {
while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
taosMsleep(50);
}
ctgTestSetRspTableMeta();
STableMeta *tableMeta = NULL;
@ -977,7 +975,6 @@ TEST(tableMeta, normalTable) {
}
}
tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
@ -990,10 +987,10 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
SDbVgVersion *dbs = NULL;
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
int32_t i = 0;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
int32_t i = 0;
while (i < 5) {
++i;
code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum);
@ -1030,9 +1027,9 @@ TEST(tableMeta, normalTable) {
}
TEST(tableMeta, childTableCase) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -1047,7 +1044,7 @@ TEST(tableMeta, childTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
@ -1072,7 +1069,6 @@ TEST(tableMeta, childTableCase) {
}
}
tableMeta = NULL;
code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta);
ASSERT_EQ(code, 0);
@ -1099,10 +1095,10 @@ TEST(tableMeta, childTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
SDbVgVersion *dbs = NULL;
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
int32_t i = 0;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
int32_t i = 0;
while (i < 5) {
++i;
code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum);
@ -1139,9 +1135,9 @@ TEST(tableMeta, childTableCase) {
}
TEST(tableMeta, superTableCase) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo vgInfo = {0};
ctgTestSetRspDbVgroupsAndSuperMeta();
@ -1154,7 +1150,7 @@ TEST(tableMeta, superTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -1181,7 +1177,6 @@ TEST(tableMeta, superTableCase) {
}
}
ctgTestSetRspCTableMeta();
tableMeta = NULL;
@ -1208,7 +1203,6 @@ TEST(tableMeta, superTableCase) {
}
}
tableMeta = NULL;
code = catalogRefreshGetTableMeta(pCtg, mockPointer, &n, &tableMeta, 0);
ASSERT_EQ(code, 0);
@ -1221,10 +1215,10 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.precision, 1);
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
SDbVgVersion *dbs = NULL;
SDbVgVersion *dbs = NULL;
SSTableVersion *stb = NULL;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
int32_t i = 0;
uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0;
int32_t i = 0;
while (i < 5) {
++i;
code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum);
@ -1262,9 +1256,9 @@ TEST(tableMeta, superTableCase) {
}
TEST(tableMeta, rmStbMeta) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -1279,7 +1273,7 @@ TEST(tableMeta, rmStbMeta) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -1306,7 +1300,6 @@ TEST(tableMeta, rmStbMeta) {
}
}
code = catalogRemoveStbMeta(pCtg, "1.db1", ctgTestDbId, ctgTestSTablename, ctgTestSuid - 1);
ASSERT_EQ(code, 0);
@ -1320,7 +1313,6 @@ TEST(tableMeta, rmStbMeta) {
}
}
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
@ -1332,9 +1324,9 @@ TEST(tableMeta, rmStbMeta) {
}
TEST(tableMeta, updateStbMeta) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo vgInfo = {0};
ctgTestInitLogFile();
@ -1349,7 +1341,7 @@ TEST(tableMeta, updateStbMeta) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -1376,7 +1368,6 @@ TEST(tableMeta, updateStbMeta) {
}
}
taosMemoryFreeClear(tableMeta);
STableMetaRsp rsp = {0};
@ -1422,10 +1413,10 @@ TEST(tableMeta, updateStbMeta) {
}
TEST(refreshGetMeta, normal2normal) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1434,7 +1425,7 @@ TEST(refreshGetMeta, normal2normal) {
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestRspFunc[2] = CTGT_RSP_TBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1447,7 +1438,7 @@ TEST(refreshGetMeta, normal2normal) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -1470,7 +1461,7 @@ TEST(refreshGetMeta, normal2normal) {
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
@ -1487,7 +1478,7 @@ TEST(refreshGetMeta, normal2normal) {
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
@ -1501,10 +1492,10 @@ TEST(refreshGetMeta, normal2normal) {
}
TEST(refreshGetMeta, normal2notexist) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1513,7 +1504,7 @@ TEST(refreshGetMeta, normal2notexist) {
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestRspFunc[2] = CTGT_RSP_TBMETA_NOT_EXIST;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1526,7 +1517,7 @@ TEST(refreshGetMeta, normal2notexist) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -1549,7 +1540,7 @@ TEST(refreshGetMeta, normal2notexist) {
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
@ -1570,12 +1561,11 @@ TEST(refreshGetMeta, normal2notexist) {
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
}
TEST(refreshGetMeta, normal2child) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1585,7 +1575,7 @@ TEST(refreshGetMeta, normal2child) {
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestRspFunc[2] = CTGT_RSP_CTBMETA;
ctgTestRspFunc[3] = CTGT_RSP_STBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1598,7 +1588,7 @@ TEST(refreshGetMeta, normal2child) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentCTableName = ctgTestTablename;
@ -1623,7 +1613,7 @@ TEST(refreshGetMeta, normal2child) {
ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 8);
ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->uid, ctgTestNormalTblUid - 1);
ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
@ -1655,10 +1645,10 @@ TEST(refreshGetMeta, normal2child) {
}
TEST(refreshGetMeta, stable2child) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1669,7 +1659,7 @@ TEST(refreshGetMeta, stable2child) {
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_CTBMETA;
ctgTestRspFunc[4] = CTGT_RSP_STBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1682,7 +1672,7 @@ TEST(refreshGetMeta, stable2child) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentSTableName = ctgTestTablename;
@ -1741,10 +1731,10 @@ TEST(refreshGetMeta, stable2child) {
}
TEST(refreshGetMeta, stable2stable) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1755,7 +1745,7 @@ TEST(refreshGetMeta, stable2stable) {
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_STBMETA;
ctgTestRspFunc[4] = CTGT_RSP_STBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1768,7 +1758,7 @@ TEST(refreshGetMeta, stable2stable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentSTableName = ctgTestTablename;
@ -1826,12 +1816,11 @@ TEST(refreshGetMeta, stable2stable) {
ctgTestCurrentSTableName = NULL;
}
TEST(refreshGetMeta, child2stable) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1842,7 +1831,7 @@ TEST(refreshGetMeta, child2stable) {
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_STBMETA;
ctgTestRspFunc[4] = CTGT_RSP_STBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1855,7 +1844,7 @@ TEST(refreshGetMeta, child2stable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentCTableName = ctgTestTablename;
@ -1913,12 +1902,11 @@ TEST(refreshGetMeta, child2stable) {
ctgTestCurrentSTableName = NULL;
}
TEST(tableDistVgroup, normalTable) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1927,7 +1915,7 @@ TEST(tableDistVgroup, normalTable) {
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestRspFunc[2] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1940,7 +1928,7 @@ TEST(tableDistVgroup, normalTable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -1956,10 +1944,10 @@ TEST(tableDistVgroup, normalTable) {
}
TEST(tableDistVgroup, childTableCase) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -1969,7 +1957,7 @@ TEST(tableDistVgroup, childTableCase) {
ctgTestRspFunc[1] = CTGT_RSP_CTBMETA;
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
@ -1982,7 +1970,7 @@ TEST(tableDistVgroup, childTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
@ -1998,10 +1986,10 @@ TEST(tableDistVgroup, childTableCase) {
}
TEST(tableDistVgroup, superTableCase) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
SVgroupInfo *vgInfo = NULL;
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -2011,11 +1999,9 @@ TEST(tableDistVgroup, superTableCase) {
ctgTestRspFunc[1] = CTGT_RSP_STBMETA;
ctgTestRspFunc[2] = CTGT_RSP_STBMETA;
ctgTestRspFunc[3] = CTGT_RSP_VGINFO;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
int32_t code = catalogInit(NULL);
@ -2025,7 +2011,7 @@ TEST(tableDistVgroup, superTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -2047,12 +2033,12 @@ TEST(tableDistVgroup, superTableCase) {
}
TEST(dbVgroup, getSetDbVgroupCase) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo *dbVgroup = NULL;
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo *dbVgroup = NULL;
SArray *vgList = NULL;
ctgTestInitLogFile();
@ -2061,10 +2047,8 @@ TEST(dbVgroup, getSetDbVgroupCase) {
ctgTestRspFunc[0] = CTGT_RSP_VGINFO;
ctgTestRspFunc[1] = CTGT_RSP_TBMETA;
ctgTestSetRspByIdx();
initQueryModuleMsgHandle();
// sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
@ -2075,7 +2059,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2119,7 +2103,6 @@ TEST(dbVgroup, getSetDbVgroupCase) {
}
}
code = catalogGetTableHashVgroup(pCtg, mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 7);
@ -2138,12 +2121,12 @@ TEST(dbVgroup, getSetDbVgroupCase) {
}
TEST(multiThread, getSetRmSameDbVgroup) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
ctgTestInitLogFile();
@ -2160,7 +2143,7 @@ TEST(multiThread, getSetRmSameDbVgroup) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2190,12 +2173,12 @@ TEST(multiThread, getSetRmSameDbVgroup) {
}
TEST(multiThread, getSetRmDiffDbVgroup) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
ctgTestInitLogFile();
@ -2212,7 +2195,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2242,12 +2225,12 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
}
TEST(multiThread, ctableMeta) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
ctgTestInitLogFile();
@ -2264,7 +2247,7 @@ TEST(multiThread, ctableMeta) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2293,16 +2276,16 @@ TEST(multiThread, ctableMeta) {
}
TEST(rentTest, allRent) {
struct SCatalog *pCtg = NULL;
struct SCatalog *pCtg = NULL;
SRequestConnInfo *mockPointer = (SRequestConnInfo *)0x1;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
SVgroupInfo vgInfo = {0};
SVgroupInfo *pvgInfo = NULL;
SDBVgInfo dbVgroup = {0};
SArray *vgList = NULL;
ctgTestStop = false;
SDbVgVersion *dbs = NULL;
SDbVgVersion *dbs = NULL;
SSTableVersion *stable = NULL;
uint32_t num = 0;
uint32_t num = 0;
ctgTestInitLogFile();
@ -2316,7 +2299,7 @@ TEST(rentTest, allRent) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}};
strcpy(n.dbname, "db1");
for (int32_t i = 1; i <= 10; ++i) {

4
source/libs/command/test/commandTest.cpp
View File

@ -16,6 +16,6 @@
#include <gtest/gtest.h>
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

20
source/libs/executor/inc/dataSinkInt.h
View File

@ -20,16 +20,16 @@
extern "C" {
#endif
#include "tcommon.h"
#include "dataSinkMgt.h"
#include "plannodes.h"
#include "tcommon.h"
struct SDataSink;
struct SDataSinkHandle;
typedef struct SDataSinkManager {
SDataSinkMgtCfg cfg;
TdThreadMutex mutex;
TdThreadMutex mutex;
} SDataSinkManager;
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);
@ -40,17 +40,19 @@ typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle);
typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size);
typedef struct SDataSinkHandle {
FPutDataBlock fPut;
FEndPut fEndPut;
FGetDataLength fGetLen;
FGetDataBlock fGetData;
FPutDataBlock fPut;
FEndPut fEndPut;
FGetDataLength fGetLen;
FGetDataBlock fGetData;
FDestroyDataSinker fDestroy;
FGetCacheSize fGetCacheSize;
FGetCacheSize fGetCacheSize;
} SDataSinkHandle;
int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle);
int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam);
int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam);
int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle,
void* pParam);
int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle,
void* pParam);
#ifdef __cplusplus
}

8
source/libs/executor/inc/executorInt.h
View File

@ -23,10 +23,10 @@ extern "C" {
extern int32_t exchangeObjRefPool;
typedef struct {
char* pData;
bool isNull;
int16_t type;
int32_t bytes;
char* pData;
bool isNull;
int16_t type;
int32_t bytes;
} SGroupKeys, SStateKeys;
uint64_t calcGroupId(char* pData, int32_t len);

10
source/libs/executor/inc/tlinearhash.h
View File

@ -24,7 +24,7 @@ extern "C" {
enum {
LINEAR_HASH_STATIS = 0x1,
LINEAR_HASH_DATA = 0x2,
LINEAR_HASH_DATA = 0x2,
};
typedef struct SLHashObj SLHashObj;
@ -32,11 +32,11 @@ typedef struct SLHashObj SLHashObj;
SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage);
void* tHashCleanup(SLHashObj* pHashObj);
int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data, size_t size);
char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen);
int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen);
int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data, size_t size);
char* tHashGet(SLHashObj* pHashObj, const void* key, size_t keyLen);
int32_t tHashRemove(SLHashObj* pHashObj, const void* key, size_t keyLen);
void tHashPrint(const SLHashObj* pHashObj, int32_t type);
void tHashPrint(const SLHashObj* pHashObj, int32_t type);
#ifdef __cplusplus
}

2
source/libs/executor/inc/tsimplehash.h
View File

@ -112,7 +112,7 @@ void tSimpleHashCleanup(SSHashObj *pHashObj);
size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj);
#pragma pack(push, 4)
typedef struct SHNode{
typedef struct SHNode {
struct SHNode *next;
uint32_t keyLen : 20;
uint32_t dataLen : 12;

30
source/libs/executor/inc/tsort.h
View File

@ -20,8 +20,8 @@
extern "C" {
#endif
#include "tcommon.h"
#include "os.h"
#include "tcommon.h"
enum {
SORT_MULTISOURCE_MERGE = 0x1,
@ -31,29 +31,29 @@ enum {
typedef struct SMultiMergeSource {
int32_t type;
int32_t rowIndex;
SSDataBlock *pBlock;
SSDataBlock* pBlock;
} SMultiMergeSource;
typedef struct SSortSource {
SMultiMergeSource src;
union{
struct{
SArray* pageIdList;
int32_t pageIndex;
union {
struct {
SArray* pageIdList;
int32_t pageIndex;
};
void *param;
void* param;
};
} SSortSource;
typedef struct SMsortComparParam {
void **pSources;
int32_t numOfSources;
SArray *orderInfo; // SArray<SBlockOrderInfo>
bool cmpGroupId;
void** pSources;
int32_t numOfSources;
SArray* orderInfo; // SArray<SBlockOrderInfo>
bool cmpGroupId;
} SMsortComparParam;
typedef struct SSortHandle SSortHandle;
typedef struct SSortHandle SSortHandle;
typedef struct STupleHandle STupleHandle;
typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param);
@ -64,7 +64,8 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void*
* @param type
* @return
*/
SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr);
SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages,
SSDataBlock* pBlock, const char* idstr);
/**
*
@ -90,7 +91,8 @@ int32_t tsortClose(SSortHandle* pHandle);
*
* @return
*/
int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*), void* param);
int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*),
void* param);
/**
*

56
source/libs/executor/src/cachescanoperator.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "function.h"
#include "os.h"
#include "tname.h"
#include "tdatablock.h"
@ -26,8 +26,8 @@
#include "ttypes.h"
static SSDataBlock* doScanCache(SOperatorInfo* pOperator);
static void destroyLastrowScanOperator(void* param);
static int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
static void destroyLastrowScanOperator(void* param);
static int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo) {
@ -40,11 +40,11 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
}
pInfo->readHandle = *readHandle;
pInfo->pRes = createResDataBlock(pScanNode->scan.node.pOutputDataBlockDesc);
pInfo->pRes = createResDataBlock(pScanNode->scan.node.pOutputDataBlockDesc);
int32_t numOfCols = 0;
pInfo->pColMatchInfo = extractColMatchInfo(pScanNode->scan.pScanCols, pScanNode->scan.node.pOutputDataBlockDesc, &numOfCols,
COL_MATCH_FROM_COL_ID);
pInfo->pColMatchInfo = extractColMatchInfo(pScanNode->scan.pScanCols, pScanNode->scan.node.pOutputDataBlockDesc,
&numOfCols, COL_MATCH_FROM_COL_ID);
code = extractTargetSlotId(pInfo->pColMatchInfo, pTaskInfo, &pInfo->pSlotIds);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@ -58,7 +58,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
// partition by tbname
if (taosArrayGetSize(pTableList->pGroupList) == taosArrayGetSize(pTableList->pTableList)) {
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL|CACHESCAN_RETRIEVE_LAST_ROW;
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | CACHESCAN_RETRIEVE_LAST_ROW;
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pTableList->pTableList,
taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader);
if (code != TSDB_CODE_SUCCESS) {
@ -67,23 +67,24 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
blockDataEnsureCapacity(pInfo->pBufferredRes, pOperator->resultInfo.capacity);
} else { // by tags
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE|CACHESCAN_RETRIEVE_LAST_ROW;
} else { // by tags
pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | CACHESCAN_RETRIEVE_LAST_ROW;
}
if (pScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* pPseudoExpr = &pInfo->pseudoExprSup;
pPseudoExpr->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &pPseudoExpr->numOfExprs);
pPseudoExpr->pCtx = createSqlFunctionCtx(pPseudoExpr->pExprInfo, pPseudoExpr->numOfExprs, &pPseudoExpr->rowEntryInfoOffset);
pPseudoExpr->pCtx =
createSqlFunctionCtx(pPseudoExpr->pExprInfo, pPseudoExpr->numOfExprs, &pPseudoExpr->rowEntryInfoOffset);
}
pOperator->name = "LastrowScanOperator";
pOperator->name = "LastrowScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
pOperator->fpSet =
@ -92,7 +93,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pOperator->cost.openCost = 0;
return pOperator;
_error:
_error:
pTaskInfo->code = code;
destroyLastrowScanOperator(pInfo);
taosMemoryFree(pOperator);
@ -121,7 +122,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pBufferredRes);
taosArrayClear(pInfo->pUidList);
int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, pInfo->pUidList);
int32_t code =
tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@ -133,15 +135,15 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
}
if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) {
for(int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
SColMatchInfo* pMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
int32_t slotId = pMatchInfo->targetSlotId;
int32_t slotId = pMatchInfo->targetSlotId;
SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId);
SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, slotId);
char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes);
bool isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes);
bool isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes);
colDataAppend(pDst, 0, p, isNull);
}
@ -150,8 +152,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
if (pInfo->pseudoExprSup.numOfExprs > 0) {
SExprSupp* pSup = &pInfo->pseudoExprSup;
int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes,
GET_TASKID(pTaskInfo));
int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes,
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
return NULL;
@ -180,7 +182,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
SArray* pGroupTableList = taosArrayGetP(pTableList->pGroupList, pInfo->currentGroupIndex);
tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pGroupTableList,
taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader);
taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader);
taosArrayClear(pInfo->pUidList);
int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList);
@ -200,8 +202,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
pInfo->pRes->info.groupId = pKeyInfo->groupId;
code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes,
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
return NULL;
}
@ -233,10 +235,10 @@ void destroyLastrowScanOperator(void* param) {
}
int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds) {
size_t numOfCols = taosArrayGetSize(pColMatchInfo);
size_t numOfCols = taosArrayGetSize(pColMatchInfo);
*pSlotIds = taosMemoryMalloc(numOfCols * sizeof(int32_t));
if (*pSlotIds == NULL) {
if (*pSlotIds == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}

31
source/libs/executor/src/dataDispatcher.c
View File

@ -93,8 +93,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pBuf->useSize = sizeof(SDataCacheEntry);
blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4));
pBuf->useSize += pEntry->dataLen;
@ -103,14 +103,14 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
}
static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
/*
uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueItemSize(pDispatcher->pDataBlocks));
return false;
}
*/
/*
uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueItemSize(pDispatcher->pDataBlocks));
return false;
}
*/
pBuf->allocSize = sizeof(SDataCacheEntry) + blockGetEncodeSize(pInput->pData);
@ -176,11 +176,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData;
*pLen = pEntry->dataLen;
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4));
*pQueryEnd = pDispatcher->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen,
((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
}
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
@ -199,8 +200,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed;
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4));
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);

80
source/libs/executor/src/dataInserter.c
View File

@ -27,7 +27,7 @@ extern SDataSinkStat gDataSinkStat;
typedef struct SSubmitRes {
int64_t affectedRows;
int32_t code;
SSubmitRsp *pRsp;
SSubmitRsp* pRsp;
} SSubmitRes;
typedef struct SDataInserterHandle {
@ -44,7 +44,7 @@ typedef struct SDataInserterHandle {
uint64_t useconds;
uint64_t cachedSize;
TdThreadMutex mutex;
tsem_t ready;
tsem_t ready;
} SDataInserterHandle;
typedef struct SSubmitRspParam {
@ -52,14 +52,14 @@ typedef struct SSubmitRspParam {
} SSubmitRspParam;
int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
SSubmitRspParam* pParam = (SSubmitRspParam*)param;
SSubmitRspParam* pParam = (SSubmitRspParam*)param;
SDataInserterHandle* pInserter = pParam->pInserter;
pInserter->submitRes.code = code;
if (code == TSDB_CODE_SUCCESS) {
pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp));
SDecoder coder = {0};
SDecoder coder = {0};
tDecoderInit(&coder, pMsg->pData, pMsg->len);
code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp);
if (code) {
@ -67,10 +67,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
pInserter->submitRes.code = code;
goto _return;
}
if (pInserter->submitRes.pRsp->nBlocks > 0) {
for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) {
SSubmitBlkRsp *blk = pInserter->submitRes.pRsp->pBlocks + i;
SSubmitBlkRsp* blk = pInserter->submitRes.pRsp->pBlocks + i;
if (TSDB_CODE_SUCCESS != blk->code) {
code = blk->code;
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
@ -79,9 +79,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
}
}
}
pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows;
qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows);
qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows,
pInserter->submitRes.affectedRows);
tFreeSSubmitRsp(pInserter->submitRes.pRsp);
}
@ -91,11 +92,10 @@ _return:
tsem_post(&pInserter->ready);
taosMemoryFree(pMsg->pData);
return TSDB_CODE_SUCCESS;
}
static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) {
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
@ -109,7 +109,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs
pParam->pInserter = pInserter;
pMsgSendInfo->param = pParam;
pMsgSendInfo->paramFreeFp = taosMemoryFree;
pMsgSendInfo->paramFreeFp = taosMemoryFree;
pMsgSendInfo->msgInfo.pData = pMsg;
pMsgSendInfo->msgInfo.len = ntohl(pMsg->length);
pMsgSendInfo->msgType = TDMT_VND_SUBMIT;
@ -119,17 +119,16 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs
return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo);
}
int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
const SArray* pBlocks = pInserter->pDataBlocks;
const STSchema* pTSchema = pInserter->pSchema;
int64_t uid = pInserter->pNode->tableId;
int64_t suid = pInserter->pNode->stableId;
int32_t vgId = pInserter->pNode->vgId;
bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols);
const SArray* pBlocks = pInserter->pDataBlocks;
const STSchema* pTSchema = pInserter->pSchema;
int64_t uid = pInserter->pNode->tableId;
int64_t suid = pInserter->pNode->stableId;
int32_t vgId = pInserter->pNode->vgId;
bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols);
SSubmitReq* ret = NULL;
int32_t sz = taosArrayGetSize(pBlocks);
int32_t sz = taosArrayGetSize(pBlocks);
// cal size
int32_t cap = sizeof(SSubmitReq);
@ -164,7 +163,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
int32_t rows = 0;
int32_t dataLen = 0;
STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk));
int64_t lastTs = TSKEY_MIN;
int64_t lastTs = TSKEY_MIN;
bool ignoreRow = false;
for (int32_t j = 0; j < pDataBlock->info.rows; j++) {
SRowBuilder rb = {0};
@ -176,13 +175,13 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
const STColumn* pColumn = &pTSchema->columns[k];
SColumnInfoData* pColData = NULL;
int16_t colIdx = k;
int16_t colIdx = k;
if (!fullCol) {
int16_t *slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId));
int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId));
if (NULL == slotId) {
continue;
}
colIdx = *slotId;
}
@ -192,13 +191,13 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
terrno = TSDB_CODE_APP_ERROR;
return TSDB_CODE_APP_ERROR;
}
if (colDataIsNull_s(pColData, j)) {
if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) {
ignoreRow = true;
break;
}
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k);
} else {
void* data = colDataGetData(pColData, j);
@ -213,7 +212,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k);
}
}
if(!fullCol) {
if (!fullCol) {
rb.hasNone = true;
}
tdSRowEnd(&rb);
@ -221,13 +220,13 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
if (ignoreRow) {
continue;
}
rows++;
int32_t rowLen = TD_ROW_LEN(rowData);
rowData = POINTER_SHIFT(rowData, rowLen);
dataLen += rowLen;
}
blkHead->dataLen = htonl(dataLen);
blkHead->numOfRows = htonl(rows);
@ -242,12 +241,11 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
return TSDB_CODE_SUCCESS;
}
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
taosArrayPush(pInserter->pDataBlocks, &pInput->pData);
SSubmitReq* pMsg = NULL;
int32_t code = dataBlockToSubmit(pInserter, &pMsg);
int32_t code = dataBlockToSubmit(pInserter, &pMsg);
if (code) {
return code;
}
@ -264,7 +262,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
}
*pContinue = true;
return TSDB_CODE_SUCCESS;
}
@ -279,10 +277,9 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) {
SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle;
*pLen = pDispatcher->submitRes.affectedRows;
qDebug("got total affectedRows %" PRId64 , *pLen);
qDebug("got total affectedRows %" PRId64, *pLen);
}
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
@ -301,14 +298,15 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) {
return TSDB_CODE_SUCCESS;
}
int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam) {
int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle,
void* pParam) {
SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle));
if (NULL == inserter) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
SQueryInserterNode* pInserterNode = (SQueryInserterNode *)pDataSink;
SQueryInserterNode* pInserterNode = (SQueryInserterNode*)pDataSink;
inserter->sink.fPut = putDataBlock;
inserter->sink.fEndPut = endPut;
inserter->sink.fGetLen = getDataLength;
@ -322,7 +320,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
inserter->queryEnd = false;
int64_t suid = 0;
int32_t code = tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
int32_t code =
tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
if (code) {
return code;
}
@ -339,15 +338,16 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT),
false, HASH_NO_LOCK);
SNode* pNode = NULL;
FOREACH(pNode, pInserterNode->pCols) {
SColumnNode* pCol = (SColumnNode*)pNode;
taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId));
}
tsem_init(&inserter->ready, 0, 0);
*pHandle = inserter;
return TSDB_CODE_SUCCESS;
}

16
source/libs/executor/src/dataSinkMgt.c
View File

@ -13,28 +13,27 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tarray.h"
#include "dataSinkMgt.h"
#include "dataSinkInt.h"
#include "planner.h"
#include "tarray.h"
static SDataSinkManager gDataSinkManager = {0};
SDataSinkStat gDataSinkStat = {0};
SDataSinkStat gDataSinkStat = {0};
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg) {
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) {
gDataSinkManager.cfg = *cfg;
taosThreadMutexInit(&gDataSinkManager.mutex, NULL);
return 0; // to avoid compiler eror
return 0; // to avoid compiler eror
}
int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat) {
int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) {
pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize);
return 0;
}
int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle, void* pParam) {
int32_t dsCreateDataSinker(const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam) {
switch ((int)nodeType(pDataSink)) {
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle);
@ -66,12 +65,11 @@ int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) {
return pHandleImpl->fGetData(pHandleImpl, pOutput);
}
int32_t dsGetCacheSize(DataSinkHandle handle, uint64_t *pSize) {
int32_t dsGetCacheSize(DataSinkHandle handle, uint64_t* pSize) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
return pHandleImpl->fGetCacheSize(pHandleImpl, pSize);
}
void dsScheduleProcess(void* ahandle, void* pItem) {
// todo
}

19
source/libs/executor/src/joinoperator.c
View File

@ -267,18 +267,19 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t
size_t rightNumJoin = taosArrayGetSize(rightRowLocations);
code = blockDataEnsureCapacity(pRes, *nRows + leftNumJoin * rightNumJoin);
if (code != TSDB_CODE_SUCCESS) {
qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), leftNumJoin, rightNumJoin);
qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo),
leftNumJoin, rightNumJoin);
}
if (code == TSDB_CODE_SUCCESS) {
for (int32_t i = 0; i < leftNumJoin; ++i) {
for (int32_t j = 0; j < rightNumJoin; ++j) {
SRowLocation *leftRow = taosArrayGet(leftRowLocations, i);
SRowLocation *rightRow = taosArrayGet(rightRowLocations, j);
mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock,
rightRow->pos);
++*nRows;
}
for (int32_t i = 0; i < leftNumJoin; ++i) {
for (int32_t j = 0; j < rightNumJoin; ++j) {
SRowLocation* leftRow = taosArrayGet(leftRowLocations, i);
SRowLocation* rightRow = taosArrayGet(rightRowLocations, j);
mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock,
rightRow->pos);
++*nRows;
}
}
}
for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) {

3
source/libs/executor/src/sortoperator.c
View File

@ -658,7 +658,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
blockDataDestroy(p);
qDebug("%s get sorted block, groupId:%0x"PRIx64" rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows);
qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}

2
source/libs/executor/src/tfill.c
View File

@ -561,7 +561,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];

4
source/libs/executor/src/timewindowoperator.c
View File

@ -1218,7 +1218,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
pBlock->info.rows, numOfOutput);
}
static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
if (OPTR_IS_OPENED(pOperator)) {
return TSDB_CODE_SUCCESS;
}
@ -2757,7 +2757,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
if (pStateNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr);
int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

94
source/libs/executor/src/tlinearhash.c
View File

@ -14,27 +14,27 @@
*/
#include "tlinearhash.h"
#include "tdef.h"
#include "taoserror.h"
#include "tdef.h"
#include "tpagedbuf.h"
#define LHASH_CAP_RATIO 0.85
// Always located in memory
typedef struct SLHashBucket {
SArray *pPageIdList;
int32_t size; // the number of element in this entry
SArray* pPageIdList;
int32_t size; // the number of element in this entry
} SLHashBucket;
struct SLHashObj {
SDiskbasedBuf *pBuf;
SDiskbasedBuf* pBuf;
_hash_fn_t hashFn;
SLHashBucket **pBucket; // entry list
SLHashBucket** pBucket; // entry list
int32_t tuplesPerPage;
int32_t numOfAlloc; // number of allocated bucket ptr slot
int32_t bits; // the number of bits used in hash
int32_t numOfBuckets; // the number of buckets
int64_t size; // the number of total items
int32_t numOfAlloc; // number of allocated bucket ptr slot
int32_t bits; // the number of bits used in hash
int32_t numOfBuckets; // the number of buckets
int64_t size; // the number of total items
};
/**
@ -44,19 +44,17 @@ struct SLHashObj {
* +-----------+-------+--------+
*/
typedef struct SLHashNode {
uint16_t keyLen;
uint16_t dataLen;
uint16_t keyLen;
uint16_t dataLen;
} SLHashNode;
#define GET_LHASH_NODE_KEY(_n) (((char*)(_n)) + sizeof(SLHashNode))
#define GET_LHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen)
#define GET_LHASH_NODE_LEN(_n) (sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen + ((SLHashNode*)(_n))->dataLen)
#define GET_LHASH_NODE_KEY(_n) (((char*)(_n)) + sizeof(SLHashNode))
#define GET_LHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen)
#define GET_LHASH_NODE_LEN(_n) (sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen + ((SLHashNode*)(_n))->dataLen)
static int32_t doAddNewBucket(SLHashObj* pHashObj);
static int32_t doGetBucketIdFromHashVal(int32_t hashv, int32_t bits) {
return hashv & ((1ul << (bits)) - 1);
}
static int32_t doGetBucketIdFromHashVal(int32_t hashv, int32_t bits) { return hashv & ((1ul << (bits)) - 1); }
static int32_t doGetAlternativeBucketId(int32_t bucketId, int32_t bits, int32_t numOfBuckets) {
int32_t v = bucketId - (1ul << (bits - 1));
@ -70,9 +68,9 @@ static int32_t doGetRelatedSplitBucketId(int32_t bucketId, int32_t bits) {
}
static void doCopyObject(char* p, const void* key, int32_t keyLen, const void* data, int32_t size) {
*(uint16_t*) p = keyLen;
*(uint16_t*)p = keyLen;
p += sizeof(uint16_t);
*(uint16_t*) p = size;
*(uint16_t*)p = size;
p += sizeof(uint16_t);
memcpy(p, key, keyLen);
@ -86,7 +84,7 @@ static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t
int32_t pageId = *(int32_t*)taosArrayGetLast(pBucket->pPageIdList);
SFilePage* pPage = getBufPage(pHashObj->pBuf, pageId);
ASSERT (pPage != NULL);
ASSERT(pPage != NULL);
// put to current buf page
size_t nodeSize = sizeof(SLHashNode) + keyLen + size;
@ -96,7 +94,7 @@ static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t
releaseBufPage(pHashObj->pBuf, pPage);
// allocate the overflow buffer page to hold this k/v.
int32_t newPageId = -1;
int32_t newPageId = -1;
SFilePage* pNewPage = getNewBufPage(pHashObj->pBuf, &newPageId);
if (pNewPage == NULL) {
return terrno;
@ -110,7 +108,7 @@ static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t
setBufPageDirty(pNewPage, true);
releaseBufPage(pHashObj->pBuf, pNewPage);
} else {
char* p = (char*) pPage + pPage->num;
char* p = (char*)pPage + pPage->num;
doCopyObject(p, key, keyLen, data, size);
pPage->num += nodeSize;
setBufPageDirty(pPage, true);
@ -118,7 +116,7 @@ static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t
}
pBucket->size += 1;
// printf("===> add to bucket:0x%x, num:%d, key:%d\n", index, pBucket->size, *(int*) key);
// printf("===> add to bucket:0x%x, num:%d, key:%d\n", index, pBucket->size, *(int*) key);
return TSDB_CODE_SUCCESS;
}
@ -127,7 +125,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket
ASSERT(pPage != NULL && pNode != NULL && pBucket->size >= 1);
int32_t len = GET_LHASH_NODE_LEN(pNode);
char* p = (char*) pNode + len;
char* p = (char*)pNode + len;
char* pEnd = (char*)pPage + pPage->num;
memmove(pNode, p, (pEnd - p));
@ -141,7 +139,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket
pBucket->size -= 1;
}
static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
static void doTrimBucketPages(SLHashObj* pHashObj, SLHashBucket* pBucket) {
size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList);
if (numOfPages <= 1) {
return;
@ -188,7 +186,7 @@ static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
}
nodeSize = GET_LHASH_NODE_LEN(pStart);
} else { // move to the front of pLast page
} else { // move to the front of pLast page
if (pStart != pLast->data) {
memmove(pLast->data, pStart, (((char*)pLast) + pLast->num - pStart));
setBufPageDirty(pLast, true);
@ -214,7 +212,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) {
}
memset(p + POINTER_BYTES * pHashObj->numOfBuckets, 0, newLen - pHashObj->numOfBuckets);
pHashObj->pBucket = (SLHashBucket**) p;
pHashObj->pBucket = (SLHashBucket**)p;
pHashObj->numOfAlloc = newLen;
}
@ -226,7 +224,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t pageId = -1;
int32_t pageId = -1;
SFilePage* p = getNewBufPage(pHashObj->pBuf, &pageId);
if (p == NULL) {
return terrno;
@ -239,7 +237,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) {
taosArrayPush(pBucket->pPageIdList, &pageId);
pHashObj->numOfBuckets += 1;
// printf("---------------add new bucket, id:0x%x, total:%d\n", pHashObj->numOfBuckets - 1, pHashObj->numOfBuckets);
// printf("---------------add new bucket, id:0x%x, total:%d\n", pHashObj->numOfBuckets - 1, pHashObj->numOfBuckets);
return TSDB_CODE_SUCCESS;
}
@ -266,14 +264,14 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_
setBufPageCompressOnDisk(pHashObj->pBuf, false);
/**
* The number of bits in the hash value, which is used to decide the exact bucket where the object should be located in.
* The initial value is 0.
* The number of bits in the hash value, which is used to decide the exact bucket where the object should be located
* in. The initial value is 0.
*/
pHashObj->bits = 0;
pHashObj->bits = 0;
pHashObj->hashFn = fn;
pHashObj->tuplesPerPage = numOfTuplePerPage;
pHashObj->numOfAlloc = 4; // initial allocated array list
pHashObj->numOfAlloc = 4; // initial allocated array list
pHashObj->pBucket = taosMemoryCalloc(pHashObj->numOfAlloc, POINTER_BYTES);
code = doAddNewBucket(pHashObj);
@ -289,7 +287,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_
void* tHashCleanup(SLHashObj* pHashObj) {
destroyDiskbasedBuf(pHashObj->pBuf);
for(int32_t i = 0; i < pHashObj->numOfBuckets; ++i) {
for (int32_t i = 0; i < pHashObj->numOfBuckets; ++i) {
taosArrayDestroy(pHashObj->pBucket[i]->pPageIdList);
taosMemoryFreeClear(pHashObj->pBucket[i]);
}
@ -299,7 +297,7 @@ void* tHashCleanup(SLHashObj* pHashObj) {
return NULL;
}
int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data, size_t size) {
ASSERT(pHashObj != NULL && key != NULL);
if (pHashObj->bits == 0) {
@ -311,12 +309,12 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
if (v >= pHashObj->numOfBuckets) {
int32_t newBucketId = doGetAlternativeBucketId(v, pHashObj->bits, pHashObj->numOfBuckets);
// printf("bucketId: 0x%x not exists, put it into 0x%x instead\n", v, newBucketId);
// printf("bucketId: 0x%x not exists, put it into 0x%x instead\n", v, newBucketId);
v = newBucketId;
}
SLHashBucket* pBucket = pHashObj->pBucket[v];
int32_t code = doAddToBucket(pHashObj, pBucket, v, key, keyLen, data, size);
int32_t code = doAddToBucket(pHashObj, pBucket, v, key, keyLen, data, size);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -335,7 +333,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
int32_t numOfBits = ceil(log(pHashObj->numOfBuckets) / log(2));
if (numOfBits > pHashObj->bits) {
// printf("extend the bits from %d to %d, new bucket:%d\n", pHashObj->bits, numOfBits, newBucketId);
// printf("extend the bits from %d to %d, new bucket:%d\n", pHashObj->bits, numOfBits, newBucketId);
ASSERT(numOfBits == pHashObj->bits + 1);
pHashObj->bits = numOfBits;
}
@ -344,31 +342,31 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
// load all data in this bucket and check if the data needs to relocated into the new bucket
SLHashBucket* pBucket = pHashObj->pBucket[splitBucketId];
// printf("split %d items' bucket:0x%x to new bucket:0x%x\n", pBucket->size, splitBucketId, newBucketId);
// printf("split %d items' bucket:0x%x to new bucket:0x%x\n", pBucket->size, splitBucketId, newBucketId);
for (int32_t i = 0; i < taosArrayGetSize(pBucket->pPageIdList); ++i) {
int32_t pageId = *(int32_t*)taosArrayGet(pBucket->pPageIdList, i);
SFilePage* p = getBufPage(pHashObj->pBuf, pageId);
char* pStart = p->data;
while (pStart - ((char*) p) < p->num) {
while (pStart - ((char*)p) < p->num) {
SLHashNode* pNode = (SLHashNode*)pStart;
ASSERT(pNode->keyLen > 0 && pNode->dataLen >= 0);
char* k = GET_LHASH_NODE_KEY(pNode);
char* k = GET_LHASH_NODE_KEY(pNode);
int32_t hashv = pHashObj->hashFn(k, pNode->keyLen);
int32_t v1 = doGetBucketIdFromHashVal(hashv, pHashObj->bits);
if (v1 != splitBucketId) { // place it into the new bucket
ASSERT(v1 == newBucketId);
// printf("move key:%d to 0x%x bucket, remain items:%d\n", *(int32_t*)k, v1, pBucket->size - 1);
// printf("move key:%d to 0x%x bucket, remain items:%d\n", *(int32_t*)k, v1, pBucket->size - 1);
SLHashBucket* pNewBucket = pHashObj->pBucket[newBucketId];
doAddToBucket(pHashObj, pNewBucket, newBucketId, (void*)GET_LHASH_NODE_KEY(pNode), pNode->keyLen,
GET_LHASH_NODE_KEY(pNode), pNode->dataLen);
doRemoveFromBucket(p, pNode, pBucket);
} else {
// printf("check key:%d, located into: %d, skip it\n", *(int*) k, v1);
// printf("check key:%d, located into: %d, skip it\n", *(int*) k, v1);
int32_t nodeSize = GET_LHASH_NODE_LEN(pStart);
pStart += nodeSize;
@ -383,7 +381,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
return TSDB_CODE_SUCCESS;
}
char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen) {
char* tHashGet(SLHashObj* pHashObj, const void* key, size_t keyLen) {
ASSERT(pHashObj != NULL && key != NULL && keyLen > 0);
int32_t hashv = pHashObj->hashFn(key, keyLen);
@ -393,7 +391,7 @@ char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen) {
}
SLHashBucket* pBucket = pHashObj->pBucket[bucketId];
int32_t num = taosArrayGetSize(pBucket->pPageIdList);
int32_t num = taosArrayGetSize(pBucket->pPageIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t pageId = *(int32_t*)taosArrayGet(pBucket->pPageIdList, i);
@ -418,7 +416,7 @@ char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen) {
return NULL;
}
int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen) {
int32_t tHashRemove(SLHashObj* pHashObj, const void* key, size_t keyLen) {
// todo
return TSDB_CODE_SUCCESS;
}
@ -431,8 +429,8 @@ void tHashPrint(const SLHashObj* pHashObj, int32_t type) {
if (type == LINEAR_HASH_DATA) {
for (int32_t i = 0; i < pHashObj->numOfBuckets; ++i) {
// printf("bucket: 0x%x, obj:%d, page:%d\n", i, pHashObj->pBucket[i]->size,
// (int)taosArrayGetSize(pHashObj->pBucket[i]->pPageIdList));
// printf("bucket: 0x%x, obj:%d, page:%d\n", i, pHashObj->pBucket[i]->size,
// (int)taosArrayGetSize(pHashObj->pBucket[i]->pPageIdList));
}
} else {
dBufPrintStatis(pHashObj->pBuf);

166
source/libs/executor/src/tsort.c
View File

@ -13,16 +13,16 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tcommon.h"
#include "query.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tlosertree.h"
#include "tpagedbuf.h"
#include "tsort.h"
#include "tutil.h"
#include "tcompare.h"
struct STupleHandle {
SSDataBlock* pBlock;
@ -30,37 +30,37 @@ struct STupleHandle {
};
struct SSortHandle {
int32_t type;
int32_t pageSize;
int32_t numOfPages;
SDiskbasedBuf *pBuf;
int32_t type;
int32_t pageSize;
int32_t numOfPages;
SDiskbasedBuf* pBuf;
SArray *pSortInfo;
SArray *pOrderedSource;
SArray* pSortInfo;
SArray* pOrderedSource;
int32_t loops;
uint64_t sortElapsed;
int64_t startTs;
uint64_t totalElapsed;
int32_t loops;
uint64_t sortElapsed;
int64_t startTs;
uint64_t totalElapsed;
int32_t sourceId;
SSDataBlock *pDataBlock;
SSDataBlock* pDataBlock;
SMsortComparParam cmpParam;
int32_t numOfCompletedSources;
bool opened;
const char *idStr;
const char* idStr;
bool inMemSort;
bool needAdjust;
STupleHandle tupleHandle;
void *param;
void* param;
void (*beforeFp)(SSDataBlock* pBlock, void* param);
_sort_fetch_block_fn_t fetchfp;
_sort_merge_compar_fn_t comparFn;
SMultiwayMergeTreeInfo *pMergeTree;
SMultiwayMergeTreeInfo* pMergeTree;
};
static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param);
static int32_t msortComparFn(const void* pLeft, const void* pRight, void* param);
SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) {
return createOneDataBlock(pSortHandle->pDataBlock, false);
@ -71,20 +71,21 @@ SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) {
* @param type
* @return
*/
SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr) {
SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages,
SSDataBlock* pBlock, const char* idstr) {
SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle));
pSortHandle->type = type;
pSortHandle->pageSize = pageSize;
pSortHandle->type = type;
pSortHandle->pageSize = pageSize;
pSortHandle->numOfPages = numOfPages;
pSortHandle->pSortInfo = pSortInfo;
pSortHandle->loops = 0;
pSortHandle->pSortInfo = pSortInfo;
pSortHandle->loops = 0;
if (pBlock != NULL) {
pSortHandle->pDataBlock = createOneDataBlock(pBlock, false);
}
pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
pSortHandle->cmpParam.orderInfo = pSortInfo;
pSortHandle->cmpParam.cmpGroupId = false;
@ -98,8 +99,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
}
static int32_t sortComparCleanup(SMsortComparParam* cmpParam) {
for(int32_t i = 0; i < cmpParam->numOfSources; ++i) {
SSortSource* pSource = cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE
for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
SSortSource* pSource =
cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE
blockDataDestroy(pSource->src.pBlock);
taosMemoryFreeClear(pSource);
}
@ -121,7 +123,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
destroyDiskbasedBuf(pSortHandle->pBuf);
taosMemoryFreeClear(pSortHandle->idStr);
blockDataDestroy(pSortHandle->pDataBlock);
for (size_t i = 0; i < taosArrayGetSize(pSortHandle->pOrderedSource); i++){
for (size_t i = 0; i < taosArrayGetSize(pSortHandle->pOrderedSource); i++) {
SSortSource** pSource = taosArrayGet(pSortHandle->pOrderedSource, i);
taosMemoryFreeClear(*pSource);
}
@ -134,7 +136,8 @@ int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) {
return TSDB_CODE_SUCCESS;
}
static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId, SArray* pPageIdList) {
static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSources, SSDataBlock* pBlock,
int32_t* sourceId, SArray* pPageIdList) {
SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource));
if (pSource == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
@ -149,7 +152,8 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
// The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize;
int32_t numOfRows =
(getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock))) / rowSize;
ASSERT(numOfRows > 0);
return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
}
@ -163,7 +167,8 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
qError("Add to buf failed since %s", terrstr(terrno));
return terrno;
}
int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", tsTempDir);
int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize,
"doAddToBuf", tsTempDir);
dBufSetPrintInfo(pHandle->pBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -171,7 +176,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
}
SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t));
while(start < pDataBlock->info.rows) {
while (start < pDataBlock->info.rows) {
int32_t stop = 0;
blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pHandle->pageSize);
SSDataBlock* p = blockDataExtractBlock(pDataBlock, start, stop - start + 1);
@ -180,7 +185,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
}
int32_t pageId = -1;
void* pPage = getNewBufPage(pHandle->pBuf, &pageId);
void* pPage = getNewBufPage(pHandle->pBuf, &pageId);
if (pPage == NULL) {
blockDataDestroy(p);
return terrno;
@ -188,7 +193,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
taosArrayPush(pPageIdList, &pageId);
int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
assert(size <= getBufPageSize(pHandle->pBuf));
blockDataToBuf(pPage, p);
@ -211,8 +216,9 @@ static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) {
++pHandle->numOfCompletedSources;
}
static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) {
cmpParam->pSources = taosArrayGet(pSources, startIndex);
static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
SSortHandle* pHandle) {
cmpParam->pSources = taosArrayGet(pSources, startIndex);
cmpParam->numOfSources = (endIndex - startIndex + 1);
int32_t code = 0;
@ -246,7 +252,8 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
qError("Sort compare init failed since %s", terrstr(terrno));
return code;
}
code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", tsTempDir);
code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize,
"sortComparInit", tsTempDir);
dBufSetPrintInfo(pHandle->pBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -267,12 +274,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
return code;
}
static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) {
static void appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* pSource, int32_t* rowIndex) {
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i);
bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL);
bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL);
if (isNull) {
colDataAppend(pColInfo, pBlock->info.rows, NULL, true);
@ -286,7 +293,8 @@ static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSou
*rowIndex += 1;
}
static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeTreeInfo *pTree, SSortHandle *pHandle, int32_t* numOfCompleted) {
static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeTreeInfo* pTree, SSortHandle* pHandle,
int32_t* numOfCompleted) {
/*
* load a new SDataBlock into memory of a given intermediate data-set source,
* since it's last record in buffer has been chosen to be processed, as the winner of loser-tree
@ -295,7 +303,7 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT
pSource->src.rowIndex = 0;
if (pHandle->type == SORT_SINGLESOURCE_SORT) {
pSource->pageIndex ++;
pSource->pageIndex++;
if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) {
(*numOfCompleted) += 1;
pSource->src.rowIndex = -1;
@ -304,8 +312,8 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT
} else {
int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage);
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -344,14 +352,14 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT
static SSDataBlock* getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) {
blockDataCleanup(pHandle->pDataBlock);
while(1) {
while (1) {
if (cmpParam->numOfSources == pHandle->numOfCompletedSources) {
break;
}
int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree);
SSortSource *pSource = (*cmpParam).pSources[index];
SSortSource* pSource = (*cmpParam).pSources[index];
appendOneRowToDataBlock(pHandle->pDataBlock, pSource->src.pBlock, &pSource->src.rowIndex);
int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources);
@ -365,18 +373,18 @@ static SSDataBlock* getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparPa
}
}
return (pHandle->pDataBlock->info.rows > 0)? pHandle->pDataBlock:NULL;
return (pHandle->pDataBlock->info.rows > 0) ? pHandle->pDataBlock : NULL;
}
int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
int32_t pLeftIdx = *(int32_t *)pLeft;
int32_t pRightIdx = *(int32_t *)pRight;
int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) {
int32_t pLeftIdx = *(int32_t*)pLeft;
int32_t pRightIdx = *(int32_t*)pRight;
SMsortComparParam *pParam = (SMsortComparParam *)param;
SMsortComparParam* pParam = (SMsortComparParam*)param;
SArray *pInfo = pParam->orderInfo;
SArray* pInfo = pParam->orderInfo;
SSortSource* pLeftSource = pParam->pSources[pLeftIdx];
SSortSource* pLeftSource = pParam->pSources[pLeftIdx];
SSortSource* pRightSource = pParam->pSources[pRightIdx];
// this input is exhausted, set the special value to denote this
@ -397,42 +405,44 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
}
}
for(int32_t i = 0; i < pInfo->size; ++i) {
for (int32_t i = 0; i < pInfo->size; ++i) {
SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
bool leftNull = false;
bool leftNull = false;
if (pLeftColInfoData->hasNull) {
if (pLeftBlock->pBlockAgg == NULL) {
leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex);
} else {
leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[i]);
leftNull =
colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[i]);
}
}
SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
bool rightNull = false;
bool rightNull = false;
if (pRightColInfoData->hasNull) {
if (pLeftBlock->pBlockAgg == NULL) {
rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex);
} else {
rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg[i]);
rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex,
pRightBlock->pBlockAgg[i]);
}
}
if (leftNull && rightNull) {
continue; // continue to next slot
continue; // continue to next slot
}
if (rightNull) {
return pOrder->nullFirst? 1:-1;
return pOrder->nullFirst ? 1 : -1;
}
if (leftNull) {
return pOrder->nullFirst? -1:1;
return pOrder->nullFirst ? -1 : 1;
}
void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex);
__compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order);
@ -464,8 +474,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
", sort elapsed:%" PRId64 ", total elapsed:%" PRId64,
pHandle->idStr, (int32_t)(sortPass + 1), s, pHandle->sortElapsed, pHandle->totalElapsed);
} else {
qDebug("%s ordered source:%"PRIzu", available buf:%d, no need internal sort", pHandle->idStr, numOfSources,
pHandle->numOfPages);
qDebug("%s ordered source:%" PRIzu ", available buf:%d, no need internal sort", pHandle->idStr, numOfSources,
pHandle->numOfPages);
}
int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize);
@ -475,7 +485,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
pHandle->loops = sortPass + 2;
size_t numOfSorted = taosArrayGetSize(pHandle->pOrderedSource);
for(int32_t t = 0; t < sortPass; ++t) {
for (int32_t t = 0; t < sortPass; ++t) {
int64_t st = taosGetTimestampUs();
SArray* pResList = taosArrayInit(4, POINTER_BYTES);
@ -484,7 +494,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources;
// Only *numOfInputSources* can be loaded into buffer to perform the external sort.
for(int32_t i = 0; i < sortGroup; ++i) {
for (int32_t i = 0; i < sortGroup; ++i) {
pHandle->sourceId += 1;
int32_t end = (i + 1) * numOfInputSources - 1;
@ -499,7 +509,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
return code;
}
code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn);
code =
tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -512,14 +523,15 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
}
int32_t pageId = -1;
void* pPage = getNewBufPage(pHandle->pBuf, &pageId);
void* pPage = getNewBufPage(pHandle->pBuf, &pageId);
if (pPage == NULL) {
return terrno;
}
taosArrayPush(pPageIdList, &pageId);
int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t);
int32_t size =
blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t);
assert(size <= getBufPageSize(pHandle->pBuf));
blockDataToBuf(pPage, pDataBlock);
@ -551,8 +563,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
pHandle->totalElapsed += el;
SDiskbasedBufStatis statis = getDBufStatis(pHandle->pBuf);
qDebug("%s %d round mergesort, elapsed:%"PRId64" readDisk:%.2f Kb, flushDisk:%.2f Kb", pHandle->idStr, t + 1, el, statis.loadBytes/1024.0,
statis.flushBytes/1024.0);
qDebug("%s %d round mergesort, elapsed:%" PRId64 " readDisk:%.2f Kb, flushDisk:%.2f Kb", pHandle->idStr, t + 1, el,
statis.loadBytes / 1024.0, statis.flushBytes / 1024.0);
if (pHandle->type == SORT_MULTISOURCE_MERGE) {
pHandle->type = SORT_SINGLESOURCE_SORT;
@ -645,7 +657,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
pHandle->inMemSort = true;
pHandle->loops = 1;
pHandle->tupleHandle.rowIndex = -1;
pHandle->tupleHandle.rowIndex = -1;
pHandle->tupleHandle.pBlock = pHandle->pDataBlock;
return 0;
} else {
@ -701,7 +713,8 @@ int32_t tsortClose(SSortHandle* pHandle) {
return TSDB_CODE_SUCCESS;
}
int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*), void* param) {
int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetchFp, void (*fp)(SSDataBlock*, void*),
void* param) {
pHandle->fetchfp = fetchFp;
pHandle->beforeFp = fp;
pHandle->param = param;
@ -734,8 +747,8 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) {
return &pHandle->tupleHandle;
}
int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree);
SSortSource *pSource = pHandle->cmpParam.pSources[index];
int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree);
SSortSource* pSource = pHandle->cmpParam.pSources[index];
if (pHandle->needAdjust) {
int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources);
@ -778,16 +791,14 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
}
}
uint64_t tsortGetGroupId(STupleHandle* pVHandle) {
return pVHandle->pBlock->info.groupId;
}
uint64_t tsortGetGroupId(STupleHandle* pVHandle) { return pVHandle->pBlock->info.groupId; }
SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
SSortExecInfo info = {0};
info.sortBuffer = pHandle->pageSize * pHandle->numOfPages;
info.sortMethod = pHandle->inMemSort? SORT_QSORT_T:SORT_SPILLED_MERGE_SORT_T;
info.loops = pHandle->loops;
info.sortMethod = pHandle->inMemSort ? SORT_QSORT_T : SORT_SPILLED_MERGE_SORT_T;
info.loops = pHandle->loops;
if (pHandle->pBuf != NULL) {
SDiskbasedBufStatis st = getDBufStatis(pHandle->pBuf);
@ -797,4 +808,3 @@ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
return info;
}

21
source/libs/executor/test/lhashTests.cpp
View File

@ -32,41 +32,42 @@ TEST(testCase, linear_hash_Tests) {
int64_t st = taosGetTimestampUs();
SLHashObj* pHashObj = tHashInit(4098*4*2, 512, fn, 40);
for(int32_t i = 0; i < 1000000; ++i) {
SLHashObj* pHashObj = tHashInit(4098 * 4 * 2, 512, fn, 40);
for (int32_t i = 0; i < 1000000; ++i) {
int32_t code = tHashPut(pHashObj, &i, sizeof(i), &i, sizeof(i));
assert(code == 0);
}
// tHashPrint(pHashObj, LINEAR_HASH_STATIS);
// tHashPrint(pHashObj, LINEAR_HASH_STATIS);
int64_t et = taosGetTimestampUs();
for(int32_t i = 0; i < 1000000; ++i) {
for (int32_t i = 0; i < 1000000; ++i) {
if (i == 950000) {
printf("kf\n");
}
char* v = tHashGet(pHashObj, &i, sizeof(i));
if (v != NULL) {
// printf("find value: %d, key:%d\n", *(int32_t*) v, i);
// printf("find value: %d, key:%d\n", *(int32_t*) v, i);
} else {
// printf("failed to found key:%d in hash\n", i);
// printf("failed to found key:%d in hash\n", i);
}
}
// tHashPrint(pHashObj, LINEAR_HASH_STATIS);
// tHashPrint(pHashObj, LINEAR_HASH_STATIS);
tHashCleanup(pHashObj);
int64_t et1 = taosGetTimestampUs();
SHashObj* pHashObj1 = taosHashInit(1000, fn, false, HASH_NO_LOCK);
for(int32_t i = 0; i < 1000000; ++i) {
for (int32_t i = 0; i < 1000000; ++i) {
taosHashPut(pHashObj1, &i, sizeof(i), &i, sizeof(i));
}
for(int32_t i = 0; i < 1000000; ++i) {
for (int32_t i = 0; i < 1000000; ++i) {
void* v = taosHashGet(pHashObj1, &i, sizeof(i));
}
taosHashCleanup(pHashObj1);
int64_t et2 = taosGetTimestampUs();
printf("linear hash time:%.2f ms, buildHash:%.2f ms, hash:%.2f\n", (et1-st)/1000.0, (et-st)/1000.0, (et2-et1)/1000.0);
printf("linear hash time:%.2f ms, buildHash:%.2f ms, hash:%.2f\n", (et1 - st) / 1000.0, (et - st) / 1000.0,
(et2 - et1) / 1000.0);
}

80
source/libs/executor/test/sortTests.cpp
View File

@ -25,14 +25,14 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "executorimpl.h"
#include "executor.h"
#include "executorimpl.h"
#include "taos.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tdef.h"
#include "trpc.h"
#include "tvariant.h"
#include "tcompare.h"
namespace {
typedef struct {
@ -44,8 +44,7 @@ typedef struct {
int16_t VARCOUNT = 16;
float rand_f2()
{
float rand_f2() {
unsigned r = taosRand();
r &= 0x007fffff;
r |= 0x40800000;
@ -53,10 +52,10 @@ float rand_f2()
}
static const int32_t TEST_NUMBER = 1;
#define bigendian() ((*(char *)&TEST_NUMBER) == 0)
#define bigendian() ((*(char*)&TEST_NUMBER) == 0)
SSDataBlock* getSingleColDummyBlock(void* param) {
_info* pInfo = (_info*) param;
_info* pInfo = (_info*)param;
if (--pInfo->count < 0) {
return NULL;
}
@ -65,11 +64,11 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
SColumnInfoData colInfo = {0};
colInfo.info.type = pInfo->type;
if (pInfo->type == TSDB_DATA_TYPE_NCHAR){
if (pInfo->type == TSDB_DATA_TYPE_NCHAR) {
colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE;
} else if(pInfo->type == TSDB_DATA_TYPE_BINARY) {
} else if (pInfo->type == TSDB_DATA_TYPE_BINARY) {
colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE;
} else{
} else {
colInfo.info.bytes = tDataTypes[pInfo->type].bytes;
}
colInfo.info.colId = 1;
@ -80,39 +79,39 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
for (int32_t i = 0; i < pInfo->pageRows; ++i) {
SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
if (pInfo->type == TSDB_DATA_TYPE_NCHAR){
if (pInfo->type == TSDB_DATA_TYPE_NCHAR) {
int32_t size = taosRand() % VARCOUNT;
char str[128] = {0};
char strOri[128] = {0};
char str[128] = {0};
char strOri[128] = {0};
taosRandStr(strOri, size);
int32_t len = 0;
bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len);
if (!ret){
bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len);
if (!ret) {
printf("error\n");
return NULL;
}
varDataSetLen(str, len);
colDataAppend(pColInfo, i, reinterpret_cast<const char*>(str), false);
pBlock->info.hasVarCol = true;
printf("nchar: %s\n",strOri);
} else if(pInfo->type == TSDB_DATA_TYPE_BINARY){
printf("nchar: %s\n", strOri);
} else if (pInfo->type == TSDB_DATA_TYPE_BINARY) {
int32_t size = taosRand() % VARCOUNT;
char str[64] = {0};
char str[64] = {0};
taosRandStr(varDataVal(str), size);
varDataSetLen(str, size);
colDataAppend(pColInfo, i, reinterpret_cast<const char*>(str), false);
pBlock->info.hasVarCol = true;
printf("binary: %s\n", varDataVal(str));
} else if(pInfo->type == TSDB_DATA_TYPE_DOUBLE || pInfo->type == TSDB_DATA_TYPE_FLOAT) {
} else if (pInfo->type == TSDB_DATA_TYPE_DOUBLE || pInfo->type == TSDB_DATA_TYPE_FLOAT) {
double v = rand_f2();
colDataAppend(pColInfo, i, reinterpret_cast<const char*>(&v), false);
printf("float: %f\n", v);
} else{
} else {
int64_t v = ++pInfo->startVal;
char *result = static_cast<char*>(taosMemoryCalloc(tDataTypes[pInfo->type].bytes, 1));
if (!bigendian()){
char* result = static_cast<char*>(taosMemoryCalloc(tDataTypes[pInfo->type].bytes, 1));
if (!bigendian()) {
memcpy(result, &v, tDataTypes[pInfo->type].bytes);
}else{
} else {
memcpy(result, (char*)(&v) + sizeof(int64_t) - tDataTypes[pInfo->type].bytes, tDataTypes[pInfo->type].bytes);
}
@ -126,17 +125,16 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
return pBlock;
}
int32_t docomp(const void* p1, const void* p2, void* param) {
int32_t pLeftIdx = *(int32_t *)p1;
int32_t pRightIdx = *(int32_t *)p2;
int32_t pLeftIdx = *(int32_t*)p1;
int32_t pRightIdx = *(int32_t*)p2;
SMsortComparParam *pParam = (SMsortComparParam *)param;
SSortSource** px = reinterpret_cast<SSortSource**>(pParam->pSources);
SMsortComparParam* pParam = (SMsortComparParam*)param;
SSortSource** px = reinterpret_cast<SSortSource**>(pParam->pSources);
SArray *pInfo = pParam->orderInfo;
SArray* pInfo = pParam->orderInfo;
SSortSource* pLeftSource = px[pLeftIdx];
SSortSource* pLeftSource = px[pLeftIdx];
SSortSource* pRightSource = px[pRightIdx];
// this input is exhausted, set the special value to denote this
@ -151,36 +149,38 @@ int32_t docomp(const void* p1, const void* p2, void* param) {
SSDataBlock* pLeftBlock = pLeftSource->src.pBlock;
SSDataBlock* pRightBlock = pRightSource->src.pBlock;
for(int32_t i = 0; i < pInfo->size; ++i) {
for (int32_t i = 0; i < pInfo->size; ++i) {
SBlockOrderInfo* pOrder = (SBlockOrderInfo*)TARRAY_GET_ELEM(pInfo, i);
SColumnInfoData* pLeftColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
bool leftNull = false;
bool leftNull = false;
if (pLeftColInfoData->hasNull) {
leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg[pOrder->slotId]);
leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex,
pLeftBlock->pBlockAgg[pOrder->slotId]);
}
SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
bool rightNull = false;
SColumnInfoData* pRightColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
bool rightNull = false;
if (pRightColInfoData->hasNull) {
rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg[pOrder->slotId]);
rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex,
pRightBlock->pBlockAgg[pOrder->slotId]);
}
if (leftNull && rightNull) {
continue; // continue to next slot
continue; // continue to next slot
}
if (rightNull) {
return pOrder->nullFirst? 1:-1;
return pOrder->nullFirst ? 1 : -1;
}
if (leftNull) {
return pOrder->nullFirst? -1:1;
return pOrder->nullFirst ? -1 : 1;
}
void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex);
void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex);
__compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order);
int ret = fn(left1, right1);

10
source/libs/executor/test/tSimpleHashTests.cpp
View File

@ -31,8 +31,7 @@
// }
TEST(testCase, tSimpleHashTest_intKey) {
SSHashObj *pHashObj =
tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
SSHashObj *pHashObj = tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
assert(pHashObj != nullptr);
@ -76,10 +75,8 @@ TEST(testCase, tSimpleHashTest_intKey) {
tSimpleHashCleanup(pHashObj);
}
TEST(testCase, tSimpleHashTest_binaryKey) {
SSHashObj *pHashObj =
tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
SSHashObj *pHashObj = tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
assert(pHashObj != nullptr);
@ -93,7 +90,7 @@ TEST(testCase, tSimpleHashTest_binaryKey) {
size_t keyLen = sizeof(SCombineKey);
size_t dataLen = sizeof(int64_t);
int64_t originDataSum = 0;
int64_t originDataSum = 0;
SCombineKey combineKey = {0};
for (int64_t i = 1; i <= 100; ++i) {
combineKey.suid = i;
@ -140,5 +137,4 @@ TEST(testCase, tSimpleHashTest_binaryKey) {
tSimpleHashCleanup(pHashObj);
}
#pragma GCC diagnostic pop

30
source/libs/tfs/inc/tfsInt.h
View File

@ -26,12 +26,14 @@
#include "tlog.h"
// For debug purpose
// clang-format off
#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", DEBUG_DEBUG, fsDebugFlag, __VA_ARGS__); }}
#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", DEBUG_TRACE, fsDebugFlag, __VA_ARGS__); }}
// clang-format on
typedef struct {
int32_t level;
@ -42,12 +44,12 @@ typedef struct {
typedef struct {
TdThreadSpinlock lock;
int32_t level;
int32_t nextid; // next disk id to allocate
int32_t ndisk; // # of disks mounted to this tier
int32_t nAvailDisks; // # of Available disks
STfsDisk *disks[TFS_MAX_DISKS_PER_TIER];
SDiskSize size;
int32_t level;
int32_t nextid; // next disk id to allocate
int32_t ndisk; // # of disks mounted to this tier
int32_t nAvailDisks; // # of Available disks
STfsDisk *disks[TFS_MAX_DISKS_PER_TIER];
SDiskSize size;
} STfsTier;
typedef struct {
@ -65,10 +67,10 @@ typedef struct STfsDir {
typedef struct STfs {
TdThreadSpinlock lock;
SDiskSize size;
int32_t nlevel;
STfsTier tiers[TFS_MAX_TIERS];
SHashObj *hash; // name to did map
SDiskSize size;
int32_t nlevel;
STfsTier tiers[TFS_MAX_TIERS];
SHashObj *hash; // name to did map
} STfs;
STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *dir);
@ -82,15 +84,15 @@ void tfsUpdateTierSize(STfsTier *pTier);
int32_t tfsAllocDiskOnTier(STfsTier *pTier);
void tfsPosNextId(STfsTier *pTier);
#define tfsLockTier(pTier) taosThreadSpinLock(&(pTier)->lock)
#define tfsLockTier(pTier) taosThreadSpinLock(&(pTier)->lock)
#define tfsUnLockTier(pTier) taosThreadSpinUnlock(&(pTier)->lock)
#define tfsLock(pTfs) taosThreadSpinLock(&(pTfs)->lock)
#define tfsLock(pTfs) taosThreadSpinLock(&(pTfs)->lock)
#define tfsUnLock(pTfs) taosThreadSpinUnlock(&(pTfs)->lock)
#define TFS_TIER_AT(pTfs, level) (&(pTfs)->tiers[level])
#define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id])
#define TFS_PRIMARY_DISK(pTfs) ((pTfs)->tiers[0].disks[0])
#define TFS_DISK_AT(pTfs, did) ((pTfs)->tiers[(did).level].disks[(did).id])
#define TFS_PRIMARY_DISK(pTfs) ((pTfs)->tiers[0].disks[0])
#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)

4
source/libs/tfs/src/tfs.c
View File

@ -213,7 +213,7 @@ void tfsDirname(const STfsFile *pFile, char *dest) {
void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) {
STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId);
snprintf(aname, TSDB_FILENAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname);
}
@ -285,7 +285,7 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) {
int32_t tfsRmdir(STfs *pTfs, const char *rname) {
ASSERT(rname[0] != 0);
char aname[TMPNAME_LEN] = "\0";
for (int32_t level = 0; level < pTfs->nlevel; level++) {

4
source/libs/tfs/test/tfsTest.cpp
View File

@ -16,7 +16,7 @@
class TfsTest : public ::testing::Test {
protected:
#ifdef _TD_DARWIN_64
#ifdef _TD_DARWIN_64
static void SetUpTestSuite() { root = "/private" TD_TMP_DIR_PATH "tfsTest"; }
#else
static void SetUpTestSuite() { root = TD_TMP_DIR_PATH "tfsTest"; }
@ -303,7 +303,7 @@ TEST_F(TfsTest, 04_File) {
TEST_F(TfsTest, 05_MultiDisk) {
int32_t code = 0;
#ifdef _TD_DARWIN_64
#ifdef _TD_DARWIN_64
const char *root00 = "/private" TD_TMP_DIR_PATH "tfsTest00";
const char *root01 = "/private" TD_TMP_DIR_PATH "tfsTest01";
const char *root10 = "/private" TD_TMP_DIR_PATH "tfsTest10";

20
tools/scripts/codeFormat.sh
View File

@ -12,7 +12,25 @@ FORMAT_DIR_LIST=(
"source/os"
"source/util"
"source/common"
# "source/libs"
"source/libs/cache"
"source/libs/catalog"
"source/libs/command"
"source/libs/executor"
# "source/libs/function"
# "source/libs/index"
# "source/libs/monitor"
# "source/libs/nodes"
# "source/libs/parser"
# "source/libs/planner"
# "source/libs/qcom"
# "source/libs/qworker"
# "source/libs/scalar"
# "source/libs/stream"
# "source/libs/sync"
"source/libs/tdb"
"source/libs/tfs"
# "source/libs/transport"
"source/libs/wal"
# "source/client"
"source/dnode"
)