xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-11818]merge 3.0

This commit is contained in:
Haojun Liao 2021-12-16 15:59:15 +08:00
parent 275234aa2f e35bac2d28
commit f17fa8cc23
152 changed files with 5615 additions and 7734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
include/common/taosmsg.h
View File

@ -77,7 +77,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STB_VGROUP, "stb-vgroup" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" )
@ -168,8 +168,8 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_SCORES,
TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_GRANTS,
TSDB_MGMT_TABLE_VNODES, TSDB_MGMT_TABLE_VNODES,
TSDB_MGMT_TABLE_STREAMTABLES,
TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_CLUSTER,
TSDB_MGMT_TABLE_STREAMTABLES,
TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_TP,
TSDB_MGMT_TABLE_FUNCTION, TSDB_MGMT_TABLE_FUNCTION,
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
@ -214,6 +214,17 @@ typedef enum _mgmt_table {
extern char *taosMsg[]; extern char *taosMsg[];
typedef struct SBuildTableMetaInput {
int32_t vgId;
char *tableFullName;
} SBuildTableMetaInput;
typedef struct SBuildUseDBInput {
char db[TSDB_TABLE_FNAME_LEN];
int32_t vgVersion;
} SBuildUseDBInput;
#pragma pack(push, 1) #pragma pack(push, 1)
// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
@ -384,19 +395,22 @@ typedef struct {
int32_t maxDbs; int32_t maxDbs;
int32_t maxTimeSeries; int32_t maxTimeSeries;
int32_t maxStreams; int32_t maxStreams;
int64_t maxStorage; // In unit of GB
int32_t accessState; // Configured only by command int32_t accessState; // Configured only by command
int64_t maxStorage; // In unit of GB
int32_t reserve[8];
} SCreateAcctMsg, SAlterAcctMsg; } SCreateAcctMsg, SAlterAcctMsg;
typedef struct { typedef struct {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
int32_t reserve[8];
} SDropUserMsg, SDropAcctMsg; } SDropUserMsg, SDropAcctMsg;
typedef struct { typedef struct {
int8_t type; int8_t type;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_PASSWORD_LEN];
int8_t superUser; // denote if it is a super user or not int8_t superUser; // denote if it is a super user or not
int32_t reserve[8];
} SCreateUserMsg, SAlterUserMsg; } SCreateUserMsg, SAlterUserMsg;
typedef struct { typedef struct {
@ -614,6 +628,7 @@ typedef struct {
typedef struct { typedef struct {
char db[TSDB_TABLE_FNAME_LEN]; char db[TSDB_TABLE_FNAME_LEN];
int8_t ignoreNotExists; int8_t ignoreNotExists;
int32_t vgVersion;
int32_t reserve[8]; int32_t reserve[8];
} SUseDbMsg; } SUseDbMsg;
@ -775,9 +790,8 @@ typedef struct {
} SStbInfoMsg; } SStbInfoMsg;
typedef struct { typedef struct {
SMsgHead msgHead;
char tableFname[TSDB_TABLE_FNAME_LEN]; char tableFname[TSDB_TABLE_FNAME_LEN];
int8_t createFlag;
char tags[];
} STableInfoMsg; } STableInfoMsg;
typedef struct { typedef struct {
@ -792,6 +806,21 @@ typedef struct SSTableVgroupMsg {
int32_t numOfTables; int32_t numOfTables;
} SSTableVgroupMsg, SSTableVgroupRspMsg; } SSTableVgroupMsg, SSTableVgroupRspMsg;
typedef struct SVgroupInfo {
int32_t vgId;
int32_t hashBegin;
int32_t hashEnd;
int8_t inUse;
int8_t numOfEps;
SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
} SVgroupInfo;
typedef struct SVgroupListRspMsg {
int32_t vgroupNum;
int32_t vgroupVersion;
SVgroupInfo vgroupInfo[];
} SVgroupListRspMsg;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int8_t numOfEps; int8_t numOfEps;
@ -815,7 +844,7 @@ typedef struct {
int32_t tversion; int32_t tversion;
uint64_t tuid; uint64_t tuid;
uint64_t suid; uint64_t suid;
SVgroupMsg vgroup; int32_t vgId;
SSchema pSchema[]; SSchema pSchema[];
} STableMetaMsg; } STableMetaMsg;
@ -836,6 +865,16 @@ typedef struct {
char *data; char *data;
} STagData; } STagData;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
int32_t vgVersion;
int32_t vgNum;
int8_t hashMethod;
SVgroupInfo vgroupInfo[];
} SUseDbRsp;
/* /*
* sql: show tables like '%a_%' * sql: show tables like '%a_%'
* payload is the query condition, e.g., '%a_%' * payload is the query condition, e.g., '%a_%'
@ -860,16 +899,19 @@ typedef struct SShowRsp {
} SShowRsp; } SShowRsp;
typedef struct { typedef struct {
char ep[TSDB_EP_LEN]; // end point, hostname:port char ep[TSDB_EP_LEN]; // end point, hostname:port
int32_t reserve[8];
} SCreateDnodeMsg; } SCreateDnodeMsg;
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
int32_t reserve[8];
} SDropDnodeMsg; } SDropDnodeMsg;
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN]; char config[TSDB_DNODE_CONFIG_LEN];
int32_t reserve[8];
} SCfgDnodeMsg; } SCfgDnodeMsg;
typedef struct { typedef struct {

59
include/common/tglobal.h
View File

@ -29,12 +29,7 @@ extern char tsLocalFqdn[];
extern char tsLocalEp[]; extern char tsLocalEp[];
extern uint16_t tsServerPort; extern uint16_t tsServerPort;
extern int32_t tsStatusInterval; extern int32_t tsStatusInterval;
extern int32_t tsNumOfMnodes;
extern int8_t tsEnableVnodeBak;
extern int8_t tsEnableTelemetryReporting; extern int8_t tsEnableTelemetryReporting;
extern char tsArbitrator[];
extern int8_t tsArbOnline;
extern int64_t tsArbOnlineTimestamp;
// common // common
extern int tsRpcTimer; extern int tsRpcTimer;
@ -60,7 +55,6 @@ extern int tsCompatibleModel; // 2.0 compatible model
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing
extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked
extern int8_t tsKeepOriginalColumnName; extern int8_t tsKeepOriginalColumnName;
// client // client
@ -78,66 +72,15 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the
extern int32_t tsProjectExecInterval; extern int32_t tsProjectExecInterval;
extern int64_t tsMaxRetentWindow; extern int64_t tsMaxRetentWindow;
// db parameters in client
extern int32_t tsCacheBlockSize;
extern int32_t tsBlocksPerVnode;
extern int32_t tsTableIncStepPerVnode;
extern int32_t tsMaxVgroupsPerDb;
extern int16_t tsDaysPerFile;
extern int32_t tsDaysToKeep;
extern int32_t tsMinRowsInFileBlock;
extern int32_t tsMaxRowsInFileBlock;
extern int16_t tsCommitTime; // seconds
extern int32_t tsTimePrecision;
extern int8_t tsCompression;
extern int8_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int16_t tsPartitons;
extern int32_t tsQuorum;
extern int8_t tsUpdate;
extern int8_t tsCacheLastRow;
//tsdb
extern bool tsdbForceKeepFile;
// balance // balance
extern int8_t tsEnableBalance;
extern int8_t tsAlternativeRole;
extern int32_t tsBalanceInterval;
extern int32_t tsOfflineThreshold;
extern int8_t tsEnableFlowCtrl;
extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableSlaveQuery;
extern int8_t tsEnableAdjustMaster;
// restful
extern int32_t tsRestRowLimit;
extern int8_t tsTelegrafUseFieldNum;
// mqtt // interna
extern int8_t tsEnableMqttModule;
extern char tsMqttHostName[];
extern char tsMqttPort[];
extern char tsMqttUser[];
extern char tsMqttPass[];
extern char tsMqttClientId[];
extern char tsMqttTopic[];
// monitor
extern int8_t tsEnableMonitorModule;
extern char tsMonitorDbName[];
extern char tsInternalPass[];
extern int32_t tsMonitorInterval;
// stream
extern int8_t tsEnableStream;
// internal
extern int8_t tsPrintAuth; extern int8_t tsPrintAuth;
extern char tsVnodeDir[]; extern char tsVnodeDir[];
extern char tsMnodeDir[]; extern char tsMnodeDir[];
extern int64_t tsTickPerDay[3]; extern int64_t tsTickPerDay[3];
extern int32_t tsTopicBianryLen;
// system info // system info
extern float tsTotalLogDirGB; extern float tsTotalLogDirGB;

70
include/dnode/mnode/sdb/sdb.h
View File

@ -99,7 +99,7 @@ extern "C" {
sdbFreeRaw(pRaw); \ sdbFreeRaw(pRaw); \
return NULL; \ return NULL; \
} \ } \
dataPos += sizeof(int16_t); \ dataPos += sizeof(int16_t); \
} }
#define SDB_SET_INT8(pRaw, dataPos, val) \ #define SDB_SET_INT8(pRaw, dataPos, val) \
@ -144,9 +144,10 @@ typedef struct SSdbRow SSdbRow;
typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType; typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType;
typedef enum { typedef enum {
SDB_STATUS_CREATING = 1, SDB_STATUS_CREATING = 1,
SDB_STATUS_READY = 2, SDB_STATUS_UPDATING = 2,
SDB_STATUS_DROPPING = 3, SDB_STATUS_DROPPING = 3,
SDB_STATUS_DROPPED = 4 SDB_STATUS_READY = 4,
SDB_STATUS_DROPPED = 5
} ESdbStatus; } ESdbStatus;
typedef enum { typedef enum {
@ -174,67 +175,19 @@ typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
typedef struct { typedef struct {
/** ESdbType sdbType;
* @brief The sdb type of the table. EKeyType keyType;
*
*/
ESdbType sdbType;
/**
* @brief The key type of the table.
*
*/
EKeyType keyType;
/**
* @brief The callback function when the table is first deployed.
*
*/
SdbDeployFp deployFp; SdbDeployFp deployFp;
/**
* @brief Encode one row of the table into rawdata.
*
*/
SdbEncodeFp encodeFp; SdbEncodeFp encodeFp;
/**
* @brief Decode one row of the table from rawdata.
*
*/
SdbDecodeFp decodeFp; SdbDecodeFp decodeFp;
/**
* @brief The callback function when insert a row to sdb.
*
*/
SdbInsertFp insertFp; SdbInsertFp insertFp;
/**
* @brief The callback function when undate a row in sdb.
*
*/
SdbUpdateFp updateFp; SdbUpdateFp updateFp;
/**
* @brief The callback function when delete a row from sdb.
*
*/
SdbDeleteFp deleteFp; SdbDeleteFp deleteFp;
} SSdbTable; } SSdbTable;
typedef struct SSdbOpt { typedef struct SSdbOpt {
/**
* @brief The path of the sdb file.
*
*/
const char *path; const char *path;
SMnode *pMnode;
/**
* @brief The mnode object.
*
*/
SMnode *pMnode;
} SSdbOpt; } SSdbOpt;
/** /**
@ -342,6 +295,15 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter);
*/ */
int32_t sdbGetSize(SSdb *pSdb, ESdbType type); int32_t sdbGetSize(SSdb *pSdb, ESdbType type);
/**
* @brief Get the max id of the table, keyType of table should be INT32
*
* @param pSdb The sdb object.
* @param pIter The type of the table.
* @record int32_t The max id of the table
*/
int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type);
SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen);
void sdbFreeRaw(SSdbRaw *pRaw); void sdbFreeRaw(SSdbRaw *pRaw);
int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val);

191
include/dnode/vnode/tq/tq.h
View File

@ -16,9 +16,9 @@
#ifndef _TD_TQ_H_ #ifndef _TD_TQ_H_
#define _TD_TQ_H_ #define _TD_TQ_H_
#include "mallocator.h"
#include "os.h" #include "os.h"
#include "tutil.h" #include "tutil.h"
#include "mallocator.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -97,128 +97,125 @@ typedef struct TmqHeartbeatReq {
typedef struct TmqHeartbeatRsp { typedef struct TmqHeartbeatRsp {
} TmqHeartbeatRsp; } TmqHeartbeatRsp;
typedef struct TqTopicVhandle { typedef struct STqTopicVhandle {
int64_t topicId; int64_t topicId;
// executor for filter // executor for filter
void* filterExec; void* filterExec;
// callback for mnode // callback for mnode
// trigger when vnode list associated topic change // trigger when vnode list associated topic change
void* (*mCallback)(void*, void*); void* (*mCallback)(void*, void*);
} TqTopicVhandle; } STqTopicVhandle;
#define TQ_BUFFER_SIZE 8 #define TQ_BUFFER_SIZE 8
typedef struct TqBufferItem { typedef struct STqBufferItem {
int64_t offset; int64_t offset;
// executors are identical but not concurrent // executors are identical but not concurrent
// so there must be a copy in each item // so there must be a copy in each item
void* executor; void* executor;
int64_t size; int64_t size;
void* content; void* content;
} TqBufferItem; } STqBufferItem;
typedef struct TqBufferHandle { typedef struct STqBufferHandle {
// char* topic; //c style, end with '\0' // char* topic; //c style, end with '\0'
// int64_t cgId; // int64_t cgId;
// void* ahandle; // void* ahandle;
int64_t nextConsumeOffset; int64_t nextConsumeOffset;
int64_t topicId; int64_t floatingCursor;
int32_t head; int64_t topicId;
int32_t tail; int32_t head;
TqBufferItem buffer[TQ_BUFFER_SIZE]; int32_t tail;
} TqBufferHandle; STqBufferItem buffer[TQ_BUFFER_SIZE];
} STqBufferHandle;
typedef struct TqListHandle { typedef struct STqListHandle {
TqBufferHandle bufHandle; STqBufferHandle bufHandle;
struct TqListHandle* next; struct STqListHandle* next;
} TqListHandle; } STqListHandle;
typedef struct TqGroupHandle { typedef struct STqGroupHandle {
int64_t cId; int64_t cId;
int64_t cgId; int64_t cgId;
void* ahandle; void* ahandle;
int32_t topicNum; int32_t topicNum;
TqListHandle* head; STqListHandle* head;
} TqGroupHandle; } STqGroupHandle;
typedef struct TqQueryExec { typedef struct STqQueryExec {
void* src; void* src;
TqBufferItem* dest; STqBufferItem* dest;
void* executor; void* executor;
} TqQueryExec; } STqQueryExec;
typedef struct TqQueryMsg { typedef struct STqQueryMsg {
TqQueryExec* exec; STqQueryExec* exec;
struct TqQueryMsg* next; struct STqQueryMsg* next;
} TqQueryMsg; } STqQueryMsg;
typedef struct TqLogReader { typedef struct STqLogReader {
void* logHandle; void* logHandle;
int32_t (*logRead)(void* logHandle, void** data, int64_t ver); int32_t (*logRead)(void* logHandle, void** data, int64_t ver);
int64_t (*logGetFirstVer)(void* logHandle); int64_t (*logGetFirstVer)(void* logHandle);
int64_t (*logGetSnapshotVer)(void* logHandle); int64_t (*logGetSnapshotVer)(void* logHandle);
int64_t (*logGetLastVer)(void* logHandle); int64_t (*logGetLastVer)(void* logHandle);
} TqLogReader; } STqLogReader;
typedef struct STqCfg { typedef struct STqCfg {
// TODO // TODO
} STqCfg; } STqCfg;
typedef struct TqMemRef { typedef struct STqMemRef {
SMemAllocatorFactory *pAlloctorFactory; SMemAllocatorFactory* pAlloctorFactory;
SMemAllocator *pAllocator; SMemAllocator* pAllocator;
} TqMemRef; } STqMemRef;
typedef struct TqSerializedHead { typedef struct STqSerializedHead {
int16_t ver; int16_t ver;
int16_t action; int16_t action;
int32_t checksum; int32_t checksum;
int64_t ssize; int64_t ssize;
char content[]; char content[];
} TqSerializedHead; } STqSerializedHead;
typedef int (*TqSerializeFun)(const void* pObj, TqSerializedHead** ppHead); typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
typedef const void* (*TqDeserializeFun)(const TqSerializedHead* pHead, void** ppObj); typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj);
typedef void (*TqDeleteFun)(void*); typedef void (*FTqDelete)(void*);
#define TQ_BUCKET_MASK 0xFF #define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256 #define TQ_BUCKET_SIZE 256
#define TQ_PAGE_SIZE 4096 #define TQ_PAGE_SIZE 4096
//key + offset + size // key + offset + size
#define TQ_IDX_SIZE 24 #define TQ_IDX_SIZE 24
//4096 / 24 // 4096 / 24
#define TQ_MAX_IDX_ONE_PAGE 170 #define TQ_MAX_IDX_ONE_PAGE 170
//24 * 170 // 24 * 170
#define TQ_IDX_PAGE_BODY_SIZE 4080 #define TQ_IDX_PAGE_BODY_SIZE 4080
//4096 - 4080 // 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16 #define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0 #define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1 #define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2 #define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3 #define TQ_ACTION_INTXN 3
#define TQ_SVER 0 #define TQ_SVER 0
//TODO: inplace mode is not implemented // TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0 #define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1 #define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0 #define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2 #define TQ_DUP_INTXN_REJECT 2
static inline bool TqUpdateAppend(int32_t tqConfigFlag) { static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
return tqConfigFlag & TQ_UPDATE_APPEND;
}
static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
return tqConfigFlag & TQ_DUP_INTXN_REJECT;
}
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef struct TqMetaHandle { typedef struct TqMetaHandle {
int64_t key; int64_t key;
@ -226,44 +223,44 @@ typedef struct TqMetaHandle {
int64_t serializedSize; int64_t serializedSize;
void* valueInUse; void* valueInUse;
void* valueInTxn; void* valueInTxn;
} TqMetaHandle; } STqMetaHandle;
typedef struct TqMetaList { typedef struct TqMetaList {
TqMetaHandle handle; STqMetaHandle handle;
struct TqMetaList* next; struct TqMetaList* next;
//struct TqMetaList* inTxnPrev; // struct TqMetaList* inTxnPrev;
//struct TqMetaList* inTxnNext; // struct TqMetaList* inTxnNext;
struct TqMetaList* unpersistPrev; struct TqMetaList* unpersistPrev;
struct TqMetaList* unpersistNext; struct TqMetaList* unpersistNext;
} TqMetaList; } STqMetaList;
typedef struct TqMetaStore { typedef struct TqMetaStore {
TqMetaList* bucket[TQ_BUCKET_SIZE]; STqMetaList* bucket[TQ_BUCKET_SIZE];
//a table head // a table head
TqMetaList* unpersistHead; STqMetaList* unpersistHead;
//TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int fileFd; int fileFd;
//TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int idxFd; int idxFd;
char* dirPath; char* dirPath;
int32_t tqConfigFlag; int32_t tqConfigFlag;
TqSerializeFun pSerializer; FTqSerialize pSerializer;
TqDeserializeFun pDeserializer; FTqDeserialize pDeserializer;
TqDeleteFun pDeleter; FTqDelete pDeleter;
} TqMetaStore; } STqMetaStore;
typedef struct STQ { typedef struct STQ {
// the collection of group handle // the collection of group handle
// the handle of kvstore // the handle of kvstore
char* path; char* path;
STqCfg* tqConfig; STqCfg* tqConfig;
TqLogReader* tqLogReader; STqLogReader* tqLogReader;
TqMemRef tqMemRef; STqMemRef tqMemRef;
TqMetaStore* tqMeta; STqMetaStore* tqMeta;
} STQ; } STQ;
// open in each vnode // open in each vnode
STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac); STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac);
void tqDestroy(STQ*); void tqDestroy(STQ*);
// void* will be replace by a msg type // void* will be replace by a msg type
@ -272,19 +269,19 @@ int tqCommit(STQ*);
int tqConsume(STQ*, TmqConsumeReq*); int tqConsume(STQ*, TmqConsumeReq*);
TqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
TqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(TqGroupHandle*); int tqMoveOffsetToNext(STqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqRegisterContext(TqGroupHandle*, void* ahandle); int tqRegisterContext(STqGroupHandle*, void* ahandle);
int tqLaunchQuery(TqGroupHandle*); int tqLaunchQuery(STqGroupHandle*);
int tqSendLaunchQuery(TqGroupHandle*); int tqSendLaunchQuery(STqGroupHandle*);
int tqSerializeGroupHandle(const TqGroupHandle* gHandle, TqSerializedHead** ppHead); int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead);
const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle** gHandle); const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle);
#ifdef __cplusplus #ifdef __cplusplus
} }

10
include/dnode/vnode/tsdb/tsdb.h
View File

@ -21,15 +21,14 @@ extern "C" {
#endif #endif
// TYPES EXPOSED // TYPES EXPOSED
typedef struct STsdb STsdb; typedef struct STsdb STsdb;
typedef struct STsdbCfg STsdbCfg; typedef struct STsdbCfg STsdbCfg;
typedef struct STsdbMemAllocator STsdbMemAllocator;
// STsdb // STsdb
STsdb *tsdbOpen(const char *path, const STsdbCfg *); STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg);
void tsdbClose(STsdb *); void tsdbClose(STsdb *);
void tsdbRemove(const char *path); void tsdbRemove(const char *path);
int tsdbInsertData(STsdb *pTsdb, void *pData, int len); int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg);
// STsdbCfg // STsdbCfg
int tsdbOptionsInit(STsdbCfg *); int tsdbOptionsInit(STsdbCfg *);
@ -41,7 +40,6 @@ struct STsdbCfg {
uint32_t keep0; uint32_t keep0;
uint32_t keep1; uint32_t keep1;
uint32_t keep2; uint32_t keep2;
/* TODO */
}; };
#ifdef __cplusplus #ifdef __cplusplus

4
include/dnode/vnode/vnode.h
View File

@ -68,9 +68,11 @@ typedef struct SVnodeCfg {
/** /**
* @brief Initialize the vnode module * @brief Initialize the vnode module
* *
* @param nthreads number of commit threads. 0 for no threads and
* a schedule queue should be given (TODO)
* @return int 0 for success and -1 for failure * @return int 0 for success and -1 for failure
*/ */
int vnodeInit(); int vnodeInit(uint16_t nthreads);
/** /**
* @brief clear a vnode * @brief clear a vnode

88
include/libs/catalog/catalog.h
View File

@ -27,83 +27,83 @@ extern "C" {
#include "transport.h" #include "transport.h"
#include "common.h" #include "common.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "query.h"
struct SCatalog; struct SCatalog;
typedef struct SMetaReq { typedef struct SCatalogReq {
char clusterId[TSDB_CLUSTER_ID_LEN]; char dbName[TSDB_DB_NAME_LEN];
SArray *pTableName; // table full name SArray *pTableName; // table full name
SArray *pUdf; // udf name SArray *pUdf; // udf name
bool qNodeEpset; // valid qnode bool qNodeRequired; // valid qnode
} SMetaReq; } SCatalogReq;
typedef struct SMetaData { typedef struct SMetaData {
SArray *pTableMeta; // tableMeta SArray *pTableMeta; // STableMeta array
SArray *pVgroupInfo; // vgroupInfo list SArray *pVgroupInfo; // SVgroupInfo list
SArray *pUdfList; // udf info list SArray *pUdfList; // udf info list
SEpSet *pEpSet; // qnode epset list SEpSet *pEpSet; // qnode epset list
} SMetaData; } SMetaData;
typedef struct STableComInfo { typedef struct SCatalogCfg {
uint8_t numOfTags; // the number of tags in schema bool enableVgroupCache;
uint8_t precision; // the number of precision uint32_t maxTblCacheNum;
int16_t numOfColumns; // the number of columns uint32_t maxDBCacheNum;
int32_t rowSize; // row size of the schema } SCatalogCfg;
} STableComInfo;
/* int32_t catalogInit(SCatalogCfg *cfg);
* ASSERT(sizeof(SCTableMeta) == 24)
* ASSERT(tableType == TSDB_CHILD_TABLE)
* The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
*/
typedef struct SCTableMeta {
int32_t vgId:24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
} SCTableMeta;
/*
* Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
*/
typedef struct STableMeta {
int32_t vgId:24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
int16_t sversion;
int16_t tversion;
STableComInfo tableInfo;
SSchema schema[];
} STableMeta;
/** /**
* Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side.
* There is ONLY one SCatalog object for one process space, and this function returns a singleton. * There is ONLY one SCatalog object for one process space, and this function returns a singleton.
* @param pMgmtEps * @param clusterId
* @return * @return
*/ */
struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle);
int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version);
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo);
int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName);
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
/**
* get table's vgroup list.
* @param clusterId
* @pVgroupList - array of SVgroupInfo
* @return
*/
int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList);
/** /**
* Get the required meta data from mnode. * Get the required meta data from mnode.
* Note that this is a synchronized API and is also thread-safety. * Note that this is a synchronized API and is also thread-safety.
* @param pCatalog * @param pCatalog
* @param pMgmtEps
* @param pMetaReq * @param pMetaReq
* @param pMetaData * @param pMetaData
* @return * @return
*/ */
int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet);
/** /**
* Destroy catalog service handle * Destroy catalog and relase all resources
* @param pCatalog * @param pCatalog
*/ */
void destroyCatalog(struct SCatalog* pCatalog); void catalogDestroy(void);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /*_TD_CATALOG_H_*/ #endif /*_TD_CATALOG_H_*/

12
include/libs/parser/parser.h
View File

@ -132,13 +132,15 @@ struct SInsertStmtInfo;
bool qIsInsertSql(const char* pStr, size_t length); bool qIsInsertSql(const char* pStr, size_t length);
typedef struct SParseContext { typedef struct SParseContext {
const char* pAcctId;
const char* pDbname;
void *pRpc;
const char* pClusterId;
const SEpSet* pEpSet;
int64_t id; // query id, generated by uuid generator
int8_t schemaAttached; // denote if submit block is built with table schema or not
const char* pSql; // sql string const char* pSql; // sql string
size_t sqlLen; // length of the sql string size_t sqlLen; // length of the sql string
int64_t id; // operator id, generated by uuid generator
const char* pDbname;
const SEpSet* pEpSet;
int8_t schemaAttached; // denote if submit block is built with table schema or not
char* pMsg; // extended error message if exists to help avoid the problem in sql statement. char* pMsg; // extended error message if exists to help avoid the problem in sql statement.
int32_t msgLen; // max length of the msg int32_t msgLen; // max length of the msg
} SParseContext; } SParseContext;

187
include/libs/planner/planner.h
View File

@ -20,124 +20,115 @@
extern "C" { extern "C" {
#endif #endif
#include "taosmsg.h"
#define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_MERGE 1
#define QUERY_TYPE_PARTIAL 2 #define QUERY_TYPE_PARTIAL 2
#define QUERY_TYPE_SCAN 3
enum OPERATOR_TYPE_E { enum OPERATOR_TYPE_E {
OP_TableScan = 1, OP_Unknown,
OP_DataBlocksOptScan = 2, #define INCLUDE_AS_ENUM
OP_TableSeqScan = 3, #include "plannerOp.h"
OP_TagScan = 4, #undef INCLUDE_AS_ENUM
OP_TableBlockInfoScan= 5, OP_TotalNum
OP_Aggregate = 6,
OP_Project = 7,
OP_Groupby = 8,
OP_Limit = 9,
OP_SLimit = 10,
OP_TimeWindow = 11,
OP_SessionWindow = 12,
OP_StateWindow = 22,
OP_Fill = 13,
OP_MultiTableAggregate = 14,
OP_MultiTableTimeInterval = 15,
// OP_DummyInput = 16, //TODO remove it after fully refactor.
// OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream.
// OP_GlobalAggregate = 18, // global merge for the multi-way data sources.
OP_Filter = 19,
OP_Distinct = 20,
OP_Join = 21,
OP_AllTimeWindow = 23,
OP_AllMultiTableTimeInterval = 24,
OP_Order = 25,
OP_Exchange = 26,
}; };
struct SEpSet; struct SEpSet;
struct SQueryPlanNode;
struct SQueryDistPlanNode;
struct SQueryStmtInfo; struct SQueryStmtInfo;
typedef struct SSubquery { typedef SSchema SSlotSchema;
int64_t queryId; // the subquery id created by qnode
int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL
int32_t level; // the execution level of current subquery, starting from 0.
SArray *pUpstream; // the upstream,from which to fetch the result
struct SQueryDistPlanNode *pNode; // physical plan of current subquery
} SSubquery;
typedef struct SQueryJob { typedef struct SDataBlockSchema {
SArray **pSubqueries; SSlotSchema *pSchema;
int32_t numOfLevels; int32_t numOfCols; // number of columns
int32_t currentLevel; } SDataBlockSchema;
} SQueryJob;
typedef struct SQueryNodeBasicInfo {
int32_t type; // operator type
const char *name; // operator name
} SQueryNodeBasicInfo;
typedef struct SPhyNode {
SQueryNodeBasicInfo info;
SArray *pTargets; // target list to be computed or scanned at this node
SArray *pConditions; // implicitly-ANDed qual conditions
SDataBlockSchema targetSchema;
// children plan to generated result for current node to process
// in case of join, multiple plan nodes exist.
SArray *pChildren;
struct SPhyNode *pParent;
} SPhyNode;
typedef struct SScanPhyNode {
SPhyNode node;
uint64_t uid; // unique id of the table
int8_t tableType;
} SScanPhyNode;
typedef SScanPhyNode SSystemTableScanPhyNode;
typedef SScanPhyNode STagScanPhyNode;
typedef struct STableScanPhyNode {
SScanPhyNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow window;
SArray *pTagsConditions; // implicitly-ANDed tag qual conditions
} STableScanPhyNode;
typedef STableScanPhyNode STableSeqScanPhyNode;
typedef struct SProjectPhyNode {
SPhyNode node;
} SProjectPhyNode;
typedef struct SExchangePhyNode {
SPhyNode node;
uint64_t srcTemplateId; // template id of datasource suplans
SArray *pSourceEpSet; // SEpSet, scheduler fill by calling qSetSuplanExecutionNode
} SExchangePhyNode;
typedef struct SSubplanId {
uint64_t queryId;
uint64_t templateId;
uint64_t subplanId;
} SSubplanId;
typedef struct SSubplan {
SSubplanId id; // unique id of the subplan
int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN
int32_t level; // the execution level of current subplan, starting from 0.
SEpSet execEpSet; // for the scan sub plan, the optional execution node
SArray *pChildern; // the datasource subplan,from which to fetch the result
SArray *pParents; // the data destination subplan, get data from current subplan
SPhyNode *pNode; // physical plan of current subplan
} SSubplan;
typedef struct SQueryDag {
SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
} SQueryDag;
/**
* Create the physical plan for the query, according to the AST.
*/
int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag);
int32_t qSetSuplanExecutionNode(SArray* subplans, SArray* nodes);
int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str);
/** /**
* Optimize the query execution plan, currently not implement yet. * Convert to subplan to string for the scheduler to send to the executor
* @param pQueryNode
* @return
*/ */
int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode); int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str);
/**
* Create the query plan according to the bound AST, which is in the form of pQueryInfo
* @param pQueryInfo
* @param pQueryNode
* @return
*/
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode);
/**
* Convert the query plan to string, in order to display it in the shell.
* @param pQueryNode
* @return
*/
int32_t qQueryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
/**
* Restore the SQL statement according to the logic query plan.
* @param pQueryNode
* @param sql
* @return
*/
int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
/**
* Create the physical plan for the query, according to the logic plan.
* @param pQueryNode
* @param pPhyNode
* @return
*/
int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDistPlanNode *pPhyNode);
/**
* Convert to physical plan to string to enable to print it out in the shell.
* @param pPhyNode
* @param str
* @return
*/
int32_t qPhyPlanToString(struct SQueryDistPlanNode *pPhyNode, char** str);
/**
* Destroy the query plan object.
* @return
*/
void* qDestroyQueryPlan(struct SQueryPlanNode* pQueryNode);
/** /**
* Destroy the physical plan. * Destroy the physical plan.
* @param pQueryPhyNode * @param pQueryPhyNode
* @return * @return
*/ */
void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode); void qDestroyQueryDag(struct SQueryDag* pDag);
/**
* Create the query job from the physical execution plan
* @param pPhyNode
* @param pJob
* @return
*/
int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQueryJob** pJob);
#ifdef __cplusplus #ifdef __cplusplus
} }

48
include/libs/planner/plannerOp.h Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#if defined(INCLUDE_AS_ENUM) // enum define mode
#undef OP_ENUM_MACRO
#define OP_ENUM_MACRO(op) OP_##op,
#elif defined(INCLUDE_AS_NAME) // comment define mode
#undef OP_ENUM_MACRO
#define OP_ENUM_MACRO(op) #op,
#else
#error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME
#endif
OP_ENUM_MACRO(TableScan)
OP_ENUM_MACRO(DataBlocksOptScan)
OP_ENUM_MACRO(TableSeqScan)
OP_ENUM_MACRO(TagScan)
OP_ENUM_MACRO(SystemTableScan)
OP_ENUM_MACRO(Aggregate)
OP_ENUM_MACRO(Project)
OP_ENUM_MACRO(Groupby)
OP_ENUM_MACRO(Limit)
OP_ENUM_MACRO(SLimit)
OP_ENUM_MACRO(TimeWindow)
OP_ENUM_MACRO(SessionWindow)
OP_ENUM_MACRO(StateWindow)
OP_ENUM_MACRO(Fill)
OP_ENUM_MACRO(MultiTableAggregate)
OP_ENUM_MACRO(MultiTableTimeInterval)
OP_ENUM_MACRO(Filter)
OP_ENUM_MACRO(Distinct)
OP_ENUM_MACRO(Join)
OP_ENUM_MACRO(AllTimeWindow)
OP_ENUM_MACRO(AllMultiTableTimeInterval)
OP_ENUM_MACRO(Order)
OP_ENUM_MACRO(Exchange)

93
include/libs/query/query.h Normal file
View File

@ -0,0 +1,93 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_QUERY_H_
#define _TD_QUERY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tarray.h"
#include "thash.h"
typedef SVgroupListRspMsg SVgroupListInfo;
typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema
uint8_t precision; // the number of precision
int16_t numOfColumns; // the number of columns
int32_t rowSize; // row size of the schema
} STableComInfo;
/*
* ASSERT(sizeof(SCTableMeta) == 24)
* ASSERT(tableType == TSDB_CHILD_TABLE)
* The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
*/
typedef struct SCTableMeta {
int32_t vgId:24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
} SCTableMeta;
/*
* Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
*/
typedef struct STableMeta {
//BEGIN: KEEP THIS PART SAME WITH SCTableMeta
int32_t vgId:24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
//END: KEEP THIS PART SAME WITH SCTableMeta
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
int16_t sversion;
int16_t tversion;
STableComInfo tableInfo;
SSchema schema[];
} STableMeta;
typedef struct SDBVgroupInfo {
int32_t vgVersion;
int8_t hashMethod;
SHashObj *vgInfo; //key:vgId, value:SVgroupInfo
} SDBVgroupInfo;
typedef struct SUseDbOutput {
char db[TSDB_FULL_DB_NAME_LEN];
SDBVgroupInfo dbVgroup;
} SUseDbOutput;
typedef struct STableMetaOutput {
int32_t metaNum;
char ctbFname[TSDB_TABLE_FNAME_LEN];
char tbFname[TSDB_TABLE_FNAME_LEN];
SCTableMeta ctbMeta;
STableMeta *tbMeta;
} STableMetaOutput;
extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize);
#ifdef __cplusplus
}
#endif
#endif /*_TD_QUERY_H_*/

37
include/libs/scheduler/scheduler.h
View File

@ -20,7 +20,42 @@
extern "C" { extern "C" {
#endif #endif
struct SQueryJob; typedef struct SQueryProfileSummary {
int64_t startTs; // Object created and added into the message queue
int64_t endTs; // the timestamp when the task is completed
int64_t cputime; // total cpu cost, not execute elapsed time
int64_t loadRemoteDataDuration; // remote io time
int64_t loadNativeDataDuration; // native disk io time
uint64_t loadNativeData; // blocks + SMA + header files
uint64_t loadRemoteData; // remote data acquired by exchange operator.
uint64_t waitDuration; // the time to waiting to be scheduled in queue does matter, so we need to record it
int64_t addQTs; // the time to be added into the message queue, used to calculate the waiting duration in queue.
uint64_t totalRows;
uint64_t loadRows;
uint32_t totalBlocks;
uint32_t loadBlocks;
uint32_t loadBlockAgg;
uint32_t skipBlocks;
uint64_t resultSize; // generated result size in Kb.
} SQueryProfileSummary;
typedef struct SQueryTask {
uint64_t queryId; // query id
uint64_t taskId; // task id
char *pSubplan; // operator tree
uint64_t status; // task status
SQueryProfileSummary summary; // task execution summary
void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage
} SQueryTask;
typedef struct SQueryJob {
SArray **pSubtasks;
// todo
} SQueryJob;
/** /**
* Process the query job, generated according to the query physical plan. * Process the query job, generated according to the query physical plan.

150
include/libs/wal/wal.h
View File

@ -16,53 +16,75 @@
#define _TD_WAL_H_ #define _TD_WAL_H_
#include "os.h" #include "os.h"
#include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlog.h" #include "tlog.h"
#include "tarray.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
extern int32_t wDebugFlag; extern int32_t wDebugFlag;
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} #define wFatal(...) \
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} { \
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} if (wDebugFlag & DEBUG_FATAL) { \
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} } \
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} }
#define wError(...) \
{ \
if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
} \
}
#define wWarn(...) \
{ \
if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
} \
}
#define wInfo(...) \
{ \
if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", 255, __VA_ARGS__); \
} \
}
#define wDebug(...) \
{ \
if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
}
#define wTrace(...) \
{ \
if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
}
#define WAL_PREFIX "wal" #define WAL_HEAD_VER 0
#define WAL_PREFIX_LEN 3
#define WAL_NOSUFFIX_LEN 20 #define WAL_NOSUFFIX_LEN 20
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) #define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
#define WAL_LOG_SUFFIX "log" #define WAL_LOG_SUFFIX "log"
#define WAL_INDEX_SUFFIX "idx" #define WAL_INDEX_SUFFIX "idx"
#define WAL_REFRESH_MS 1000 #define WAL_REFRESH_MS 1000
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) #define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) #define WAL_CUR_FAILED 1
#define WAL_CUR_POS_WRITABLE 1
#define WAL_CUR_FILE_WRITABLE 2
#define WAL_CUR_FAILED 4
#pragma pack(push,1) #pragma pack(push, 1)
typedef enum { typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType;
TAOS_WAL_NOLOG = 0,
TAOS_WAL_WRITE = 1,
TAOS_WAL_FSYNC = 2
} EWalType;
typedef struct SWalReadHead { typedef struct SWalReadHead {
int8_t sver; int8_t headVer;
uint8_t msgType; uint8_t msgType;
int8_t reserved[2]; int8_t reserved[2];
int32_t len; int32_t len;
//int64_t ingestTs; //not implemented int64_t ingestTs; // not implemented
int64_t version; int64_t version;
char body[]; char body[];
} SWalReadHead; } SWalReadHead;
typedef struct { typedef struct {
@ -72,20 +94,12 @@ typedef struct {
int32_t rollPeriod; // secs int32_t rollPeriod; // secs
int64_t retentionSize; int64_t retentionSize;
int64_t segSize; int64_t segSize;
EWalType level; // wal level EWalType level; // wal level
} SWalCfg; } SWalCfg;
typedef struct { typedef struct {
//union { uint32_t cksumHead;
//uint32_t info; uint32_t cksumBody;
//struct {
//uint32_t sver:3;
//uint32_t msgtype: 5;
//uint32_t reserved : 24;
//};
//};
uint32_t cksumHead;
uint32_t cksumBody;
SWalReadHead head; SWalReadHead head;
} SWalHead; } SWalHead;
@ -100,38 +114,38 @@ typedef struct SWalVer {
typedef struct SWal { typedef struct SWal {
// cfg // cfg
SWalCfg cfg; SWalCfg cfg;
int32_t fsyncSeq;
// meta
SWalVer vers; SWalVer vers;
//file set
int32_t writeCur;
int64_t writeLogTfd; int64_t writeLogTfd;
int64_t writeIdxTfd; int64_t writeIdxTfd;
SArray* fileInfoSet; int32_t writeCur;
//ctl SArray *fileInfoSet;
int32_t curStatus; // status
int32_t fsyncSeq;
int64_t totSize; int64_t totSize;
int64_t refId;
int64_t lastRollSeq; int64_t lastRollSeq;
// ctl
int64_t refId;
pthread_mutex_t mutex; pthread_mutex_t mutex;
//path // path
char path[WAL_PATH_LEN]; char path[WAL_PATH_LEN];
//reusable write head // reusable write head
SWalHead writeHead; SWalHead writeHead;
} SWal; // WAL HANDLE } SWal; // WAL HANDLE
typedef struct SWalReadHandle { typedef struct SWalReadHandle {
SWal* pWal; SWal *pWal;
int64_t readLogTfd; int64_t readLogTfd;
int64_t readIdxTfd; int64_t readIdxTfd;
int64_t curFileFirstVer; int64_t curFileFirstVer;
int64_t curVersion; int64_t curVersion;
int64_t capacity; int64_t capacity;
int64_t status; //if cursor valid int64_t status; // if cursor valid
SWalHead* pHead; SWalHead *pHead;
} SWalReadHandle; } SWalReadHandle;
#pragma pack(pop) #pragma pack(pop)
typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
// module initialization // module initialization
int32_t walInit(); int32_t walInit();
@ -151,17 +165,17 @@ int32_t walCommit(SWal *, int64_t ver);
// truncate after // truncate after
int32_t walRollback(SWal *, int64_t ver); int32_t walRollback(SWal *, int64_t ver);
// notify that previous logs can be pruned safely // notify that previous logs can be pruned safely
int32_t walBeginTakeSnapshot(SWal *, int64_t ver); int32_t walBeginSnapshot(SWal *, int64_t ver);
int32_t walEndTakeSnapshot(SWal *); int32_t walEndSnapshot(SWal *);
//int32_t walDataCorrupted(SWal*); // int32_t walDataCorrupted(SWal*);
// read // read
SWalReadHandle* walOpenReadHandle(SWal *); SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *); void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
int32_t walRead(SWal *, SWalHead **, int64_t ver); int32_t walRead(SWal *, SWalHead **, int64_t ver);
int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); // int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
// lifecycle check // lifecycle check
int64_t walGetFirstVer(SWal *); int64_t walGetFirstVer(SWal *);

4
include/os/osMemory.h
View File

@ -23,8 +23,8 @@ extern "C" {
#define tfree(x) \ #define tfree(x) \
do { \ do { \
if (x) { \ if (x) { \
free((void *)x); \ free((void *)(x)); \
x = 0; \ (x) = 0; \
} \ } \
} while (0) } while (0)

166
include/util/taoserror.h
View File

@ -117,72 +117,93 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format")
#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type")
#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range")
#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input")
// mnode // mnode-common
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0300)
#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0301)
#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) #define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302)
#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) #define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0303)
#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0304) #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0304)
#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) #define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0305)
#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) #define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0306)
#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) #define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0307)
#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) #define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0308)
#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) #define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0309)
#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) #define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030A)
#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C)
#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D)
#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x030E)
#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310)
#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311)
#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312)
#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313)
#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314)
#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0320) // mnode-show
#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0321) #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310)
#define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0322)
#define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x0323) // mnode-profile
#define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0324) #define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x0320)
#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) #define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x0321)
#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0326) #define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x0322)
#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0327) #define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0323)
#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0328) #define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0324)
#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0329) #define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0325)
#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x032A) #define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0326)
#define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x032B) #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0327)
// mnode-sdb
#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0330)
#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0331)
#define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0332)
#define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x0333)
#define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0334)
#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0335)
#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0336)
#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0337)
#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0338)
#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0339)
#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x033A)
#define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x033B)
// mnode-dnode // mnode-dnode
#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) #define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340)
#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) #define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341)
#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0332) #define TSDB_CODE_MND_TOO_MANY_DNODES TAOS_DEF_ERROR_CODE(0, 0x0342)
#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0333) #define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0343)
#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0334) #define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0344)
#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0335) #define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0345)
#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0336) #define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0346)
#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0337) #define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0347)
#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0348)
// mnode-vgroup // mnode-mnode
#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350)
#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351)
#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") #define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0352)
#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready")
// mnode-acct // mnode-acct
#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360)
#define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") #define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0361)
#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") #define TSDB_CODE_MND_TOO_MANY_ACCTS TAOS_DEF_ERROR_CODE(0, 0x0362)
#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0343) //"Account authorization has expired") #define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0363)
#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0364)
#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) //"User already exists") // mnode-user
#define TSDB_CODE_MND_USER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) //"Invalid user") #define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370)
#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0352) //"Invalid user format") #define TSDB_CODE_MND_USER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371)
#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0353) //"Invalid password format") #define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0372)
#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0354) //"Can not get user from conn") #define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0373)
#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) //"Too many users") #define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0374)
#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0375)
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") // mnode-db
#define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0380)
#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381)
#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0382)
#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0384)
#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0385)
#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0386)
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0387)
// mnode-vgroup
#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390)
#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0391)
#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392)
// mnode-stable // mnode-stable
#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360)
@ -223,30 +244,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length")
#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length")
#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380)
#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381)
#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381)
#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382)
#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0383)
#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account")
// dnode // dnode
#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400)
@ -501,6 +498,13 @@ int32_t* taosGetErrno();
// monitor // monitor
#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") #define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection")
// catalog
#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error
#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters
#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready
#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error
#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
include/util/tarray.h
View File

@ -41,7 +41,7 @@ typedef struct SArray {
* @param elemSize * @param elemSize
* @return * @return
*/ */
void* taosArrayInit(size_t size, size_t elemSize); SArray* taosArrayInit(size_t size, size_t elemSize);
/** /**
* *

5
include/util/tdef.h
View File

@ -25,7 +25,9 @@ extern "C" {
#define TSDB__packed #define TSDB__packed
#define TSKEY int64_t #define TSKEY int64_t
#define TSKEY_INITIAL_VAL INT64_MIN #define TSKEY_MIN INT64_MIN
#define TSKEY_MAX (INT64_MAX - 1)
#define TSKEY_INITIAL_VAL TSKEY_MIN
// Bytes for each type. // Bytes for each type.
extern const int32_t TYPE_BYTES[15]; extern const int32_t TYPE_BYTES[15];
@ -210,6 +212,7 @@ do { \
#define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_NAME_LEN 32
#define TSDB_STEP_DESC_LEN 128 #define TSDB_STEP_DESC_LEN 128
#define TSDB_ERROR_MSG_LEN 1024
#define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_MQTT_HOSTNAME_LEN 64 #define TSDB_MQTT_HOSTNAME_LEN 64

4
include/util/tdlist.h
View File

@ -58,8 +58,8 @@ extern "C" {
// Double linked list // Double linked list
#define TD_DLIST_NODE(TYPE) \ #define TD_DLIST_NODE(TYPE) \
struct { \ struct { \
TYPE *dl_prev_; \ struct TYPE *dl_prev_; \
TYPE *dl_next_; \ struct TYPE *dl_next_; \
} }
#define TD_DLIST(TYPE) \ #define TD_DLIST(TYPE) \

3
include/util/tlog.h
View File

@ -32,7 +32,6 @@ extern int32_t mDebugFlag;
extern int32_t cDebugFlag; extern int32_t cDebugFlag;
extern int32_t jniDebugFlag; extern int32_t jniDebugFlag;
extern int32_t tmrDebugFlag; extern int32_t tmrDebugFlag;
extern int32_t sdbDebugFlag;
extern int32_t httpDebugFlag; extern int32_t httpDebugFlag;
extern int32_t mqttDebugFlag; extern int32_t mqttDebugFlag;
extern int32_t monDebugFlag; extern int32_t monDebugFlag;
@ -45,6 +44,8 @@ extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag; extern int32_t tsdbDebugFlag;
extern int32_t cqDebugFlag; extern int32_t cqDebugFlag;
extern int32_t debugFlag; extern int32_t debugFlag;
extern int32_t ctgDebugFlag;
#define DEBUG_FATAL 1U #define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL #define DEBUG_ERROR DEBUG_FATAL

8
include/util/tmacro.h
View File

@ -29,13 +29,11 @@ extern "C" {
#define TD_MOD_UNCLEARD 0 #define TD_MOD_UNCLEARD 0
#define TD_MOD_CLEARD 1 #define TD_MOD_CLEARD 1
#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED typedef int8_t td_mode_flag_t;
#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD
#define TD_CHECK_AND_SET_MODE_INIT(MOD) \ #define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED)
atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED)
#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD) #define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD)
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/util/tref.h
View File

@ -17,6 +17,8 @@
#ifndef _TD_UTIL_REF_H #ifndef _TD_UTIL_REF_H
#define _TD_UTIL_REF_H #define _TD_UTIL_REF_H
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif

1
source/client/CMakeLists.txt
View File

@ -7,6 +7,7 @@ target_include_directories(
) )
target_link_libraries( target_link_libraries(
taos taos
PRIVATE common
INTERFACE api INTERFACE api
PRIVATE os util common transport parser catalog function PRIVATE os util common transport parser catalog function
) )

0
source/client/src/client.c Normal file
View File

6
source/common/inc/commonInt.h
View File

@ -20,8 +20,12 @@
extern "C" { extern "C" {
#endif #endif
extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /*_TD_COMMON_INT_H_*/ #endif /*_TD_COMMON_INT_H_*/

2
source/common/src/tmessage.c → source/common/src/taosmsg.c
View File

@ -16,3 +16,5 @@
#define TAOS_MESSAGE_C #define TAOS_MESSAGE_C
#include "taosmsg.h" #include "taosmsg.h"

434
source/common/src/tglobal.c
View File

@ -30,19 +30,12 @@
// cluster // cluster
char tsFirst[TSDB_EP_LEN] = {0}; char tsFirst[TSDB_EP_LEN] = {0};
char tsSecond[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0};
char tsArbitrator[TSDB_EP_LEN] = {0};
char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030; uint16_t tsServerPort = 6030;
int32_t tsStatusInterval = 1; // second int32_t tsStatusInterval = 1; // second
int32_t tsNumOfMnodes = 1;
int8_t tsEnableVnodeBak = 1;
int8_t tsEnableTelemetryReporting = 0; int8_t tsEnableTelemetryReporting = 0;
int8_t tsArbOnline = 0;
int64_t tsArbOnlineTimestamp = TSDB_ARB_DUMMY_TIME;
char tsEmail[TSDB_FQDN_LEN] = {0}; char tsEmail[TSDB_FQDN_LEN] = {0};
int32_t tsDnodeId = 0;
int64_t tsDnodeStartTime = 0;
// common // common
int32_t tsRpcTimer = 300; int32_t tsRpcTimer = 300;
@ -129,59 +122,19 @@ int32_t tsRetrieveBlockingModel = 0;
// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
int8_t tsKeepOriginalColumnName = 0; int8_t tsKeepOriginalColumnName = 0;
// db parameters
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP;
int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds
int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO;
// tsdb config // tsdb config
// For backward compatibility // For backward compatibility
bool tsdbForceKeepFile = false; bool tsdbForceKeepFile = false;
// balance // balance
int8_t tsEnableBalance = 1;
int8_t tsAlternativeRole = 0;
int32_t tsBalanceInterval = 300; // seconds
int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days
int8_t tsEnableFlowCtrl = 1; int8_t tsEnableFlowCtrl = 1;
int8_t tsEnableSlaveQuery = 1; int8_t tsEnableSlaveQuery = 1;
int8_t tsEnableAdjustMaster = 1; int8_t tsEnableAdjustMaster = 1;
// restful
int32_t tsRestRowLimit = 10240;
int8_t tsTelegrafUseFieldNum = 0;
// mqtt
int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default
char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org";
char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883";
char tsMqttUser[TSDB_MQTT_USER_LEN] = {0};
char tsMqttPass[TSDB_MQTT_PASS_LEN] = {0};
char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber";
char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // #
// monitor // monitor
int8_t tsEnableMonitorModule = 1;
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey"; char tsInternalPass[] = "secretkey";
int32_t tsMonitorInterval = 30; // seconds
// stream
int8_t tsEnableStream = 1;
// internal // internal
int8_t tsCompactMnodeWal = 0; int8_t tsCompactMnodeWal = 0;
@ -191,7 +144,6 @@ char tsDnodeDir[PATH_MAX] = {0};
char tsMnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0};
int32_t tsDiskCfgNum = 0; int32_t tsDiskCfgNum = 0;
int32_t tsTopicBianryLen = 16000;
#ifndef _STORAGE #ifndef _STORAGE
SDiskCfg tsDiskCfg[1]; SDiskCfg tsDiskCfg[1];
@ -245,14 +197,10 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() { void taosSetAllDebugFlag() {
if (debugFlag != 0) { if (debugFlag != 0) {
mDebugFlag = debugFlag; mDebugFlag = debugFlag;
sdbDebugFlag = debugFlag;
dDebugFlag = debugFlag; dDebugFlag = debugFlag;
vDebugFlag = debugFlag; vDebugFlag = debugFlag;
jniDebugFlag = debugFlag; jniDebugFlag = debugFlag;
odbcDebugFlag = debugFlag; odbcDebugFlag = debugFlag;
httpDebugFlag = debugFlag;
mqttDebugFlag = debugFlag;
monDebugFlag = debugFlag;
qDebugFlag = debugFlag; qDebugFlag = debugFlag;
rpcDebugFlag = debugFlag; rpcDebugFlag = debugFlag;
uDebugFlag = debugFlag; uDebugFlag = debugFlag;
@ -459,16 +407,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "arbitrator";
cfg.ptr = tsArbitrator;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_EP_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// dnode configs // dnode configs
cfg.option = "numOfThreadsPerCore"; cfg.option = "numOfThreadsPerCore";
cfg.ptr = &tsNumOfThreadsPerCore; cfg.ptr = &tsNumOfThreadsPerCore;
@ -509,26 +447,6 @@ static void doInitGlobalConfig(void) {
cfg.ptrLength = 0; cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "numOfMnodes";
cfg.ptr = &tsNumOfMnodes;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 3;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "vnodeBak";
cfg.ptr = &tsEnableVnodeBak;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "telemetryReporting"; cfg.option = "telemetryReporting";
cfg.ptr = &tsEnableTelemetryReporting; cfg.ptr = &tsEnableTelemetryReporting;
@ -540,37 +458,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "balance";
cfg.ptr = &tsEnableBalance;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "balanceInterval";
cfg.ptr = &tsBalanceInterval;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 30000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// 0-any; 1-mnode; 2-vnode
cfg.option = "role";
cfg.ptr = &tsAlternativeRole;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 2;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// timer // timer
cfg.option = "maxTmrCtrl"; cfg.option = "maxTmrCtrl";
cfg.ptr = &tsMaxTmrCtrl; cfg.ptr = &tsMaxTmrCtrl;
@ -582,26 +469,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "monitorInterval";
cfg.ptr = &tsMonitorInterval;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 1;
cfg.maxValue = 600;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosAddConfigOption(cfg);
cfg.option = "offlineThreshold";
cfg.ptr = &tsOfflineThreshold;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 3;
cfg.maxValue = 86400 * 365;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosAddConfigOption(cfg);
cfg.option = "rpcTimer"; cfg.option = "rpcTimer";
cfg.ptr = &tsRpcTimer; cfg.ptr = &tsRpcTimer;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -712,186 +579,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "maxVgroupsPerDb";
cfg.ptr = &tsMaxVgroupsPerDb;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 8192;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "cache";
cfg.ptr = &tsCacheBlockSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE;
cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MB;
taosAddConfigOption(cfg);
cfg.option = "blocks";
cfg.ptr = &tsBlocksPerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TOTAL_BLOCKS;
cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "days";
cfg.ptr = &tsDaysPerFile;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DAYS_PER_FILE;
cfg.maxValue = TSDB_MAX_DAYS_PER_FILE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "keep";
cfg.ptr = &tsDaysToKeep;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_KEEP;
cfg.maxValue = TSDB_MAX_KEEP;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "minRows";
cfg.ptr = &tsMinRowsInFileBlock;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK;
cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxRows";
cfg.ptr = &tsMaxRowsInFileBlock;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK;
cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "precision";
cfg.ptr = &tsTimePrecision;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_PRECISION;
cfg.maxValue = TSDB_MAX_PRECISION;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "comp";
cfg.ptr = &tsCompression;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_COMP_LEVEL;
cfg.maxValue = TSDB_MAX_COMP_LEVEL;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "walLevel";
cfg.ptr = &tsWAL;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_WAL_LEVEL;
cfg.maxValue = TSDB_MAX_WAL_LEVEL;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "fsync";
cfg.ptr = &tsFsyncPeriod;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_FSYNC_PERIOD;
cfg.maxValue = TSDB_MAX_FSYNC_PERIOD;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "replica";
cfg.ptr = &tsReplications;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION;
cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "quorum";
cfg.ptr = &tsQuorum;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_QUORUM_OPTION;
cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "update";
cfg.ptr = &tsUpdate;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_UPDATE;
cfg.maxValue = TSDB_MAX_DB_UPDATE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "cachelast";
cfg.ptr = &tsCacheLastRow;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_CACHE_LAST_ROW;
cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "mqttHostName";
cfg.ptr = tsMqttHostName;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "mqttPort";
cfg.ptr = tsMqttPort;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_MQTT_PORT_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "mqttTopic";
cfg.ptr = tsMqttTopic;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_MQTT_TOPIC_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "compressMsgSize"; cfg.option = "compressMsgSize";
cfg.ptr = &tsCompressMsgSize; cfg.ptr = &tsCompressMsgSize;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1085,76 +772,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "adjustMaster";
cfg.ptr = &tsEnableAdjustMaster;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "mqtt";
cfg.ptr = &tsEnableMqttModule;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "monitor";
cfg.ptr = &tsEnableMonitorModule;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "stream";
cfg.ptr = &tsEnableStream;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "topicBianryLen";
cfg.ptr = &tsTopicBianryLen;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 16;
cfg.maxValue = 16000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "telegrafUseFieldNum";
cfg.ptr = &tsTelegrafUseFieldNum;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "restfulRowLimit";
cfg.ptr = &tsRestRowLimit;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 1;
cfg.maxValue = 10000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// debug flag // debug flag
cfg.option = "numOfLogLines"; cfg.option = "numOfLogLines";
cfg.ptr = &tsNumOfLogLines; cfg.ptr = &tsNumOfLogLines;
@ -1236,17 +853,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "sdbDebugFlag";
cfg.ptr = &sdbDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "rpcDebugFlag"; cfg.option = "rpcDebugFlag";
cfg.ptr = &rpcDebugFlag; cfg.ptr = &rpcDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1307,36 +913,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
cfg.option = "httpDebugFlag";
cfg.ptr = &httpDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "mqttDebugFlag";
cfg.ptr = &mqttDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "monDebugFlag";
cfg.ptr = &monDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "qDebugFlag"; cfg.option = "qDebugFlag";
cfg.ptr = &qDebugFlag; cfg.ptr = &qDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1456,16 +1032,6 @@ static void doInitGlobalConfig(void) {
cfg.maxValue = 0; cfg.maxValue = 0;
cfg.ptrLength = PATH_MAX; cfg.ptrLength = PATH_MAX;
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "tsdbMetaCompactRatio";
cfg.ptr = &tsTsdbMetaCompactRatio;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 100;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg); taosAddConfigOption(cfg);
// enable kill long query // enable kill long query

4
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -72,7 +72,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_STB_VGROUP] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg;
@ -368,4 +368,4 @@ void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) {
SEpSet epSet = {0}; SEpSet epSet = {0};
dndGetMnodeEpSet(pDnode, &epSet); dndGetMnodeEpSet(pDnode, &epSet);
dndSendMsgToDnode(pDnode, &epSet, pMsg); dndSendMsgToDnode(pDnode, &epSet, pMsg);
} }

15
source/dnode/mgmt/impl/test/CMakeLists.txt
View File

@ -1,6 +1,19 @@
# add_subdirectory(acct)
add_subdirectory(acct)
# add_subdirectory(auth)
# add_subdirectory(balance)
# add_subdirectory(cluster) # add_subdirectory(cluster)
add_subdirectory(db)
add_subdirectory(dnode) add_subdirectory(dnode)
# add_subdirectory(func)
# add_subdirectory(mnode)
# add_subdirectory(profile) # add_subdirectory(profile)
# add_subdirectory(show) # add_subdirectory(show)
# add_subdirectory(stb)
# add_subdirectory(sync)
# add_subdirectory(telem)
# add_subdirectory(trans)
add_subdirectory(user) add_subdirectory(user)
# add_subdirectory(vgroup)
# add_subdirectory(common)

12
source/dnode/mgmt/impl/test/acct/CMakeLists.txt
View File

@ -1,20 +1,20 @@
add_executable(dndTestAcct "") add_executable(dnode_test_acct "")
target_sources(dndTestAcct target_sources(dnode_test_acct
PRIVATE PRIVATE
"acct.cpp" "acct.cpp"
"../sut/deploy.cpp" "../sut/deploy.cpp"
) )
target_link_libraries( target_link_libraries(
dndTestAcct dnode_test_acct
PUBLIC dnode PUBLIC dnode
PUBLIC util PUBLIC util
PUBLIC os PUBLIC os
PUBLIC gtest_main PUBLIC gtest_main
) )
target_include_directories(dndTestAcct target_include_directories(dnode_test_acct
PUBLIC PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
"${CMAKE_CURRENT_SOURCE_DIR}/../../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
@ -24,6 +24,6 @@ target_include_directories(dndTestAcct
enable_testing() enable_testing()
add_test( add_test(
NAME dndTestAcct NAME dnode_test_acct
COMMAND dndTestAcct COMMAND dnode_test_acct
) )

23
source/dnode/mgmt/impl/test/acct/acct.cpp
View File

@ -17,24 +17,27 @@
class DndTestAcct : public ::testing::Test { class DndTestAcct : public ::testing::Test {
protected: protected:
void SetUp() override {} static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
void TearDown() override {} SServer* pServer = createServer(path, fqdn, port, firstEp);
ASSERT(pServer);
return pServer;
}
static void SetUpTestSuite() { static void SetUpTestSuite() {
const char* user = "root"; initLog("/tmp/tdlog");
const char* pass = "taosdata";
const char* path = "/tmp/dndTestAcct";
const char* fqdn = "localhost";
uint16_t port = 9520;
pServer = createServer(path, fqdn, port); const char* fqdn = "localhost";
ASSERT(pServer); const char* firstEp = "localhost:9012";
pClient = createClient(user, pass, fqdn, port); pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9012, firstEp);
pClient = createClient("root", "taosdata", fqdn, 9012);
taosMsleep(300);
} }
static void TearDownTestSuite() { static void TearDownTestSuite() {
stopServer(pServer); stopServer(pServer);
dropClient(pClient); dropClient(pClient);
pServer = NULL;
pClient = NULL;
} }
static SServer* pServer; static SServer* pServer;

29
source/dnode/mgmt/impl/test/db/CMakeLists.txt Normal file
View File

@ -0,0 +1,29 @@
add_executable(dnode_test_db "")
target_sources(dnode_test_db
PRIVATE
"db.cpp"
"../sut/deploy.cpp"
)
target_link_libraries(
dnode_test_db
PUBLIC dnode
PUBLIC util
PUBLIC os
PUBLIC gtest_main
)
target_include_directories(dnode_test_db
PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
"${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
"${CMAKE_CURRENT_SOURCE_DIR}/../sut"
)
enable_testing()
add_test(
NAME dnode_test_db
COMMAND dnode_test_db
)

355
source/dnode/mgmt/impl/test/db/db.cpp Normal file
View File

@ -0,0 +1,355 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"
class DndTestDb : public ::testing::Test {
protected:
static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
SServer* pServer = createServer(path, fqdn, port, firstEp);
ASSERT(pServer);
return pServer;
}
static void SetUpTestSuite() {
initLog("/tmp/tdlog");
const char* fqdn = "localhost";
const char* firstEp = "localhost:9040";
pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp);
pClient = createClient("root", "taosdata", fqdn, 9040);
taosMsleep(300);
}
static void TearDownTestSuite() {
stopServer(pServer);
dropClient(pClient);
pServer = NULL;
pClient = NULL;
}
static SServer* pServer;
static SClient* pClient;
static int32_t connId;
public:
void SetUp() override {}
void TearDown() override {}
void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) {
SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg));
pShow->type = showType;
strcpy(pShow->db, "");
SRpcMsg showRpcMsg = {0};
showRpcMsg.pCont = pShow;
showRpcMsg.contLen = sizeof(SShowMsg);
showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW;
sendMsg(pClient, &showRpcMsg);
ASSERT_NE(pClient->pRsp, nullptr);
ASSERT_EQ(pClient->pRsp->code, 0);
ASSERT_NE(pClient->pRsp->pCont, nullptr);
SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont;
ASSERT_NE(pShowRsp, nullptr);
pShowRsp->showId = htonl(pShowRsp->showId);
pMeta = &pShowRsp->tableMeta;
pMeta->numOfTags = htons(pMeta->numOfTags);
pMeta->numOfColumns = htons(pMeta->numOfColumns);
pMeta->sversion = htons(pMeta->sversion);
pMeta->tversion = htons(pMeta->tversion);
pMeta->tuid = htobe64(pMeta->tuid);
pMeta->suid = htobe64(pMeta->suid);
showId = pShowRsp->showId;
EXPECT_NE(pShowRsp->showId, 0);
EXPECT_STREQ(pMeta->tbFname, showName);
EXPECT_EQ(pMeta->numOfTags, 0);
EXPECT_EQ(pMeta->numOfColumns, columns);
EXPECT_EQ(pMeta->precision, 0);
EXPECT_EQ(pMeta->tableType, 0);
EXPECT_EQ(pMeta->update, 0);
EXPECT_EQ(pMeta->sversion, 0);
EXPECT_EQ(pMeta->tversion, 0);
EXPECT_EQ(pMeta->tuid, 0);
EXPECT_EQ(pMeta->suid, 0);
}
void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) {
SSchema* pSchema = &pMeta->pSchema[index];
pSchema->bytes = htons(pSchema->bytes);
EXPECT_EQ(pSchema->colId, 0);
EXPECT_EQ(pSchema->type, type);
EXPECT_EQ(pSchema->bytes, bytes);
EXPECT_STREQ(pSchema->name, name);
}
void SendThenCheckShowRetrieveMsg(int32_t rows) {
SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg));
pRetrieve->showId = htonl(showId);
pRetrieve->free = 0;
SRpcMsg retrieveRpcMsg = {0};
retrieveRpcMsg.pCont = pRetrieve;
retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg);
retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE;
sendMsg(pClient, &retrieveRpcMsg);
ASSERT_NE(pClient->pRsp, nullptr);
ASSERT_EQ(pClient->pRsp->code, 0);
ASSERT_NE(pClient->pRsp->pCont, nullptr);
pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont;
ASSERT_NE(pRetrieveRsp, nullptr);
pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows);
pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset);
pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds);
pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen);
EXPECT_EQ(pRetrieveRsp->numOfRows, rows);
EXPECT_EQ(pRetrieveRsp->offset, 0);
EXPECT_EQ(pRetrieveRsp->useconds, 0);
// EXPECT_EQ(pRetrieveRsp->completed, completed);
EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI);
EXPECT_EQ(pRetrieveRsp->compressed, 0);
EXPECT_EQ(pRetrieveRsp->reserved, 0);
EXPECT_EQ(pRetrieveRsp->compLen, 0);
pData = pRetrieveRsp->data;
pos = 0;
}
void CheckInt8(int8_t val) {
int8_t data = *((int8_t*)(pData + pos));
pos += sizeof(int8_t);
EXPECT_EQ(data, val);
}
void CheckInt16(int16_t val) {
int16_t data = *((int16_t*)(pData + pos));
pos += sizeof(int16_t);
EXPECT_EQ(data, val);
}
void CheckInt32(int32_t val) {
int32_t data = *((int32_t*)(pData + pos));
pos += sizeof(int32_t);
EXPECT_EQ(data, val);
}
void CheckInt64(int64_t val) {
int64_t data = *((int64_t*)(pData + pos));
pos += sizeof(int64_t);
EXPECT_EQ(data, val);
}
void CheckTimestamp() {
int64_t data = *((int64_t*)(pData + pos));
pos += sizeof(int64_t);
EXPECT_GT(data, 0);
}
void CheckBinary(const char* val, int32_t len) {
pos += sizeof(VarDataLenT);
char* data = (char*)(pData + pos);
pos += len;
EXPECT_STREQ(data, val);
}
int32_t showId;
STableMetaMsg* pMeta;
SRetrieveTableRsp* pRetrieveRsp;
char* pData;
int32_t pos;
};
SServer* DndTestDb::pServer;
SClient* DndTestDb::pClient;
int32_t DndTestDb::connId;
TEST_F(DndTestDb, 01_ShowDb) {
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name");
CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time");
CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "replica");
CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "quorum");
CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "days");
CheckSchema(5, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2");
CheckSchema(6, TSDB_DATA_TYPE_INT, 4, "cache(MB)");
CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "blocks");
CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "minrows");
CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "maxrows");
CheckSchema(10, TSDB_DATA_TYPE_TINYINT, 1, "wallevel");
CheckSchema(11, TSDB_DATA_TYPE_INT, 4, "fsync");
CheckSchema(12, TSDB_DATA_TYPE_TINYINT, 1, "comp");
CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "cachelast");
CheckSchema(14, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision");
CheckSchema(15, TSDB_DATA_TYPE_TINYINT, 1, "update");
SendThenCheckShowRetrieveMsg(0);
}
TEST_F(DndTestDb, 02_CreateDb) {
{
SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg));
strcpy(pReq->db, "1.d1");
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replications = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->ignoreExist = 1;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateDbMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
CheckTimestamp();
CheckInt16(1); // replica
CheckInt16(1); // quorum
CheckInt16(10); // days
CheckBinary("3650,3650,3650", 24); // days
CheckInt32(16); // cache
CheckInt32(10); // blocks
CheckInt32(100); // minrows
CheckInt32(4096); // maxrows
CheckInt8(1); // wallevel
CheckInt32(3000); // fsync
CheckInt8(2); // comp
CheckInt8(0); // cachelast
CheckBinary("ms", 3); // precision
CheckInt8(0); // update
}
TEST_F(DndTestDb, 03_AlterDb) {
{
SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg));
strcpy(pReq->db, "1.d1");
pReq->totalBlocks = htonl(12);
pReq->daysToKeep0 = htonl(300);
pReq->daysToKeep1 = htonl(400);
pReq->daysToKeep2 = htonl(500);
pReq->fsyncPeriod = htonl(4000);
pReq->walLevel = 2;
pReq->quorum = 2;
pReq->cacheLastRow = 1;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SAlterDbMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
CheckTimestamp();
CheckInt16(1); // replica
CheckInt16(2); // quorum
CheckInt16(10); // days
CheckBinary("300,400,500", 24); // days
CheckInt32(16); // cache
CheckInt32(12); // blocks
CheckInt32(100); // minrows
CheckInt32(4096); // maxrows
CheckInt8(2); // wallevel
CheckInt32(4000); // fsync
CheckInt8(2); // comp
CheckInt8(1); // cachelast
CheckBinary("ms", 3); // precision
CheckInt8(0); // update
}
TEST_F(DndTestDb, 04_RestartDnode) {
stopServer(pServer);
pServer = NULL;
uInfo("start all server");
const char* fqdn = "localhost";
const char* firstEp = "localhost:9040";
pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp);
uInfo("all server is running");
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
CheckTimestamp();
CheckInt16(1); // replica
CheckInt16(2); // quorum
CheckInt16(10); // days
CheckBinary("300,400,500", 24); // days
CheckInt32(16); // cache
CheckInt32(12); // blocks
CheckInt32(100); // minrows
CheckInt32(4096); // maxrows
CheckInt8(2); // wallevel
CheckInt32(4000); // fsync
CheckInt8(2); // comp
CheckInt8(1); // cachelast
CheckBinary("ms", 3); // precision
CheckInt8(0); // update
}
TEST_F(DndTestDb, 05_DropDb) {
{
SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg));
strcpy(pReq->db, "1.d1");
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropDbMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendThenCheckShowRetrieveMsg(0);
}

12
source/dnode/mgmt/impl/test/dnode/CMakeLists.txt
View File

@ -1,20 +1,20 @@
add_executable(dndTestDnode "") add_executable(dnode_test_dnode "")
target_sources(dndTestDnode target_sources(dnode_test_dnode
PRIVATE PRIVATE
"dnode.cpp" "dnode.cpp"
"../sut/deploy.cpp" "../sut/deploy.cpp"
) )
target_link_libraries( target_link_libraries(
dndTestDnode dnode_test_dnode
PUBLIC dnode PUBLIC dnode
PUBLIC util PUBLIC util
PUBLIC os PUBLIC os
PUBLIC gtest_main PUBLIC gtest_main
) )
target_include_directories(dndTestDnode target_include_directories(dnode_test_dnode
PUBLIC PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
"${CMAKE_CURRENT_SOURCE_DIR}/../../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
@ -24,6 +24,6 @@ target_include_directories(dndTestDnode
enable_testing() enable_testing()
add_test( add_test(
NAME dndTestDnode NAME dnode_test_dnode
COMMAND dndTestDnode COMMAND dnode_test_dnode
) )

124
source/dnode/mgmt/impl/test/dnode/dnode.cpp
View File

@ -24,16 +24,16 @@ class DndTestDnode : public ::testing::Test {
} }
static void SetUpTestSuite() { static void SetUpTestSuite() {
initLog("/tmp/dndTestDnode"); initLog("/tmp/tdlog");
const char* fqdn = "localhost"; const char* fqdn = "localhost";
const char* firstEp = "localhost:9521"; const char* firstEp = "localhost:9041";
pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp);
pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp);
pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp);
pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp);
pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp);
pClient = createClient("root", "taosdata", fqdn, 9521); pClient = createClient("root", "taosdata", fqdn, 9041);
taosMsleep(300); taosMsleep(300);
} }
@ -188,7 +188,7 @@ SServer* DndTestDnode::pServer4;
SServer* DndTestDnode::pServer5; SServer* DndTestDnode::pServer5;
SClient* DndTestDnode::pClient; SClient* DndTestDnode::pClient;
TEST_F(DndTestDnode, ShowDnode) { TEST_F(DndTestDnode, 01_ShowDnode) {
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id");
CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point");
@ -200,7 +200,7 @@ TEST_F(DndTestDnode, ShowDnode) {
SendThenCheckShowRetrieveMsg(1); SendThenCheckShowRetrieveMsg(1);
CheckInt16(1); CheckInt16(1);
CheckBinary("localhost:9521", TSDB_EP_LEN); CheckBinary("localhost:9041", TSDB_EP_LEN);
CheckInt16(0); CheckInt16(0);
CheckInt16(1); CheckInt16(1);
CheckBinary("ready", 10); CheckBinary("ready", 10);
@ -208,7 +208,7 @@ TEST_F(DndTestDnode, ShowDnode) {
CheckBinary("", 24); CheckBinary("", 24);
} }
TEST_F(DndTestDnode, ConfigDnode_01) { TEST_F(DndTestDnode, 02_ConfigDnode) {
SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg));
pReq->dnodeId = htonl(1); pReq->dnodeId = htonl(1);
strcpy(pReq->config, "ddebugflag 131"); strcpy(pReq->config, "ddebugflag 131");
@ -224,9 +224,9 @@ TEST_F(DndTestDnode, ConfigDnode_01) {
ASSERT_EQ(pMsg->code, 0); ASSERT_EQ(pMsg->code, 0);
} }
TEST_F(DndTestDnode, CreateDnode_01) { TEST_F(DndTestDnode, 03_CreateDnode) {
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9522"); strcpy(pReq->ep, "localhost:9042");
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq; rpcMsg.pCont = pReq;
@ -243,8 +243,8 @@ TEST_F(DndTestDnode, CreateDnode_01) {
SendThenCheckShowRetrieveMsg(2); SendThenCheckShowRetrieveMsg(2);
CheckInt16(1); CheckInt16(1);
CheckInt16(2); CheckInt16(2);
CheckBinary("localhost:9521", TSDB_EP_LEN); CheckBinary("localhost:9041", TSDB_EP_LEN);
CheckBinary("localhost:9522", TSDB_EP_LEN); CheckBinary("localhost:9042", TSDB_EP_LEN);
CheckInt16(0); CheckInt16(0);
CheckInt16(0); CheckInt16(0);
CheckInt16(1); CheckInt16(1);
@ -257,7 +257,7 @@ TEST_F(DndTestDnode, CreateDnode_01) {
CheckBinary("", 24); CheckBinary("", 24);
} }
TEST_F(DndTestDnode, DropDnode_01) { TEST_F(DndTestDnode, 04_DropDnode) {
SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg));
pReq->dnodeId = htonl(2); pReq->dnodeId = htonl(2);
@ -274,7 +274,7 @@ TEST_F(DndTestDnode, DropDnode_01) {
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
SendThenCheckShowRetrieveMsg(1); SendThenCheckShowRetrieveMsg(1);
CheckInt16(1); CheckInt16(1);
CheckBinary("localhost:9521", TSDB_EP_LEN); CheckBinary("localhost:9041", TSDB_EP_LEN);
CheckInt16(0); CheckInt16(0);
CheckInt16(1); CheckInt16(1);
CheckBinary("ready", 10); CheckBinary("ready", 10);
@ -282,10 +282,10 @@ TEST_F(DndTestDnode, DropDnode_01) {
CheckBinary("", 24); CheckBinary("", 24);
} }
TEST_F(DndTestDnode, CreateDnode_02) { TEST_F(DndTestDnode, 05_CreateDnode) {
{ {
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9523"); strcpy(pReq->ep, "localhost:9043");
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq; rpcMsg.pCont = pReq;
@ -300,7 +300,7 @@ TEST_F(DndTestDnode, CreateDnode_02) {
{ {
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9524"); strcpy(pReq->ep, "localhost:9044");
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq; rpcMsg.pCont = pReq;
@ -315,7 +315,7 @@ TEST_F(DndTestDnode, CreateDnode_02) {
{ {
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9525"); strcpy(pReq->ep, "localhost:9045");
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq; rpcMsg.pCont = pReq;
@ -335,10 +335,10 @@ TEST_F(DndTestDnode, CreateDnode_02) {
CheckInt16(3); CheckInt16(3);
CheckInt16(4); CheckInt16(4);
CheckInt16(5); CheckInt16(5);
CheckBinary("localhost:9521", TSDB_EP_LEN); CheckBinary("localhost:9041", TSDB_EP_LEN);
CheckBinary("localhost:9523", TSDB_EP_LEN); CheckBinary("localhost:9043", TSDB_EP_LEN);
CheckBinary("localhost:9524", TSDB_EP_LEN); CheckBinary("localhost:9044", TSDB_EP_LEN);
CheckBinary("localhost:9525", TSDB_EP_LEN); CheckBinary("localhost:9045", TSDB_EP_LEN);
CheckInt16(0); CheckInt16(0);
CheckInt16(0); CheckInt16(0);
CheckInt16(0); CheckInt16(0);
@ -361,7 +361,7 @@ TEST_F(DndTestDnode, CreateDnode_02) {
CheckBinary("", 24); CheckBinary("", 24);
} }
TEST_F(DndTestDnode, RestartDnode_01) { TEST_F(DndTestDnode, 06_RestartDnode) {
uInfo("stop all server"); uInfo("stop all server");
stopServer(pServer1); stopServer(pServer1);
stopServer(pServer2); stopServer(pServer2);
@ -377,43 +377,43 @@ TEST_F(DndTestDnode, RestartDnode_01) {
uInfo("start all server"); uInfo("start all server");
const char* fqdn = "localhost"; const char* fqdn = "localhost";
const char* firstEp = "localhost:9521"; const char* firstEp = "localhost:9041";
pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp);
// pServer1 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp);
// pServer1 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp);
// pServer1 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp);
uInfo("all server is running"); uInfo("all server is running");
// taosMsleep(1300); taosMsleep(1300);
// SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
// SendThenCheckShowRetrieveMsg(4); SendThenCheckShowRetrieveMsg(4);
// CheckInt16(1); CheckInt16(1);
// CheckInt16(3); CheckInt16(3);
// CheckInt16(4); CheckInt16(4);
// CheckInt16(5); CheckInt16(5);
// CheckBinary("localhost:9521", TSDB_EP_LEN); CheckBinary("localhost:9041", TSDB_EP_LEN);
// CheckBinary("localhost:9523", TSDB_EP_LEN); CheckBinary("localhost:9043", TSDB_EP_LEN);
// CheckBinary("localhost:9524", TSDB_EP_LEN); CheckBinary("localhost:9044", TSDB_EP_LEN);
// CheckBinary("localhost:9525", TSDB_EP_LEN); CheckBinary("localhost:9045", TSDB_EP_LEN);
// CheckInt16(0); CheckInt16(0);
// CheckInt16(0); CheckInt16(0);
// CheckInt16(0); CheckInt16(0);
// CheckInt16(0); CheckInt16(0);
// CheckInt16(1); CheckInt16(1);
// CheckInt16(1); CheckInt16(1);
// CheckInt16(1); CheckInt16(1);
// CheckInt16(1); CheckInt16(1);
// CheckBinary("ready", 10); CheckBinary("ready", 10);
// CheckBinary("ready", 10); CheckBinary("ready", 10);
// CheckBinary("ready", 10); CheckBinary("ready", 10);
// CheckBinary("ready", 10); CheckBinary("ready", 10);
// CheckTimestamp(); CheckTimestamp();
// CheckTimestamp(); CheckTimestamp();
// CheckTimestamp(); CheckTimestamp();
// CheckTimestamp(); CheckTimestamp();
// CheckBinary("", 24); CheckBinary("", 24);
// CheckBinary("", 24); CheckBinary("", 24);
// CheckBinary("", 24); CheckBinary("", 24);
// CheckBinary("", 24); CheckBinary("", 24);
} }

6
source/dnode/mgmt/impl/test/sut/deploy.cpp
View File

@ -18,14 +18,10 @@
void initLog(const char* path) { void initLog(const char* path) {
dDebugFlag = 143; dDebugFlag = 143;
vDebugFlag = 0; vDebugFlag = 0;
mDebugFlag = 143; mDebugFlag = 207;
cDebugFlag = 0; cDebugFlag = 0;
jniDebugFlag = 0; jniDebugFlag = 0;
tmrDebugFlag = 0; tmrDebugFlag = 0;
sdbDebugFlag = 0;
httpDebugFlag = 0;
mqttDebugFlag = 0;
monDebugFlag = 0;
uDebugFlag = 143; uDebugFlag = 143;
rpcDebugFlag = 0; rpcDebugFlag = 0;
odbcDebugFlag = 0; odbcDebugFlag = 0;

12
source/dnode/mgmt/impl/test/user/CMakeLists.txt
View File

@ -1,20 +1,20 @@
add_executable(dndTestUser "") add_executable(dnode_test_user "")
target_sources(dndTestUser target_sources(dnode_test_user
PRIVATE PRIVATE
"user.cpp" "user.cpp"
"../sut/deploy.cpp" "../sut/deploy.cpp"
) )
target_link_libraries( target_link_libraries(
dndTestUser dnode_test_user
PUBLIC dnode PUBLIC dnode
PUBLIC util PUBLIC util
PUBLIC os PUBLIC os
PUBLIC gtest_main PUBLIC gtest_main
) )
target_include_directories(dndTestUser target_include_directories(dnode_test_user
PUBLIC PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
"${CMAKE_CURRENT_SOURCE_DIR}/../../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
@ -24,6 +24,6 @@ target_include_directories(dndTestUser
enable_testing() enable_testing()
add_test( add_test(
NAME dndTestUser NAME dnode_test_user
COMMAND dndTestUser COMMAND dnode_test_user
) )

22
source/dnode/mgmt/impl/test/user/user.cpp
View File

@ -24,12 +24,12 @@ class DndTestUser : public ::testing::Test {
} }
static void SetUpTestSuite() { static void SetUpTestSuite() {
initLog("/tmp/dndTestUser"); initLog("/tmp/tdlog");
const char* fqdn = "localhost"; const char* fqdn = "localhost";
const char* firstEp = "localhost:9530"; const char* firstEp = "localhost:9140";
pServer = CreateServer("/tmp/dndTestUser", fqdn, 9530, firstEp); pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9140, firstEp);
pClient = createClient("root", "taosdata", fqdn, 9530); pClient = createClient("root", "taosdata", fqdn, 9140);
taosMsleep(300); taosMsleep(300);
} }
@ -170,7 +170,7 @@ SServer* DndTestUser::pServer;
SClient* DndTestUser::pClient; SClient* DndTestUser::pClient;
int32_t DndTestUser::connId; int32_t DndTestUser::connId;
TEST_F(DndTestUser, ShowUser) { TEST_F(DndTestUser, 01_ShowUser) {
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4);
CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name");
CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege");
@ -184,7 +184,7 @@ TEST_F(DndTestUser, ShowUser) {
CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN);
} }
TEST_F(DndTestUser, CreateUser_01) { TEST_F(DndTestUser, 02_CreateUser) {
{ {
SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg));
strcpy(pReq->user, "u1"); strcpy(pReq->user, "u1");
@ -233,7 +233,7 @@ TEST_F(DndTestUser, CreateUser_01) {
CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN);
} }
TEST_F(DndTestUser, AlterUser_01) { TEST_F(DndTestUser, 03_AlterUser) {
SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg));
strcpy(pReq->user, "u1"); strcpy(pReq->user, "u1");
strcpy(pReq->pass, "p2"); strcpy(pReq->pass, "p2");
@ -264,7 +264,7 @@ TEST_F(DndTestUser, AlterUser_01) {
CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN);
} }
TEST_F(DndTestUser, DropUser_01) { TEST_F(DndTestUser, 04_DropUser) {
SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg));
strcpy(pReq->user, "u1"); strcpy(pReq->user, "u1");
@ -290,15 +290,15 @@ TEST_F(DndTestUser, DropUser_01) {
CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN);
} }
TEST_F(DndTestUser, RestartDnode) { TEST_F(DndTestUser, 05_RestartDnode) {
stopServer(pServer); stopServer(pServer);
pServer = NULL; pServer = NULL;
uInfo("start all server"); uInfo("start all server");
const char* fqdn = "localhost"; const char* fqdn = "localhost";
const char* firstEp = "localhost:9530"; const char* firstEp = "localhost:9140";
pServer = startServer("/tmp/dndTestUser", fqdn, 9530, firstEp); pServer = startServer("/tmp/dnode_test_user", fqdn, 9140, firstEp);
uInfo("all server is running"); uInfo("all server is running");

23
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -17,14 +17,16 @@
#define _TD_MND_DEF_H_ #define _TD_MND_DEF_H_
#include "os.h" #include "os.h"
#include "cJSON.h"
#include "sync.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "thash.h"
#include "tlog.h" #include "tlog.h"
#include "trpc.h" #include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
#include "thash.h"
#include "cJSON.h"
#include "mnode.h" #include "mnode.h"
#include "sync.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -41,11 +43,8 @@ extern int32_t mDebugFlag;
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
typedef struct SClusterObj SClusterObj; typedef struct SClusterObj SClusterObj;
typedef struct SDnodeObj SDnodeObj;
typedef struct SMnodeObj SMnodeObj; typedef struct SMnodeObj SMnodeObj;
typedef struct SAcctObj SAcctObj; typedef struct SAcctObj SAcctObj;
typedef struct SUserObj SUserObj;
typedef struct SDbObj SDbObj;
typedef struct SVgObj SVgObj; typedef struct SVgObj SVgObj;
typedef struct SFuncObj SFuncObj; typedef struct SFuncObj SFuncObj;
typedef struct SOperObj SOperObj; typedef struct SOperObj SOperObj;
@ -73,7 +72,8 @@ typedef enum {
TRN_STAGE_EXECUTE = 2, TRN_STAGE_EXECUTE = 2,
TRN_STAGE_COMMIT = 3, TRN_STAGE_COMMIT = 3,
TRN_STAGE_ROLLBACK = 4, TRN_STAGE_ROLLBACK = 4,
TRN_STAGE_RETRY = 5 TRN_STAGE_RETRY = 5,
TRN_STAGE_OVER = 6,
} ETrnStage; } ETrnStage;
typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy; typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy;
@ -103,7 +103,6 @@ typedef struct STrans {
int32_t id; int32_t id;
ETrnStage stage; ETrnStage stage;
ETrnPolicy policy; ETrnPolicy policy;
SMnode *pMnode;
void *rpcHandle; void *rpcHandle;
SArray *redoLogs; SArray *redoLogs;
SArray *undoLogs; SArray *undoLogs;
@ -119,7 +118,7 @@ typedef struct SClusterObj {
int64_t updateTime; int64_t updateTime;
} SClusterObj; } SClusterObj;
typedef struct SDnodeObj { typedef struct {
int32_t id; int32_t id;
int64_t createdTime; int64_t createdTime;
int64_t updateTime; int64_t updateTime;
@ -178,7 +177,7 @@ typedef struct SAcctObj {
SAcctInfo info; SAcctInfo info;
} SAcctObj; } SAcctObj;
typedef struct SUserObj { typedef struct {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_PASSWORD_LEN];
char acct[TSDB_USER_LEN]; char acct[TSDB_USER_LEN];
@ -209,12 +208,13 @@ typedef struct {
int8_t cacheLastRow; int8_t cacheLastRow;
} SDbCfg; } SDbCfg;
typedef struct SDbObj { typedef struct {
char name[TSDB_FULL_DB_NAME_LEN]; char name[TSDB_FULL_DB_NAME_LEN];
char acct[TSDB_USER_LEN]; char acct[TSDB_USER_LEN];
int64_t createdTime; int64_t createdTime;
int64_t updateTime; int64_t updateTime;
int64_t uid; int64_t uid;
int32_t version;
SDbCfg cfg; SDbCfg cfg;
} SDbObj; } SDbObj;
@ -304,6 +304,7 @@ typedef struct SMnodeMsg {
typedef struct { typedef struct {
int32_t id; int32_t id;
int32_t code;
void *rpcHandle; void *rpcHandle;
} STransMsg; } STransMsg;

2
source/dnode/mnode/impl/inc/mndShow.h
View File

@ -27,7 +27,7 @@ void mndCleanupShow(SMnode *pMnode);
void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp);
void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp);
void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp);
void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); void mndVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow);
char *mndShowStr(int32_t showType); char *mndShowStr(int32_t showType);
#ifdef __cplusplus #ifdef __cplusplus

2
source/dnode/mnode/impl/inc/mndSync.h
View File

@ -25,7 +25,7 @@ extern "C" {
int32_t mndInitSync(SMnode *pMnode); int32_t mndInitSync(SMnode *pMnode);
void mndCleanupSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode);
bool mndIsMaster(SMnode *pMnode); bool mndIsMaster(SMnode *pMnode);
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -32,10 +32,10 @@ int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg); int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg);
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg); int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
int32_t mndTransPrepare(STrans *pTrans);
void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code);
int32_t mndTransExecute(SSdb *pSdb, int32_t tranId); char *mndTransStageStr(ETrnStage stage);
char *mndTransPolicyStr(ETrnPolicy policy);
#ifdef __cplusplus #ifdef __cplusplus
} }

38
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -19,12 +19,12 @@
#define SDB_ACCT_VER 1 #define SDB_ACCT_VER 1
static int32_t mnodeCreateDefaultAcct(SMnode *pMnode); static int32_t mndCreateDefaultAcct(SMnode *pMnode);
static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct);
static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw);
static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct);
static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct);
static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct);
static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg);
static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg);
static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg);
@ -32,12 +32,12 @@ static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg);
int32_t mndInitAcct(SMnode *pMnode) { int32_t mndInitAcct(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_ACCT, SSdbTable table = {.sdbType = SDB_ACCT,
.keyType = SDB_KEY_BINARY, .keyType = SDB_KEY_BINARY,
.deployFp = mnodeCreateDefaultAcct, .deployFp = mndCreateDefaultAcct,
.encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, .encodeFp = (SdbEncodeFp)mndAcctActionEncode,
.decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, .decodeFp = (SdbDecodeFp)mndAcctActionDecode,
.insertFp = (SdbInsertFp)mnodeAcctActionInsert, .insertFp = (SdbInsertFp)mndAcctActionInsert,
.updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, .updateFp = (SdbUpdateFp)mndAcctActionUpdate,
.deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; .deleteFp = (SdbDeleteFp)mndAcctActionDelete};
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg);
@ -48,7 +48,7 @@ int32_t mndInitAcct(SMnode *pMnode) {
void mndCleanupAcct(SMnode *pMnode) {} void mndCleanupAcct(SMnode *pMnode) {}
static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
SAcctObj acctObj = {0}; SAcctObj acctObj = {0};
tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN);
acctObj.createdTime = taosGetTimestampMs(); acctObj.createdTime = taosGetTimestampMs();
@ -61,7 +61,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) {
.maxStorage = INT64_MAX, .maxStorage = INT64_MAX,
.accessState = TSDB_VN_ALL_ACCCESS}; .accessState = TSDB_VN_ALL_ACCCESS};
SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); SSdbRaw *pRaw = mndAcctActionEncode(&acctObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY); sdbSetRawStatus(pRaw, SDB_STATUS_READY);
@ -69,7 +69,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) {
return sdbWrite(pMnode->pSdb, pRaw); return sdbWrite(pMnode->pSdb, pRaw);
} }
static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj));
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
@ -90,7 +90,7 @@ static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) {
return pRaw; return pRaw;
} }
static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
@ -120,18 +120,18 @@ static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw) {
return pRow; return pRow;
} }
static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) {
mTrace("acct:%s, perform insert action", pAcct->acct); mTrace("acct:%s, perform insert action", pAcct->acct);
memset(&pAcct->info, 0, sizeof(SAcctInfo)); memset(&pAcct->info, 0, sizeof(SAcctInfo));
return 0; return 0;
} }
static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) {
mTrace("acct:%s, perform delete action", pAcct->acct); mTrace("acct:%s, perform delete action", pAcct->acct);
return 0; return 0;
} }
static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) {
mTrace("acct:%s, perform update action", pOldAcct->acct); mTrace("acct:%s, perform update action", pOldAcct->acct);
memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN);

2
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -203,7 +203,7 @@ static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data,
numOfRows++; numOfRows++;
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }

127
source/dnode/mnode/impl/src/mndDb.c
View File

@ -65,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) {
void mndCleanupDb(SMnode *pMnode) {} void mndCleanupDb(SMnode *pMnode) {}
static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -74,6 +74,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) SDB_SET_INT64(pRaw, dataPos, pDb->createdTime)
SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) SDB_SET_INT64(pRaw, dataPos, pDb->updateTime)
SDB_SET_INT64(pRaw, dataPos, pDb->uid) SDB_SET_INT64(pRaw, dataPos, pDb->uid)
SDB_SET_INT32(pRaw, dataPos, pDb->version)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile)
@ -117,6 +118,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile)
@ -151,7 +153,8 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) {
static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
mTrace("db:%s, perform update action", pOldDb->name); mTrace("db:%s, perform update action", pOldDb->name);
memcpy(pOldDb, pNewDb, sizeof(SDbObj)); pOldDb->updateTime = pNewDb->createdTime;
memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg));
return 0; return 0;
} }
@ -165,109 +168,146 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
sdbRelease(pSdb, pDb); sdbRelease(pSdb, pDb);
} }
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) {
char *pos = strstr(dbName, TS_PATH_DELIMITER);
if (pos == NULL) {
terrno = TSDB_CODE_MND_INVALID_DB;
return -1;
}
int32_t acctId = atoi(dbName);
if (acctId != pUser->acctId) {
terrno = TSDB_CODE_MND_INVALID_DB_ACCT;
return -1;
}
return 0;
}
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg, char *errMsg, int32_t len) {
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) {
terrno = TSDB_CODE_MND_INVALID_DB_CACHE_SIZE; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database cache block size option", len);
return -1; return -1;
} }
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) {
terrno = TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database total blocks option", len);
return -1; return -1;
} }
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
terrno = TSDB_CODE_MND_INVALID_DB_DAYS; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database days option", len);
return -1; return -1;
} }
if (pCfg->daysToKeep0 < pCfg->daysPerFile) { if (pCfg->daysToKeep0 < pCfg->daysPerFile) {
terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database days option", len);
return -1; return -1;
} }
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) {
terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database keep0 option", len);
return -1; return -1;
} }
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) {
terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database keep1 option", len);
return -1; return -1;
} }
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) {
terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database keep2 option", len);
return -1; return -1;
} }
if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database minrows option", len);
return -1; return -1;
} }
if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
terrno = TSDB_CODE_MND_INVALID_DB_MAX_ROWS; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database maxrows option", len);
return -1; return -1;
} }
if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) {
terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database minrows option", len);
return -1; return -1;
} }
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) {
terrno = TSDB_CODE_MND_INVALID_DB_COMMIT_TIME; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database commit option", len);
return -1; return -1;
} }
if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) {
terrno = TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database fsync option", len);
return -1; return -1;
} }
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) {
terrno = TSDB_CODE_MND_INVALID_DB_WAL_LEVEL; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database wal level option", len);
return -1; return -1;
} }
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) {
terrno = TSDB_CODE_MND_INVALID_DB_PRECISION; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid precision option", len);
return -1; return -1;
} }
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) {
terrno = TSDB_CODE_MND_INVALID_DB_COMP; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database compression option", len);
return -1; return -1;
} }
if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) {
terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database replication option", len);
return -1; return -1;
} }
if (pCfg->replications > mndGetDnodeSize(pMnode)) { if (pCfg->replications > mndGetDnodeSize(pMnode)) {
terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database replication option", len);
return -1; return -1;
} }
if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) {
terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database quorum option", len);
return -1; return -1;
} }
if (pCfg->quorum > pCfg->replications) { if (pCfg->quorum > pCfg->replications) {
terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database quorum option", len);
return -1; return -1;
} }
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
terrno = TSDB_CODE_MND_INVALID_DB_UPDATE; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database update option", len);
return -1; return -1;
} }
if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) {
terrno = TSDB_CODE_MND_INVALID_DB_CACHE_LAST; terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database cachelast option", len);
return -1; return -1;
} }
@ -294,13 +334,13 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
} }
static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, char *acct) { static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) {
SDbObj dbObj = {0}; SDbObj dbObj = {0};
tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN);
tstrncpy(dbObj.acct, acct, TSDB_USER_LEN); tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN);
dbObj.createdTime = taosGetTimestampMs(); dbObj.createdTime = taosGetTimestampMs();
dbObj.updateTime = dbObj.createdTime; dbObj.updateTime = dbObj.createdTime;
dbObj.uid = 1234; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->totalBlocks, .totalBlocks = pCreate->totalBlocks,
.daysPerFile = pCreate->daysPerFile, .daysPerFile = pCreate->daysPerFile,
@ -321,7 +361,13 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
mndSetDefaultDbCfg(&dbObj.cfg); mndSetDefaultDbCfg(&dbObj.cfg);
if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { if (mndCheckDbName(dbObj.name, pUser) != 0) {
mError("db:%s, failed to create since %s", pCreate->db, terrstr());
return -1;
}
char errMsg[TSDB_ERROR_MSG_LEN] = {0};
if (mndCheckDbCfg(pMnode, &dbObj.cfg, errMsg, TSDB_ERROR_MSG_LEN) != 0) {
mError("db:%s, failed to create since %s", pCreate->db, terrstr()); mError("db:%s, failed to create since %s", pCreate->db, terrstr());
return -1; return -1;
} }
@ -357,7 +403,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -403,11 +449,10 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
return -1; return -1;
} }
int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser->acct); int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser);
mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pOperUser);
if (code != 0) { if (code != 0) {
terrno = code;
mError("db:%s, failed to create since %s", pCreate->db, terrstr()); mError("db:%s, failed to create since %s", pCreate->db, terrstr());
return -1; return -1;
} }
@ -415,7 +460,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
static int32_t mnodeSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) {
bool changed = false; bool changed = false;
if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) {
@ -491,7 +536,7 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO
} }
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -521,13 +566,14 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
SDbObj dbObj = {0}; SDbObj dbObj = {0};
memcpy(&dbObj, pDb, sizeof(SDbObj)); memcpy(&dbObj, pDb, sizeof(SDbObj));
int32_t code = mnodeSetDbCfgFromAlterDbMsg(&dbObj, pAlter); int32_t code = mndSetDbCfgFromAlterDbMsg(&dbObj, pAlter);
if (code != 0) { if (code != 0) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code));
return code; return code;
} }
dbObj.version++;
code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
@ -571,7 +617,7 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) {
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -603,7 +649,6 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) {
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
if (code != 0) { if (code != 0) {
terrno = code;
mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); mError("db:%s, failed to drop since %s", pDrop->db, terrstr());
return code; return code;
} }
@ -772,10 +817,14 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe
return 0; return 0;
} }
char *mnodeGetDbStr(char *src) { char *mnGetDbStr(char *src) {
char *pos = strstr(src, TS_PATH_DELIMITER); char *pos = strstr(src, TS_PATH_DELIMITER);
if (pos != NULL) ++pos; if (pos != NULL) ++pos;
if (pos == NULL) {
return src;
}
return pos; return pos;
} }
@ -794,7 +843,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
cols = 0; cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char *name = mnodeGetDbStr(pDb->name); char *name = mnGetDbStr(pDb->name);
if (name != NULL) { if (name != NULL) {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]);
} else { } else {
@ -887,7 +936,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
sdbRelease(pSdb, pDb); sdbRelease(pSdb, pDb);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;

53
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -19,7 +19,7 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndTrans.h" #include "mndTrans.h"
#include "ttime.h" #include "ttime.h"
#include "tutil.h" #include "tep.h"
#define TSDB_DNODE_VER 1 #define TSDB_DNODE_VER 1
#define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_DNODE_RESERVE_SIZE 64
@ -27,8 +27,6 @@
#define TSDB_CONIIG_VALUE_LEN 48 #define TSDB_CONIIG_VALUE_LEN 48
#define TSDB_CONFIG_NUMBER 8 #define TSDB_CONFIG_NUMBER 8
static int32_t id = 2;
static const char *offlineReason[] = { static const char *offlineReason[] = {
"", "",
"status msg timeout", "status msg timeout",
@ -110,7 +108,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
} }
static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -177,6 +175,7 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) {
static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) {
mTrace("dnode:%d, perform update action", pOldDnode->id); mTrace("dnode:%d, perform update action", pOldDnode->id);
pOldDnode->updateTime = pNewDnode->updateTime;
return 0; return 0;
} }
@ -389,7 +388,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) {
SDnodeObj dnodeObj = {0}; SDnodeObj dnodeObj = {0};
dnodeObj.id = id++; dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE);
dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.createdTime = taosGetTimestampMs();
dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.updateTime = dnodeObj.createdTime;
taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port);
@ -413,25 +412,9 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
SSdbRaw *pUndoRaw = mndDnodeActionEncode(&dnodeObj); if (mndTransPrepare(pMnode, pTrans) != 0) {
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -485,25 +468,9 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode)
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pUndoRaw = mndDnodeActionEncode(pDnode); if (mndTransPrepare(pMnode, pTrans) != 0) {
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -645,7 +612,7 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data,
cols++; cols++;
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }
@ -767,7 +734,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
sdbRelease(pSdb, pDnode); sdbRelease(pSdb, pDnode);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;

6
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -183,7 +183,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pC
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -226,7 +226,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) {
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -500,7 +500,7 @@ static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in
sdbRelease(pSdb, pFunc); sdbRelease(pSdb, pFunc);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }

6
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -238,7 +238,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateMnodeMsg *
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -313,7 +313,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pMnodeOb
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -457,7 +457,7 @@ static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
sdbRelease(pSdb, pMnodeObj); sdbRelease(pSdb, pMnodeObj);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;

4
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -252,7 +252,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) {
return 0; return 0;
} }
static int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) {
pConn->numOfQueries = 0; pConn->numOfQueries = 0;
pConn->numOfStreams = 0; pConn->numOfStreams = 0;
int32_t numOfQueries = htonl(pMsg->numOfQueries); int32_t numOfQueries = htonl(pMsg->numOfQueries);
@ -338,7 +338,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) {
return -1; return -1;
} }
mnodeSaveQueryStreamList(pConn, pReq); mndSaveQueryStreamList(pConn, pReq);
if (pConn->killed != 0) { if (pConn->killed != 0) {
pRsp->killConnection = 1; pRsp->killConnection = 1;
} }

4
source/dnode/mnode/impl/src/mndShow.c
View File

@ -296,6 +296,8 @@ char *mndShowStr(int32_t showType) {
return "show streamtables"; return "show streamtables";
case TSDB_MGMT_TABLE_TP: case TSDB_MGMT_TABLE_TP:
return "show topics"; return "show topics";
case TSDB_MGMT_TABLE_FUNCTION:
return "show functions";
default: default:
return "undefined"; return "undefined";
} }
@ -308,7 +310,7 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow) {
return false; return false;
} }
void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { void mndVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) {
if (rows < capacity) { if (rows < capacity) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows);

10
source/dnode/mnode/impl/src/mndStb.c
View File

@ -69,7 +69,7 @@ int32_t mndInitStb(SMnode *pMnode) {
void mndCleanupStb(SMnode *pMnode) {} void mndCleanupStb(SMnode *pMnode) {}
static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
@ -285,7 +285,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -433,7 +433,7 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) {
} }
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -605,7 +605,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM
return 0; return 0;
} }
static void mnodeExtractTableName(char *tableId, char *name) { static void mndExtractTableName(char *tableId, char *name) {
int pos = -1; int pos = -1;
int num = 0; int num = 0;
for (pos = 0; tableId[pos] != 0; ++pos) { for (pos = 0; tableId[pos] != 0; ++pos) {
@ -665,7 +665,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
} }
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
return numOfRows; return numOfRows;
} }

14
source/dnode/mnode/impl/src/mndSync.c
View File

@ -21,16 +21,16 @@
int32_t mndInitSync(SMnode *pMnode) { return 0; } int32_t mndInitSync(SMnode *pMnode) { return 0; }
void mndCleanupSync(SMnode *pMnode) {} void mndCleanupSync(SMnode *pMnode) {}
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg) { int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
int32_t code = 0; int32_t code = 0;
int32_t len = sdbGetRawTotalSize(pRaw); // int32_t len = sdbGetRawTotalSize(pRaw);
SSdbRaw *pReceived = calloc(1, len); // SSdbRaw *pReceived = calloc(1, len);
memcpy(pReceived, pRaw, len); // memcpy(pReceived, pRaw, len);
mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); // mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg);
mndTransApply(pMnode, pReceived, pMsg, code); // mndTransApply(pMnode, pReceived, code);
return 0; return code;
} }
bool mndIsMaster(SMnode *pMnode) { return true; } bool mndIsMaster(SMnode *pMnode) { return true; }

534
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -17,8 +17,9 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "mndSync.h" #include "mndSync.h"
#define SDB_TRANS_VER 1 #define TSDB_TRANS_VER 1
#define TRN_DEFAULT_ARRAY_SIZE 8 #define TSDB_TRN_ARRAY_SIZE 8
#define TSDB_TRN_RESERVE_SIZE 64
static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRaw *mndTransActionEncode(STrans *pTrans);
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
@ -26,6 +27,22 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans);
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle);
static void mndTransSendRpcRsp(STrans *pTrans, int32_t code);
static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw);
static void mndTransDropArray(SArray *pArray);
static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray);
static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
static void mndTransExecute(SMnode *pMnode, STrans *pTrans);
int32_t mndInitTrans(SMnode *pMnode) { int32_t mndInitTrans(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_TRANS, SSdbTable table = {.sdbType = SDB_TRANS,
.keyType = SDB_KEY_INT32, .keyType = SDB_KEY_INT32,
@ -41,7 +58,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
void mndCleanupTrans(SMnode *pMnode) {} void mndCleanupTrans(SMnode *pMnode) {}
static SSdbRaw *mndTransActionEncode(STrans *pTrans) { static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
int32_t rawDataLen = 16 * sizeof(int32_t); int32_t rawDataLen = 16 * sizeof(int32_t) + TSDB_TRN_RESERVE_SIZE;
int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs);
int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs);
int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs);
@ -63,7 +80,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
rawDataLen += sdbGetRawTotalSize(pTmp); rawDataLen += sdbGetRawTotalSize(pTmp);
} }
SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, SDB_TRANS_VER, rawDataLen); SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen);
if (pRaw == NULL) { if (pRaw == NULL) {
mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr());
return NULL; return NULL;
@ -71,7 +88,6 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, pTrans->id) SDB_SET_INT32(pRaw, dataPos, pTrans->id)
SDB_SET_INT8(pRaw, dataPos, pTrans->stage)
SDB_SET_INT8(pRaw, dataPos, pTrans->policy) SDB_SET_INT8(pRaw, dataPos, pTrans->policy)
SDB_SET_INT32(pRaw, dataPos, redoLogNum) SDB_SET_INT32(pRaw, dataPos, redoLogNum)
SDB_SET_INT32(pRaw, dataPos, undoLogNum) SDB_SET_INT32(pRaw, dataPos, undoLogNum)
@ -100,6 +116,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len)
} }
SDB_SET_RESERVE(pRaw, dataPos, TSDB_TRN_RESERVE_SIZE)
SDB_SET_DATALEN(pRaw, dataPos);
mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos); mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos);
return pRaw; return pRaw;
} }
@ -113,7 +131,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
return NULL; return NULL;
} }
if (sver != SDB_TRANS_VER) { if (sver != TSDB_TRANS_VER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr());
return NULL; return NULL;
@ -126,11 +144,11 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
return NULL; return NULL;
} }
pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
pTrans->redoActions == NULL || pTrans->undoActions == NULL) { pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
@ -147,7 +165,6 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_GET_INT32(pRaw, pRow, dataPos, &pTrans->id) SDB_GET_INT32(pRaw, pRow, dataPos, &pTrans->id)
SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->stage)
SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->policy) SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->policy)
SDB_GET_INT32(pRaw, pRow, dataPos, &redoLogNum) SDB_GET_INT32(pRaw, pRow, dataPos, &redoLogNum)
SDB_GET_INT32(pRaw, pRow, dataPos, &undoLogNum) SDB_GET_INT32(pRaw, pRow, dataPos, &undoLogNum)
@ -197,6 +214,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
} }
} }
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_TRN_RESERVE_SIZE)
TRANS_DECODE_OVER: TRANS_DECODE_OVER:
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, tstrerror(errno)); mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, tstrerror(errno));
@ -210,62 +229,65 @@ TRANS_DECODE_OVER:
} }
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
mTrace("trans:%d, perform insert action, stage:%d", pTrans->id, pTrans->stage); pTrans->stage = TRN_STAGE_PREPARE;
mTrace("trans:%d, perform insert action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage));
SArray *pArray = pTrans->redoLogs;
int32_t arraySize = taosArrayGetSize(pArray);
for (int32_t i = 0; i < arraySize; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i);
int32_t code = sdbWrite(pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr());
return code;
}
}
return 0; return 0;
} }
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
mTrace("trans:%d, perform delete action, stage:%d", pTrans->id, pTrans->stage); mTrace("trans:%d, perform delete action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage));
SArray *pArray = pTrans->undoLogs; mndTransDropArray(pTrans->redoLogs);
int32_t arraySize = taosArrayGetSize(pArray); mndTransDropArray(pTrans->undoLogs);
mndTransDropArray(pTrans->commitLogs);
for (int32_t i = 0; i < arraySize; ++i) { mndTransDropArray(pTrans->redoActions);
SSdbRaw *pRaw = taosArrayGetP(pArray, i); mndTransDropArray(pTrans->undoActions);
int32_t code = sdbWrite(pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr());
return code;
}
}
return 0; return 0;
} }
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) {
mTrace("trans:%d, perform update action, stage:%d", pOldTrans->id, pNewTrans->stage); mTrace("trans:%d, perform update action, stage:%s", pOldTrans->id, mndTransStageStr(pNewTrans->stage));
SArray *pArray = pOldTrans->commitLogs;
int32_t arraySize = taosArrayGetSize(pArray);
for (int32_t i = 0; i < arraySize; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i);
int32_t code = sdbWrite(pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write raw:%p to sdb since %s", pOldTrans->id, pRaw, terrstr());
return code;
}
}
pOldTrans->stage = pNewTrans->stage; pOldTrans->stage = pNewTrans->stage;
return 0; return 0;
} }
static int32_t trnGenerateTransId() { STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
static int32_t tmp = 0; SSdb *pSdb = pMnode->pSdb;
return ++tmp; return sdbAcquire(pSdb, SDB_TRANS, &transId);
}
void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pTrans);
}
char *mndTransStageStr(ETrnStage stage) {
switch (stage) {
case TRN_STAGE_PREPARE:
return "prepare";
case TRN_STAGE_EXECUTE:
return "execute";
case TRN_STAGE_COMMIT:
return "commit";
case TRN_STAGE_ROLLBACK:
return "rollback";
case TRN_STAGE_RETRY:
return "retry";
default:
return "undefined";
}
}
char *mndTransPolicyStr(ETrnPolicy policy) {
switch (policy) {
case TRN_POLICY_ROLLBACK:
return "prepare";
case TRN_POLICY_RETRY:
return "retry";
default:
return "undefined";
}
} }
STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
@ -276,16 +298,15 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
return NULL; return NULL;
} }
pTrans->id = trnGenerateTransId(); pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS);
pTrans->stage = TRN_STAGE_PREPARE; pTrans->stage = TRN_STAGE_PREPARE;
pTrans->policy = policy; pTrans->policy = policy;
pTrans->pMnode = pMnode;
pTrans->rpcHandle = rpcHandle; pTrans->rpcHandle = rpcHandle;
pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
pTrans->redoActions == NULL || pTrans->undoActions == NULL) { pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
@ -298,7 +319,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
return pTrans; return pTrans;
} }
static void trnDropArray(SArray *pArray) { static void mndTransDropArray(SArray *pArray) {
for (int32_t i = 0; i < pArray->size; ++i) { for (int32_t i = 0; i < pArray->size; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i); SSdbRaw *pRaw = taosArrayGetP(pArray, i);
tfree(pRaw); tfree(pRaw);
@ -308,17 +329,17 @@ static void trnDropArray(SArray *pArray) {
} }
void mndTransDrop(STrans *pTrans) { void mndTransDrop(STrans *pTrans) {
trnDropArray(pTrans->redoLogs); mndTransDropArray(pTrans->redoLogs);
trnDropArray(pTrans->undoLogs); mndTransDropArray(pTrans->undoLogs);
trnDropArray(pTrans->commitLogs); mndTransDropArray(pTrans->commitLogs);
trnDropArray(pTrans->redoActions); mndTransDropArray(pTrans->redoActions);
trnDropArray(pTrans->undoActions); mndTransDropArray(pTrans->undoActions);
mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans);
tfree(pTrans); tfree(pTrans);
} }
void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) {
pTrans->rpcHandle = rpcHandle; pTrans->rpcHandle = rpcHandle;
mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle); mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle);
} }
@ -340,19 +361,19 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) {
int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) {
int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw); int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw);
mTrace("trans:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code); mTrace("trans:%d, raw:%p append to redo logs, code:0x%x", pTrans->id, pRaw, code);
return code; return code;
} }
int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) {
int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw); int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw);
mTrace("trans:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code); mTrace("trans:%d, raw:%p append to undo logs, code:0x%x", pTrans->id, pRaw, code);
return code; return code;
} }
int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) {
int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw); int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw);
mTrace("trans:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code); mTrace("trans:%d, raw:%p append to commit logs, code:0x%x", pTrans->id, pRaw, code);
return code; return code;
} }
@ -368,7 +389,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
return code; return code;
} }
int32_t mndTransPrepare(STrans *pTrans) { int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
mDebug("trans:%d, prepare transaction", pTrans->id); mDebug("trans:%d, prepare transaction", pTrans->id);
SSdbRaw *pRaw = mndTransActionEncode(pTrans); SSdbRaw *pRaw = mndTransActionEncode(pTrans);
@ -376,180 +397,295 @@ int32_t mndTransPrepare(STrans *pTrans) {
mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr());
return -1; return -1;
} }
sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (sdbWriteNotFree(pTrans->pMnode->pSdb, pRaw) != 0) { mTrace("trans:%d, start sync", pTrans->id);
mError("trans:%d, failed to write trans since %s", pTrans->id, terrstr()); int32_t code = mndSyncPropose(pMnode, pRaw);
return -1; if (code != 0) {
}
STransMsg *pMsg = calloc(1, sizeof(STransMsg));
pMsg->id = pTrans->id;
pMsg->rpcHandle = pTrans->rpcHandle;
mDebug("trans:%d, start sync, RPC:%p pMsg:%p", pTrans->id, pTrans->rpcHandle, pMsg);
if (mndSyncPropose(pTrans->pMnode, pRaw, pMsg) != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
free(pMsg);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
return -1; return -1;
} }
sdbFreeRaw(pRaw); mTrace("trans:%d, sync finished", pTrans->id);
code = sdbWrite(pMnode->pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr());
return -1;
}
STrans *pNewTrans = mndAcquireTrans(pMnode, pTrans->id);
if (pNewTrans == NULL) {
mError("trans:%d, failed to ready from sdb since %s", pTrans->id, terrstr());
return -1;
}
mDebug("trans:%d, prepare finished", pNewTrans->id);
pNewTrans->rpcHandle = pTrans->rpcHandle;
mndTransExecute(pMnode, pNewTrans);
mndReleaseTrans(pMnode, pNewTrans);
return 0; return 0;
} }
static void trnSendRpcRsp(STransMsg *pMsg, int32_t code) { int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) {
mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x pMsg:%p", pMsg->id, pMsg->rpcHandle, code & 0xFFFF, pMsg); mDebug("trans:%d, commit transaction", pTrans->id);
if (pMsg->rpcHandle != NULL) {
SRpcMsg rspMsg = {.handle = pMsg->rpcHandle, .code = code}; SSdbRaw *pRaw = mndTransActionEncode(pTrans);
rpcSendResponse(&rspMsg); if (pRaw == NULL) {
mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr());
return -1;
}
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
if (taosArrayGetSize(pTrans->commitLogs) != 0) {
mTrace("trans:%d, start sync", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
sdbFreeRaw(pRaw);
return -1;
}
mTrace("trans:%d, sync finished", pTrans->id);
code = sdbWrite(pMnode->pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr());
return -1;
}
} }
free(pMsg); mDebug("trans:%d, commit finished", pTrans->id);
return 0;
}
int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
mDebug("trans:%d, rollback transaction", pTrans->id);
SSdbRaw *pRaw = mndTransActionEncode(pTrans);
if (pRaw == NULL) {
mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr());
return -1;
}
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
mTrace("trans:%d, start sync", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
sdbFreeRaw(pRaw);
return -1;
}
mTrace("trans:%d, sync finished", pTrans->id);
code = sdbWrite(pMnode->pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr());
return -1;
}
mDebug("trans:%d, rollback finished", pTrans->id);
return 0;
}
static void mndTransSendRpcRsp(STrans *pTrans, int32_t code) {
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x", pTrans->id, pTrans->rpcHandle, code & 0xFFFF);
if (pTrans->rpcHandle != NULL) {
SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = code};
rpcSendResponse(&rspMsg);
}
} }
void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) { void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) {
if (code == 0) { // todo
mDebug("trans:%d, commit transaction", pMsg->id);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (sdbWrite(pMnode->pSdb, pRaw) != 0) {
code = terrno;
mError("trans:%d, failed to write sdb while commit since %s", pMsg->id, terrstr());
}
trnSendRpcRsp(pMsg, code);
} else {
mDebug("trans:%d, rollback transaction", pMsg->id);
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
if (sdbWrite(pMnode->pSdb, pRaw) != 0) {
mError("trans:%d, failed to write sdb while rollback since %s", pMsg->id, terrstr());
}
trnSendRpcRsp(pMsg, code);
}
} }
static int32_t trnExecuteArray(SMnode *pMnode, SArray *pArray) { static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) {
for (int32_t i = 0; i < pArray->size; ++i) { SSdb *pSdb = pMnode->pSdb;
int32_t arraySize = taosArrayGetSize(pArray);
for (int32_t i = 0; i < arraySize; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i); SSdbRaw *pRaw = taosArrayGetP(pArray, i);
if (sdbWrite(pMnode->pSdb, pRaw) != 0) { int32_t code = sdbWriteNotFree(pSdb, pRaw);
return -1; if (code != 0) {
return code;
} }
} }
return 0; return 0;
} }
static int32_t trnExecuteRedoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->redoLogs); } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
static int32_t trnExecuteUndoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->undoLogs); } if (taosArrayGetSize(pTrans->redoLogs) != 0) {
code = mndTransExecuteArray(pMnode, pTrans->redoLogs);
static int32_t trnExecuteCommitLogs(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->commitLogs); } if (code != 0) {
mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr())
static int32_t trnExecuteRedoActions(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->redoActions); } } else {
mTrace("trans:%d, execute redo logs finished", pTrans->id)
static int32_t trnExecuteUndoActions(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->undoActions); } }
static int32_t trnPerformPrepareStage(STrans *pTrans) {
if (trnExecuteRedoLogs(pTrans) == 0) {
pTrans->stage = TRN_STAGE_EXECUTE;
return 0;
} else {
pTrans->stage = TRN_STAGE_ROLLBACK;
return -1;
} }
return code;
} }
static int32_t trnPerformExecuteStage(STrans *pTrans) { static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
int32_t code = trnExecuteRedoActions(pTrans); int32_t code = 0;
if (taosArrayGetSize(pTrans->undoLogs) != 0) {
code = mndTransExecuteArray(pMnode, pTrans->undoLogs);
if (code != 0) {
mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr())
} else {
mTrace("trans:%d, execute undo logs finished", pTrans->id)
}
}
return code;
}
static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (taosArrayGetSize(pTrans->commitLogs) != 0) {
code = mndTransExecuteArray(pMnode, pTrans->commitLogs);
if (code != 0) {
mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr())
} else {
mTrace("trans:%d, execute commit logs finished", pTrans->id)
}
}
return code;
}
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->redoActions) != 0) {
mTrace("trans:%d, execute redo actions finished", pTrans->id);
}
return 0;
}
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->undoActions) != 0) {
mTrace("trans:%d, execute undo actions finished", pTrans->id);
}
return 0;
}
static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteRedoLogs(pMnode, pTrans);
if (code == 0) {
pTrans->stage = TRN_STAGE_EXECUTE;
mTrace("trans:%d, stage from prepare to execute", pTrans->id);
} else {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr());
}
return 0;
}
static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteRedoActions(pMnode, pTrans);
if (code == 0) { if (code == 0) {
pTrans->stage = TRN_STAGE_COMMIT; pTrans->stage = TRN_STAGE_COMMIT;
return 0; mTrace("trans:%d, stage from execute to commit", pTrans->id);
} else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
return -1; mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code));
return code;
} else { } else {
if (pTrans->policy == TRN_POLICY_RETRY) { if (pTrans->policy == TRN_POLICY_ROLLBACK) {
pTrans->stage = TRN_STAGE_RETRY;
} else {
pTrans->stage = TRN_STAGE_ROLLBACK; pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr());
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr());
} }
return 0;
} }
return 0;
} }
static int32_t trnPerformCommitStage(STrans *pTrans) { static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) {
if (trnExecuteCommitLogs(pTrans) == 0) { int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans);
pTrans->stage = TRN_STAGE_EXECUTE;
return 0; if (code == 0) {
pTrans->stage = TRN_STAGE_OVER;
mTrace("trans:%d, commit stage finished", pTrans->id);
} else {
if (pTrans->policy == TRN_POLICY_ROLLBACK) {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr());
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr());
}
}
return code;
}
static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteUndoActions(pMnode, pTrans);
if (code == 0) {
mTrace("trans:%d, rollbacked", pTrans->id);
} else { } else {
pTrans->stage = TRN_STAGE_ROLLBACK; pTrans->stage = TRN_STAGE_ROLLBACK;
return -1; mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr());
} }
return code;
} }
static int32_t trnPerformRollbackStage(STrans *pTrans) { static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) {
if (trnExecuteCommitLogs(pTrans) == 0) { int32_t code = mndTransExecuteRedoActions(pMnode, pTrans);
pTrans->stage = TRN_STAGE_EXECUTE;
return 0; if (code == 0) {
pTrans->stage = TRN_STAGE_COMMIT;
mTrace("trans:%d, stage from retry to commit", pTrans->id);
} else { } else {
pTrans->stage = TRN_STAGE_ROLLBACK; pTrans->stage = TRN_STAGE_RETRY;
return -1; mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr());
} }
return code;
} }
static int32_t trnPerformRetryStage(STrans *pTrans) { static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
if (trnExecuteCommitLogs(pTrans) == 0) {
pTrans->stage = TRN_STAGE_EXECUTE;
return 0;
} else {
pTrans->stage = TRN_STAGE_ROLLBACK;
return -1;
}
}
int32_t mndTransExecute(SSdb *pSdb, int32_t tranId) {
int32_t code = 0; int32_t code = 0;
STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &tranId); while (code == 0) {
if (pTrans == NULL) { switch (pTrans->stage) {
return -1; case TRN_STAGE_PREPARE:
} code = mndTransPerformPrepareStage(pMnode, pTrans);
break;
if (pTrans->stage == TRN_STAGE_PREPARE) { case TRN_STAGE_EXECUTE:
if (trnPerformPrepareStage(pTrans) != 0) { code = mndTransPerformExecuteStage(pMnode, pTrans);
sdbRelease(pSdb, pTrans); break;
return -1; case TRN_STAGE_COMMIT:
code = mndTransCommit(pMnode, pTrans);
if (code == 0) {
code = mndTransPerformCommitStage(pMnode, pTrans);
}
break;
case TRN_STAGE_ROLLBACK:
code = mndTransPerformRollbackStage(pMnode, pTrans);
if (code == 0) {
code = mndTransRollback(pMnode, pTrans);
}
break;
case TRN_STAGE_RETRY:
code = mndTransPerformRetryStage(pMnode, pTrans);
break;
default:
mndTransSendRpcRsp(pTrans, 0);
return;
} }
} }
if (pTrans->stage == TRN_STAGE_EXECUTE) { mndTransSendRpcRsp(pTrans, code);
if (trnPerformExecuteStage(pTrans) != 0) { }
sdbRelease(pSdb, pTrans);
return -1;
}
}
if (pTrans->stage == TRN_STAGE_COMMIT) {
if (trnPerformCommitStage(pTrans) != 0) {
sdbRelease(pSdb, pTrans);
return -1;
}
}
if (pTrans->stage == TRN_STAGE_ROLLBACK) {
if (trnPerformRollbackStage(pTrans) != 0) {
sdbRelease(pSdb, pTrans);
return -1;
}
}
if (pTrans->stage == TRN_STAGE_RETRY) {
if (trnPerformRetryStage(pTrans) != 0) {
sdbRelease(pSdb, pTrans);
return -1;
}
}
sdbRelease(pSdb, pTrans);
return 0;
}

162
source/dnode/mnode/impl/src/mndUser.c
View File

@ -20,7 +20,8 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "tkey.h" #include "tkey.h"
#define SDB_USER_VER 1 #define TSDB_USER_VER 1
#define TSDB_USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode); static int32_t mndCreateDefaultUsers(SMnode *pMnode);
static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
@ -93,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
} }
static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, SDB_USER_VER, sizeof(SUserObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -103,6 +104,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime)
SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime)
SDB_SET_INT8(pRaw, dataPos, pUser->superUser) SDB_SET_INT8(pRaw, dataPos, pUser->superUser)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE)
SDB_SET_DATALEN(pRaw, dataPos); SDB_SET_DATALEN(pRaw, dataPos);
return pRaw; return pRaw;
@ -112,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_USER_VER) { if (sver != TSDB_USER_VER) {
mError("failed to decode user since %s", terrstr()); mError("failed to decode user since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL; return NULL;
@ -129,6 +131,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime)
SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_USER_RESERVE_SIZE)
return pRow; return pRow;
} }
@ -166,12 +169,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) {
mTrace("user:%s, perform update action", pOldUser->user); mTrace("user:%s, perform update action", pOldUser->user);
memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN);
memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN); memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN);
memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN);
pOldUser->createdTime = pNewUser->createdTime;
pOldUser->updateTime = pNewUser->updateTime; pOldUser->updateTime = pNewUser->updateTime;
pOldUser->superUser = pNewUser->superUser;
return 0; return 0;
} }
@ -207,101 +206,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass,
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING);
SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("user:%s, failed to update since %s", pOldUser->user, terrstr());
return -1;
}
mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
SSdbRaw *pUndoRaw = mndUserActionEncode(pOldUser); if (mndTransPrepare(pMnode, pTrans) != 0) {
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
if (mndTransPrepare(pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("user:%s, failed to drop since %s", pUser->user, terrstr());
return -1;
}
mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pUser);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndUserActionEncode(pUser);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndUserActionEncode(pUser);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
@ -355,6 +262,32 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("user:%s, failed to update since %s", pOldUser->user, terrstr());
return -1;
}
mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont;
@ -391,6 +324,7 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) {
memcpy(&newUser, pUser, sizeof(SUserObj)); memcpy(&newUser, pUser, sizeof(SUserObj));
memset(pUser->pass, 0, sizeof(pUser->pass)); memset(pUser->pass, 0, sizeof(pUser->pass));
taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass);
newUser.updateTime = taosGetTimestampMs();
int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg);
sdbRelease(pMnode->pSdb, pOperUser); sdbRelease(pMnode->pSdb, pOperUser);
@ -403,6 +337,32 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("user:%s, failed to drop since %s", pUser->user, terrstr());
return -1;
}
mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pUser);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; SDropUserMsg *pDrop = pMsg->rpcMsg.pCont;
@ -526,7 +486,7 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in
sdbRelease(pSdb, pUser); sdbRelease(pSdb, pUser);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }

9
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -71,7 +71,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
void mndCleanupVgroup(SMnode *pMnode) {} void mndCleanupVgroup(SMnode *pMnode) {}
static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SDbObj)); SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
if (pRaw == NULL) return NULL; if (pRaw == NULL) return NULL;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -141,11 +141,8 @@ static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) {
mTrace("vgId:%d, perform update action", pOldVgroup->vgId); mTrace("vgId:%d, perform update action", pOldVgroup->vgId);
pOldVgroup->vgId = pNewVgroup->vgId;
pOldVgroup->createdTime = pNewVgroup->createdTime;
pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->updateTime = pNewVgroup->updateTime;
pOldVgroup->version = pNewVgroup->version; pOldVgroup->version = pNewVgroup->version;
memcpy(pOldVgroup->dbName, pNewVgroup->dbName, TSDB_FULL_DB_NAME_LEN);
pOldVgroup->replica = pNewVgroup->replica; pOldVgroup->replica = pNewVgroup->replica;
memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
return 0; return 0;
@ -284,7 +281,7 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data,
numOfRows++; numOfRows++;
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }
@ -374,7 +371,7 @@ static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows; pShow->numOfReads += numOfRows;
return numOfRows; return numOfRows;
} }

4
source/dnode/mnode/impl/src/mnode.c
View File

@ -69,7 +69,7 @@ static void mndCleanupTimer(SMnode *pMnode) {
} }
} }
static int32_t mnodeCreateDir(SMnode *pMnode, const char *path) { static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
pMnode->path = strdup(path); pMnode->path = strdup(path);
if (pMnode->path == NULL) { if (pMnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -245,7 +245,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL; return NULL;
} }
int32_t code = mnodeCreateDir(pMnode, path); int32_t code = mndCreateDir(pMnode, path);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", terrstr());

1
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -59,6 +59,7 @@ typedef struct SSdb {
char *tmpDir; char *tmpDir;
int64_t lastCommitVer; int64_t lastCommitVer;
int64_t curVer; int64_t curVer;
int32_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX]; EKeyType keyTypes[SDB_MAX];
SHashObj *hashObjs[SDB_MAX]; SHashObj *hashObjs[SDB_MAX];
SRWLatch locks[SDB_MAX]; SRWLatch locks[SDB_MAX];

1
source/dnode/mnode/sdb/src/sdb.c
View File

@ -127,6 +127,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
return -1; return -1;
} }
pSdb->maxId[sdbType] = 0;
pSdb->hashObjs[sdbType] = hash; pSdb->hashObjs[sdbType] = hash;
taosInitRWLatch(&pSdb->locks[sdbType]); taosInitRWLatch(&pSdb->locks[sdbType]);
mDebug("sdb table:%d is initialized", sdbType); mDebug("sdb table:%d is initialized", sdbType);

40
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -72,6 +72,10 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) {
pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj));
}
SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; SdbInsertFp insertFp = pSdb->insertFps[pRow->type];
if (insertFp != NULL) { if (insertFp != NULL) {
code = (*insertFp)(pSdb, pRow->pObj); code = (*insertFp)(pSdb, pRow->pObj);
@ -132,11 +136,6 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosHashRemove(hash, pOldRow->pObj, keySize); taosHashRemove(hash, pOldRow->pObj, keySize);
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
SdbDeleteFp deleteFp = pSdb->deleteFps[pOldRow->type];
if (deleteFp != NULL) {
code = (*deleteFp)(pSdb, pOldRow->pObj);
}
sdbRelease(pSdb, pOldRow->pObj); sdbRelease(pSdb, pOldRow->pObj);
sdbFreeRow(pRow); sdbFreeRow(pRow);
return code; return code;
@ -161,6 +160,7 @@ int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) {
case SDB_STATUS_CREATING: case SDB_STATUS_CREATING:
code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize);
break; break;
case SDB_STATUS_UPDATING:
case SDB_STATUS_READY: case SDB_STATUS_READY:
case SDB_STATUS_DROPPING: case SDB_STATUS_DROPPING:
code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize);
@ -228,6 +228,11 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
}
sdbFreeRow(pRow); sdbFreeRow(pRow);
} }
@ -289,3 +294,28 @@ int32_t sdbGetSize(SSdb *pSdb, ESdbType type) {
return size; return size;
} }
int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) {
SHashObj *hash = sdbGetHash(pSdb, type);
if (hash == NULL) return -1;
if (pSdb->keyTypes[type] != SDB_KEY_INT32) return -1;
int32_t maxId = 0;
SRWLatch *pLock = &pSdb->locks[type];
taosRLockLatch(pLock);
SSdbRow **ppRow = taosHashIterate(hash, NULL);
while (ppRow != NULL) {
SSdbRow *pRow = *ppRow;
int32_t id = *(int32_t *)pRow->pObj;
maxId = MAX(id, maxId);
ppRow = taosHashIterate(hash, ppRow);
}
taosRUnLockLatch(pLock);
maxId = MAX(maxId, pSdb->maxId[type]);
return maxId + 1;
}

2
source/dnode/vnode/impl/CMakeLists.txt
View File

@ -19,5 +19,5 @@ target_link_libraries(
# test # test
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(test) #add_subdirectory(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})

2
source/dnode/vnode/impl/inc/vnodeBufferPool.h
View File

@ -27,6 +27,8 @@ typedef struct SVBufPool SVBufPool;
int vnodeOpenBufPool(SVnode *pVnode); int vnodeOpenBufPool(SVnode *pVnode);
void vnodeCloseBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode);
int vnodeBufPoolSwitch(SVnode *pVnode);
int vnodeBufPoolRecycle(SVnode *pVnode);
void *vnodeMalloc(SVnode *pVnode, uint64_t size); void *vnodeMalloc(SVnode *pVnode, uint64_t size);
bool vnodeBufPoolIsFull(SVnode *pVnode); bool vnodeBufPoolIsFull(SVnode *pVnode);

1
source/dnode/vnode/impl/inc/vnodeCommit.h
View File

@ -24,6 +24,7 @@ extern "C" {
#define vnodeShouldCommit vnodeBufPoolIsFull #define vnodeShouldCommit vnodeBufPoolIsFull
int vnodeAsyncCommit(SVnode *pVnode); int vnodeAsyncCommit(SVnode *pVnode);
int vnodeCommit(void *arg);
#ifdef __cplusplus #ifdef __cplusplus
} }

25
source/dnode/vnode/impl/inc/vnodeDef.h
View File

@ -21,9 +21,11 @@
#include "tcoding.h" #include "tcoding.h"
#include "tdlist.h" #include "tdlist.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tmacro.h"
#include "wal.h" #include "wal.h"
#include "vnode.h" #include "vnode.h"
#include "vnodeBufferPool.h" #include "vnodeBufferPool.h"
#include "vnodeCfg.h" #include "vnodeCfg.h"
#include "vnodeCommit.h" #include "vnodeCommit.h"
@ -37,6 +39,27 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SVnodeTask {
TD_DLIST_NODE(SVnodeTask);
void* arg;
int (*execute)(void*);
} SVnodeTask;
typedef struct SVnodeMgr {
td_mode_flag_t vnodeInitFlag;
td_mode_flag_t vnodeClearFlag;
// For commit
bool stop;
uint16_t nthreads;
pthread_t* threads;
pthread_mutex_t mutex;
pthread_cond_t hasTask;
TD_DLIST(SVnodeTask) queue;
// For vnode Mgmt
} SVnodeMgr;
extern SVnodeMgr vnodeMgr;
struct SVnode { struct SVnode {
char* path; char* path;
SVnodeCfg config; SVnodeCfg config;
@ -50,6 +73,8 @@ struct SVnode {
SVnodeFS* pFs; SVnodeFS* pFs;
}; };
int vnodeScheduleTask(SVnodeTask* task);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/vnode/impl/inc/vnodeMemAllocator.h
View File

@ -33,6 +33,7 @@ struct SVArenaNode {
}; };
struct SVMemAllocator { struct SVMemAllocator {
T_REF_DECLARE()
TD_DLIST_NODE(SVMemAllocator); TD_DLIST_NODE(SVMemAllocator);
uint64_t capacity; uint64_t capacity;
uint64_t ssize; uint64_t ssize;

26
source/dnode/vnode/impl/src/vnodeBufferPool.c
View File

@ -19,6 +19,8 @@
#define VNODE_BUF_POOL_SHARDS 3 #define VNODE_BUF_POOL_SHARDS 3
struct SVBufPool { struct SVBufPool {
pthread_mutex_t mutex;
pthread_cond_t hasFree;
TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) free;
TD_DLIST(SVMemAllocator) incycle; TD_DLIST(SVMemAllocator) incycle;
SVMemAllocator *inuse; SVMemAllocator *inuse;
@ -28,7 +30,6 @@ struct SVBufPool {
int vnodeOpenBufPool(SVnode *pVnode) { int vnodeOpenBufPool(SVnode *pVnode) {
uint64_t capacity; uint64_t capacity;
// EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) {
/* TODO */ /* TODO */
@ -79,6 +80,27 @@ void vnodeCloseBufPool(SVnode *pVnode) {
} }
} }
int vnodeBufPoolSwitch(SVnode *pVnode) {
SVMemAllocator *pvma = pVnode->pBufPool->inuse;
pVnode->pBufPool->inuse = NULL;
tDListAppend(&(pVnode->pBufPool->incycle), pvma);
return 0;
}
int vnodeBufPoolRecycle(SVnode *pVnode) {
SVBufPool * pBufPool = pVnode->pBufPool;
SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle));
ASSERT(pvma != NULL);
tDListPop(&(pBufPool->incycle), pvma);
vmaReset(pvma);
tDListAppend(&(pBufPool->free), pvma);
return 0;
}
void *vnodeMalloc(SVnode *pVnode, uint64_t size) { void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
SVBufPool *pBufPool = pVnode->pBufPool; SVBufPool *pBufPool = pVnode->pBufPool;
@ -89,6 +111,8 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
if (pBufPool->inuse) { if (pBufPool->inuse) {
tDListPop(&(pBufPool->free), pBufPool->inuse); tDListPop(&(pBufPool->free), pBufPool->inuse);
break; break;
} else {
// tsem_wait(&(pBufPool->hasFree));
} }
} }
} }

3
source/dnode/vnode/impl/src/vnodeCfg.c
View File

@ -15,7 +15,8 @@
#include "vnodeDef.h" #include "vnodeDef.h"
const SVnodeCfg defaultVnodeOptions = {.wsize = 16 * 1024 * 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ const SVnodeCfg defaultVnodeOptions = {
.wsize = 96 * 1024 * 1024, .ssize = 1 * 1024 * 1024, .lsize = 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */
void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */
vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions);

29
source/dnode/vnode/impl/src/vnodeCommit.c
View File

@ -19,28 +19,21 @@ static int vnodeStartCommit(SVnode *pVnode);
static int vnodeEndCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode);
int vnodeAsyncCommit(SVnode *pVnode) { int vnodeAsyncCommit(SVnode *pVnode) {
#if 0 vnodeBufPoolSwitch(pVnode);
if (vnodeStartCommit(pVnode) < 0) { SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask));
// TODO
}
if (tqCommit(pVnode->pTQ) < 0) { pTask->execute = vnodeCommit; // TODO
// TODO pTask->arg = pVnode; // TODO
}
if (metaCommit(pVnode->pMeta) < 0) { vnodeScheduleTask(pTask);
// TODO return 0;
} }
if (tsdbCommit(pVnode->pTsdb) < 0) { int vnodeCommit(void *arg) {
// TODO SVnode *pVnode = (SVnode *)arg;
}
if (vnodeEndCommit(pVnode) < 0) { vnodeBufPoolRecycle(pVnode);
// TODO // TODO
}
#endif
return 0; return 0;
} }

24
source/dnode/vnode/impl/src/vnodeMain.c
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tmacro.h"
#include "vnodeDef.h" #include "vnodeDef.h"
static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg);
@ -21,29 +20,6 @@ static void vnodeFree(SVnode *pVnode);
static int vnodeOpenImpl(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode);
static void vnodeCloseImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode);
TD_DEF_MOD_INIT_FLAG(vnode);
TD_DEF_MOD_CLEAR_FLAG(vnode);
int vnodeInit() {
if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) {
return 0;
}
if (walInit() < 0) {
return -1;
}
return 0;
}
void vnodeClear() {
if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) {
return;
}
walCleanUp();
}
SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) {
SVnode *pVnode = NULL; SVnode *pVnode = NULL;

115
source/dnode/vnode/impl/src/vnodeMgr.c Normal file
View File

@ -0,0 +1,115 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeDef.h"
SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED, .vnodeClearFlag = TD_MOD_UNCLEARD, .stop = false};
static void* loop(void* arg);
int vnodeInit(uint16_t nthreads) {
if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) {
return 0;
}
// Start commit handers
if (nthreads > 0) {
vnodeMgr.nthreads = nthreads;
vnodeMgr.threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t));
if (vnodeMgr.threads == NULL) {
return -1;
}
pthread_mutex_init(&(vnodeMgr.mutex), NULL);
pthread_cond_init(&(vnodeMgr.hasTask), NULL);
tDListInit(&(vnodeMgr.queue));
for (uint16_t i = 0; i < nthreads; i++) {
pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL);
}
} else {
// TODO: if no commit thread is set, then another mechanism should be
// given. Otherwise, it is a false.
ASSERT(0);
}
if (walInit() < 0) {
return -1;
}
return 0;
}
void vnodeClear() {
if (TD_CHECK_AND_SET_MOD_CLEAR(&(vnodeMgr.vnodeClearFlag)) == TD_MOD_CLEARD) {
return;
}
walCleanUp();
// Stop commit handler
pthread_mutex_lock(&(vnodeMgr.mutex));
vnodeMgr.stop = true;
pthread_cond_broadcast(&(vnodeMgr.hasTask));
pthread_mutex_unlock(&(vnodeMgr.mutex));
for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) {
pthread_join(vnodeMgr.threads[i], NULL);
}
tfree(vnodeMgr.threads);
pthread_cond_destroy(&(vnodeMgr.hasTask));
pthread_mutex_destroy(&(vnodeMgr.mutex));
}
int vnodeScheduleTask(SVnodeTask* pTask) {
pthread_mutex_lock(&(vnodeMgr.mutex));
tDListAppend(&(vnodeMgr.queue), pTask);
pthread_cond_signal(&(vnodeMgr.hasTask));
pthread_mutex_unlock(&(vnodeMgr.mutex));
return 0;
}
/* ------------------------ STATIC METHODS ------------------------ */
static void* loop(void* arg) {
SVnodeTask* pTask;
for (;;) {
pthread_mutex_lock(&(vnodeMgr.mutex));
for (;;) {
pTask = TD_DLIST_HEAD(&(vnodeMgr.queue));
if (pTask == NULL) {
if (vnodeMgr.stop) {
pthread_mutex_unlock(&(vnodeMgr.mutex));
return NULL;
} else {
pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex));
}
} else {
tDListPop(&(vnodeMgr.queue), pTask);
break;
}
}
pthread_mutex_unlock(&(vnodeMgr.mutex));
(*(pTask->execute))(pTask->arg);
}
return NULL;
}

4
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -72,7 +72,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
} }
break; break;
case TSDB_MSG_TYPE_SUBMIT: case TSDB_MSG_TYPE_SUBMIT:
/* code */ if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) {
// TODO: handle error
}
break; break;
default: default:
break; break;

4
source/dnode/vnode/impl/test/vnodeApiTests.cpp
View File

@ -92,7 +92,9 @@ TEST(vnodeApiTest, test_create_table_encode_and_decode_function) {
#endif #endif
TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) {
GTEST_ASSERT_GE(vnodeInit(), 0); vnodeDestroy("vnode1");
GTEST_ASSERT_GE(vnodeInit(2), 0);
// Create and open a vnode // Create and open a vnode
SVnode *pVnode = vnodeOpen("vnode1", NULL); SVnode *pVnode = vnodeOpen("vnode1", NULL);

28
source/dnode/vnode/tq/inc/tqMetaStore.h
View File

@ -24,29 +24,29 @@ extern "C" {
#endif #endif
TqMetaStore* tqStoreOpen(const char* path, STqMetaStore* tqStoreOpen(const char* path,
TqSerializeFun pSerializer, FTqSerialize pSerializer,
TqDeserializeFun pDeserializer, FTqDeserialize pDeserializer,
TqDeleteFun pDeleter, FTqDelete pDeleter,
int32_t tqConfigFlag int32_t tqConfigFlag
); );
int32_t tqStoreClose(TqMetaStore*); int32_t tqStoreClose(STqMetaStore*);
//int32_t tqStoreDelete(TqMetaStore*); //int32_t tqStoreDelete(TqMetaStore*);
//int32_t tqStoreCommitAll(TqMetaStore*); //int32_t tqStoreCommitAll(TqMetaStore*);
int32_t tqStorePersist(TqMetaStore*); int32_t tqStorePersist(STqMetaStore*);
//clean deleted idx and data from persistent file //clean deleted idx and data from persistent file
int32_t tqStoreCompact(TqMetaStore*); int32_t tqStoreCompact(STqMetaStore*);
void* tqHandleGet(TqMetaStore*, int64_t key); void* tqHandleGet(STqMetaStore*, int64_t key);
//make it unpersist //make it unpersist
void* tqHandleTouchGet(TqMetaStore*, int64_t key); void* tqHandleTouchGet(STqMetaStore*, int64_t key);
int32_t tqHandleMovePut(TqMetaStore*, int64_t key, void* value); int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
int32_t tqHandleCopyPut(TqMetaStore*, int64_t key, void* value, size_t vsize); int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
//delete committed kv pair //delete committed kv pair
//notice that a delete action still needs to be committed //notice that a delete action still needs to be committed
int32_t tqHandleDel(TqMetaStore*, int64_t key); int32_t tqHandleDel(STqMetaStore*, int64_t key);
int32_t tqHandleCommit(TqMetaStore*, int64_t key); int32_t tqHandleCommit(STqMetaStore*, int64_t key);
int32_t tqHandleAbort(TqMetaStore*, int64_t key); int32_t tqHandleAbort(STqMetaStore*, int64_t key);
#ifdef __cplusplus #ifdef __cplusplus
} }

318
source/dnode/vnode/tq/src/tq.c
View File

@ -16,75 +16,70 @@
#include "tqInt.h" #include "tqInt.h"
#include "tqMetaStore.h" #include "tqMetaStore.h"
//static // static
//read next version data // read next version data
// //
//send to fetch queue // send to fetch queue
// //
//handle management message // handle management message
// //
int tqGetgHandleSSize(const TqGroupHandle *gHandle); int tqGetgHandleSSize(const STqGroupHandle* gHandle);
int tqBufHandleSSize(); int tqBufHandleSSize();
int tqBufItemSSize(); int tqBufItemSSize();
TqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
TqGroupHandle* gHandle; STqGroupHandle* gHandle;
return NULL; return NULL;
} }
void* tqSerializeListHandle(TqListHandle* listHandle, void* ptr); void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr);
void* tqSerializeBufHandle(TqBufferHandle* bufHandle, void* ptr); void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr);
void* tqSerializeBufItem(TqBufferItem* bufItem, void* ptr); void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr);
const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle* bufHandle); const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle);
const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem* bufItem); const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem);
STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac) { STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ)); STQ* pTq = malloc(sizeof(STQ));
if(pTq == NULL) { if (pTq == NULL) {
//TODO: memory error // TODO: memory error
return NULL; return NULL;
} }
pTq->path = strdup(path); pTq->path = strdup(path);
pTq->tqConfig = tqConfig; pTq->tqConfig = tqConfig;
pTq->tqLogReader = tqLogReader; pTq->tqLogReader = tqLogReader;
// pTq->tqMemRef.pAlloctorFactory = allocFac; pTq->tqMemRef.pAlloctorFactory = allocFac;
// pTq->tqMemRef.pAllocator = allocFac->create(allocFac); pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
if(pTq->tqMemRef.pAllocator == NULL) { if (pTq->tqMemRef.pAllocator == NULL) {
//TODO // TODO
} }
pTq->tqMeta = tqStoreOpen(path, pTq->tqMeta =
(TqSerializeFun)tqSerializeGroupHandle, tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0);
(TqDeserializeFun)tqDeserializeGroupHandle, if (pTq->tqMeta == NULL) {
free, // TODO: free STQ
0);
if(pTq->tqMeta == NULL) {
//TODO: free STQ
return NULL; return NULL;
} }
return pTq; return pTq;
} }
static int tqProtoCheck(TmqMsgHead *pMsg) { static int tqProtoCheck(TmqMsgHead* pMsg) { return pMsg->protoVer == 0; }
return pMsg->protoVer == 0;
}
static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** ppQuery) { static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) {
//clean old item and move forward // clean old item and move forward
int32_t consumeOffset = pAck->consumeOffset; int32_t consumeOffset = pAck->consumeOffset;
int idx = consumeOffset % TQ_BUFFER_SIZE; int idx = consumeOffset % TQ_BUFFER_SIZE;
ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor);
tfree(bHandle->buffer[idx].content); tfree(bHandle->buffer[idx].content);
if( 1 /* TODO: need to launch new query */) { if (1 /* TODO: need to launch new query */) {
TqQueryMsg* pNewQuery = malloc(sizeof(TqQueryMsg)); STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg));
if(pNewQuery == NULL) { if (pNewQuery == NULL) {
//TODO: memory insufficient // TODO: memory insufficient
return -1; return -1;
} }
//TODO: lock executor // TODO: lock executor
pNewQuery->exec->executor = bHandle->buffer[idx].executor; pNewQuery->exec->executor = bHandle->buffer[idx].executor;
//TODO: read from wal and assign to src // TODO: read from wal and assign to src
pNewQuery->exec->src = 0; pNewQuery->exec->src = 0;
pNewQuery->exec->dest = &bHandle->buffer[idx]; pNewQuery->exec->dest = &bHandle->buffer[idx];
pNewQuery->next = *ppQuery; pNewQuery->next = *ppQuery;
@ -93,98 +88,94 @@ static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg**
return 0; return 0;
} }
static int tqAck(TqGroupHandle* gHandle, TmqAcks* pAcks) { static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
int32_t ackNum = pAcks->ackNum; int32_t ackNum = pAcks->ackNum;
TmqOneAck *acks = pAcks->acks; TmqOneAck* acks = pAcks->acks;
//double ptr for acks and list // double ptr for acks and list
int i = 0; int i = 0;
TqListHandle* node = gHandle->head; STqListHandle* node = gHandle->head;
int ackCnt = 0; int ackCnt = 0;
TqQueryMsg *pQuery = NULL; STqQueryMsg* pQuery = NULL;
while(i < ackNum && node->next) { while (i < ackNum && node->next) {
if(acks[i].topicId == node->next->bufHandle.topicId) { if (acks[i].topicId == node->next->bufHandle.topicId) {
ackCnt++; ackCnt++;
tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery);
} else if(acks[i].topicId < node->next->bufHandle.topicId) { } else if (acks[i].topicId < node->next->bufHandle.topicId) {
i++; i++;
} else { } else {
node = node->next; node = node->next;
} }
} }
if(pQuery) { if (pQuery) {
//post message // post message
} }
return ackCnt; return ackCnt;
} }
static int tqCommitTCGroup(TqGroupHandle* handle) { static int tqCommitTCGroup(STqGroupHandle* handle) {
//persist modification into disk // persist modification into disk
return 0; return 0;
} }
int tqCreateTCGroup(STQ *pTq, int64_t topicId, int64_t cgId, int64_t cId, TqGroupHandle** handle) { int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) {
//create in disk // create in disk
TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle)); STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle));
if(gHandle == NULL) { if (gHandle == NULL) {
//TODO // TODO
return -1; return -1;
} }
memset(gHandle, 0, sizeof(TqGroupHandle)); memset(gHandle, 0, sizeof(STqGroupHandle));
return 0; return 0;
} }
TqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
TqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
if(gHandle == NULL) { if (gHandle == NULL) {
int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle);
if(code != 0) { if (code != 0) {
//TODO // TODO
return NULL; return NULL;
} }
} }
//create // create
//open // open
return gHandle; return gHandle;
} }
int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; }
return 0;
}
int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
//delete from disk // delete from disk
return 0; return 0;
} }
static int tqFetch(TqGroupHandle* gHandle, void** msg) { static int tqFetch(STqGroupHandle* gHandle, void** msg) {
TqListHandle* head = gHandle->head; STqListHandle* head = gHandle->head;
TqListHandle* node = head; STqListHandle* node = head;
int totSize = 0; int totSize = 0;
//TODO: make it a macro // TODO: make it a macro
int sizeLimit = 4 * 1024; int sizeLimit = 4 * 1024;
TmqMsgContent* buffer = malloc(sizeLimit); TmqMsgContent* buffer = malloc(sizeLimit);
if(buffer == NULL) { if (buffer == NULL) {
//TODO:memory insufficient // TODO:memory insufficient
return -1; return -1;
} }
//iterate the list to get msgs of all topics // iterate the list to get msgs of all topics
//until all topic iterated or msgs over sizeLimit // until all topic iterated or msgs over sizeLimit
while(node->next) { while (node->next) {
node = node->next; node = node->next;
TqBufferHandle* bufHandle = &node->bufHandle; STqBufferHandle* bufHandle = &node->bufHandle;
int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if(bufHandle->buffer[idx].content != NULL && if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) {
bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset
) {
totSize += bufHandle->buffer[idx].size; totSize += bufHandle->buffer[idx].size;
if(totSize > sizeLimit) { if (totSize > sizeLimit) {
void *ptr = realloc(buffer, totSize); void* ptr = realloc(buffer, totSize);
if(ptr == NULL) { if (ptr == NULL) {
totSize -= bufHandle->buffer[idx].size; totSize -= bufHandle->buffer[idx].size;
//TODO:memory insufficient // TODO:memory insufficient
//return msgs already copied // return msgs already copied
break; break;
} }
} }
@ -194,7 +185,7 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size);
buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size);
if(totSize > sizeLimit) { if (totSize > sizeLimit) {
break; break;
} }
} }
@ -202,105 +193,98 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
return totSize; return totSize;
} }
STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; }
TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; }
return NULL;
}
int tqLaunchQuery(TqGroupHandle* gHandle) { int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; }
return 0;
}
int tqSendLaunchQuery(TqGroupHandle* gHandle) {
return 0;
}
/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/
/*return 0;*/ /*return 0;*/
/*}*/ /*}*/
int tqPushMsg(STQ* pTq , void* p, int64_t version) { int tqPushMsg(STQ* pTq, void* p, int64_t version) {
//add reference // add reference
//judge and launch new query // judge and launch new query
return 0; return 0;
} }
int tqCommit(STQ* pTq) { int tqCommit(STQ* pTq) {
//do nothing // do nothing
return 0; return 0;
} }
int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) { int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) {
if(!tqProtoCheck((TmqMsgHead *)pMsg)) { if (!tqProtoCheck((TmqMsgHead*)pMsg)) {
//proto version invalid // proto version invalid
return -1; return -1;
} }
int64_t clientId = pMsg->head.clientId; int64_t clientId = pMsg->head.clientId;
TqGroupHandle *gHandle = tqGetGroupHandle(pTq, clientId); STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId);
if(gHandle == NULL) { if (gHandle == NULL) {
//client not connect // client not connect
return -1; return -1;
} }
if(pMsg->acks.ackNum != 0) { if (pMsg->acks.ackNum != 0) {
if(tqAck(gHandle, &pMsg->acks) != 0) { if (tqAck(gHandle, &pMsg->acks) != 0) {
//ack not success // ack not success
return -1; return -1;
} }
} }
TmqConsumeRsp *pRsp = (TmqConsumeRsp*) pMsg; TmqConsumeRsp* pRsp = (TmqConsumeRsp*)pMsg;
if(tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
//fetch error // fetch error
return -1; return -1;
} }
//judge and launch new query // judge and launch new query
if(tqLaunchQuery(gHandle)) { if (tqLaunchQuery(gHandle)) {
//launch query error // launch query error
return -1; return -1;
} }
return 0; return 0;
} }
int tqSerializeGroupHandle(const TqGroupHandle *gHandle, TqSerializedHead** ppHead) { int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) {
//calculate size // calculate size
int sz = tqGetgHandleSSize(gHandle) + sizeof(TqSerializedHead); int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead);
if(sz > (*ppHead)->ssize) { if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz); void* tmpPtr = realloc(*ppHead, sz);
if(tmpPtr == NULL) { if (tmpPtr == NULL) {
free(*ppHead); free(*ppHead);
//TODO: memory err // TODO: memory err
return -1; return -1;
} }
*ppHead = tmpPtr; *ppHead = tmpPtr;
(*ppHead)->ssize = sz; (*ppHead)->ssize = sz;
} }
void* ptr = (*ppHead)->content; void* ptr = (*ppHead)->content;
//do serialization // do serialization
*(int64_t*)ptr = gHandle->cId; *(int64_t*)ptr = gHandle->cId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = gHandle->cgId; *(int64_t*)ptr = gHandle->cgId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = gHandle->topicNum; *(int32_t*)ptr = gHandle->topicNum;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
if(gHandle->topicNum > 0) { if (gHandle->topicNum > 0) {
tqSerializeListHandle(gHandle->head, ptr); tqSerializeListHandle(gHandle->head, ptr);
} }
return 0; return 0;
} }
void* tqSerializeListHandle(TqListHandle *listHandle, void* ptr) { void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) {
TqListHandle *node = listHandle; STqListHandle* node = listHandle;
ASSERT(node != NULL); ASSERT(node != NULL);
while(node) { while (node) {
ptr = tqSerializeBufHandle(&node->bufHandle, ptr); ptr = tqSerializeBufHandle(&node->bufHandle, ptr);
node = node->next; node = node->next;
} }
return ptr; return ptr;
} }
void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) { void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) {
*(int64_t*)ptr = bufHandle->nextConsumeOffset; *(int64_t*)ptr = bufHandle->nextConsumeOffset;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = bufHandle->topicId; *(int64_t*)ptr = bufHandle->topicId;
@ -309,21 +293,21 @@ void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) {
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
*(int32_t*)ptr = bufHandle->tail; *(int32_t*)ptr = bufHandle->tail;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for(int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr);
} }
return ptr; return ptr;
} }
void* tqSerializeBufItem(TqBufferItem *bufItem, void* ptr) { void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) {
//TODO: do we need serialize this? // TODO: do we need serialize this?
//mainly for executor // mainly for executor
return ptr; return ptr;
} }
const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle **ppGHandle) { const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) {
TqGroupHandle *gHandle = *ppGHandle; STqGroupHandle* gHandle = *ppGHandle;
const void* ptr = pHead->content; const void* ptr = pHead->content;
gHandle->cId = *(int64_t*)ptr; gHandle->cId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
gHandle->cgId = *(int64_t*)ptr; gHandle->cgId = *(int64_t*)ptr;
@ -332,20 +316,20 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
gHandle->topicNum = *(int32_t*)ptr; gHandle->topicNum = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
gHandle->head = NULL; gHandle->head = NULL;
TqListHandle *node = gHandle->head; STqListHandle* node = gHandle->head;
for(int i = 0; i < gHandle->topicNum; i++) { for (int i = 0; i < gHandle->topicNum; i++) {
if(gHandle->head == NULL) { if (gHandle->head == NULL) {
if((node = malloc(sizeof(TqListHandle))) == NULL) { if ((node = malloc(sizeof(STqListHandle))) == NULL) {
//TODO: error // TODO: error
return NULL; return NULL;
} }
node->next= NULL; node->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); ptr = tqDeserializeBufHandle(ptr, &node->bufHandle);
gHandle->head = node; gHandle->head = node;
} else { } else {
node->next = malloc(sizeof(TqListHandle)); node->next = malloc(sizeof(STqListHandle));
if(node->next == NULL) { if (node->next == NULL) {
//TODO: error // TODO: error
return NULL; return NULL;
} }
node->next->next = NULL; node->next->next = NULL;
@ -356,7 +340,7 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
return ptr; return ptr;
} }
const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle) { const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) {
const void* ptr = pBytes; const void* ptr = pBytes;
bufHandle->nextConsumeOffset = *(int64_t*)ptr; bufHandle->nextConsumeOffset = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@ -366,32 +350,30 @@ const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
bufHandle->tail = *(int32_t*)ptr; bufHandle->tail = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for(int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]);
} }
return ptr; return ptr;
} }
const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem *bufItem) { const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; }
return pBytes;
// TODO: make this a macro
int tqGetgHandleSSize(const STqGroupHandle* gHandle) {
return sizeof(int64_t) * 2 // cId + cgId
+ sizeof(int32_t) // topicNum
+ gHandle->topicNum * tqBufHandleSSize();
} }
//TODO: make this a macro // TODO: make this a macro
int tqGetgHandleSSize(const TqGroupHandle *gHandle) {
return sizeof(int64_t) * 2 //cId + cgId
+ sizeof(int32_t) //topicNum
+ gHandle->topicNum * tqBufHandleSSize();
}
//TODO: make this a macro
int tqBufHandleSSize() { int tqBufHandleSSize() {
return sizeof(int64_t) * 2 // nextConsumeOffset + topicId return sizeof(int64_t) * 2 // nextConsumeOffset + topicId
+ sizeof(int32_t) * 2 // head + tail + sizeof(int32_t) * 2 // head + tail
+ TQ_BUFFER_SIZE * tqBufItemSSize(); + TQ_BUFFER_SIZE * tqBufItemSSize();
} }
int tqBufItemSSize() { int tqBufItemSSize() {
//TODO: do this need serialization? // TODO: do this need serialization?
//mainly for executor // mainly for executor
return 0; return 0;
} }

449
source/dnode/vnode/tq/src/tqMetaStore.c
View File

@ -13,21 +13,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tqMetaStore.h" #include "tqMetaStore.h"
//TODO:replace by an abstract file layer // TODO:replace by an abstract file layer
#include "osDir.h"
#include <fcntl.h> #include <fcntl.h>
#include <string.h> #include <string.h>
#include <unistd.h> #include <unistd.h>
#include "osDir.h"
#define TQ_META_NAME "tq.meta" #define TQ_META_NAME "tq.meta"
#define TQ_IDX_NAME "tq.idx" #define TQ_IDX_NAME "tq.idx"
static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value);
static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key);
static int32_t tqHandlePutCommitted(TqMetaStore*, int64_t key, void* value); static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
static void* tqHandleGetUncommitted(TqMetaStore*, int64_t key); if (pNode->unpersistNext == NULL) {
static inline void tqLinkUnpersist(TqMetaStore *pMeta, TqMetaList* pNode) {
if(pNode->unpersistNext == NULL) {
pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pNode->unpersistPrev = pMeta->unpersistHead; pNode->unpersistPrev = pMeta->unpersistHead;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
@ -42,24 +41,24 @@ static inline int tqSeekLastPage(int fd) {
return lseek(fd, curPageOffset, SEEK_SET); return lseek(fd, curPageOffset, SEEK_SET);
} }
//TODO: the struct is tightly coupled with index entry // TODO: the struct is tightly coupled with index entry
typedef struct TqIdxPageHead { typedef struct STqIdxPageHead {
int16_t writeOffset; int16_t writeOffset;
int8_t unused[14]; int8_t unused[14];
} TqIdxPageHead; } STqIdxPageHead;
typedef struct TqIdxPageBuf { typedef struct STqIdxPageBuf {
TqIdxPageHead head; STqIdxPageHead head;
char buffer[TQ_IDX_PAGE_BODY_SIZE]; char buffer[TQ_IDX_PAGE_BODY_SIZE];
} TqIdxPageBuf; } STqIdxPageBuf;
static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd); int offset = tqSeekLastPage(fd);
int nBytes; int nBytes;
if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
return -1; return -1;
} }
if(nBytes == 0) { if (nBytes == 0) {
memset(pBuf, 0, TQ_PAGE_SIZE); memset(pBuf, 0, TQ_PAGE_SIZE);
pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} }
@ -68,28 +67,24 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
return lseek(fd, offset, SEEK_SET); return lseek(fd, offset, SEEK_SET);
} }
TqMetaStore* tqStoreOpen(const char* path, STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter,
TqSerializeFun serializer, int32_t tqConfigFlag) {
TqDeserializeFun deserializer, STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
TqDeleteFun deleter, if (pMeta == NULL) {
int32_t tqConfigFlag // close
) {
TqMetaStore* pMeta = malloc(sizeof(TqMetaStore));
if(pMeta == NULL) {
//close
return NULL; return NULL;
} }
memset(pMeta, 0, sizeof(TqMetaStore)); memset(pMeta, 0, sizeof(STqMetaStore));
//concat data file name and index file name // concat data file name and index file name
size_t pathLen = strlen(path); size_t pathLen = strlen(path);
pMeta->dirPath = malloc(pathLen+1); pMeta->dirPath = malloc(pathLen + 1);
if(pMeta->dirPath != NULL) { if (pMeta->dirPath != NULL) {
//TODO: memory insufficient // TODO: memory insufficient
} }
strcpy(pMeta->dirPath, path); strcpy(pMeta->dirPath, path);
char name[pathLen+10]; char name[pathLen + 10];
strcpy(name, path); strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
@ -97,98 +92,96 @@ TqMetaStore* tqStoreOpen(const char* path,
} }
strcat(name, "/" TQ_IDX_NAME); strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755); int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if(idxFd < 0) { if (idxFd < 0) {
ASSERT(false); ASSERT(false);
//close file // close file
//free memory // free memory
return NULL; return NULL;
} }
pMeta->idxFd = idxFd; pMeta->idxFd = idxFd;
pMeta->unpersistHead = malloc(sizeof(TqMetaList)); pMeta->unpersistHead = malloc(sizeof(STqMetaList));
if(pMeta->unpersistHead == NULL) { if (pMeta->unpersistHead == NULL) {
ASSERT(false); ASSERT(false);
//close file // close file
//free memory // free memory
return NULL; return NULL;
} }
memset(pMeta->unpersistHead, 0, sizeof(TqMetaList)); memset(pMeta->unpersistHead, 0, sizeof(STqMetaList));
pMeta->unpersistHead->unpersistNext pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
= pMeta->unpersistHead->unpersistPrev
= pMeta->unpersistHead;
strcpy(name, path); strcpy(name, path);
strcat(name, "/" TQ_META_NAME); strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755); int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if(fileFd < 0){ if (fileFd < 0) {
ASSERT(false); ASSERT(false);
return NULL; return NULL;
} }
pMeta->fileFd = fileFd; pMeta->fileFd = fileFd;
pMeta->pSerializer = serializer; pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer; pMeta->pDeserializer = deserializer;
pMeta->pDeleter = deleter; pMeta->pDeleter = deleter;
pMeta->tqConfigFlag = tqConfigFlag; pMeta->tqConfigFlag = tqConfigFlag;
//read idx file and load into memory // read idx file and load into memory
TqIdxPageBuf idxBuf; STqIdxPageBuf idxBuf;
TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
if(serializedObj == NULL) { if (serializedObj == NULL) {
//TODO:memory insufficient // TODO:memory insufficient
} }
int idxRead; int idxRead;
int allocated = TQ_PAGE_SIZE; int allocated = TQ_PAGE_SIZE;
bool readEnd = false; bool readEnd = false;
while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
if(idxRead == -1) { if (idxRead == -1) {
//TODO: handle error // TODO: handle error
ASSERT(false); ASSERT(false);
} }
ASSERT(idxBuf.head.writeOffset == idxRead); ASSERT(idxBuf.head.writeOffset == idxRead);
//loop read every entry // loop read every entry
for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
TqMetaList *pNode = malloc(sizeof(TqMetaList)); STqMetaList* pNode = malloc(sizeof(STqMetaList));
if(pNode == NULL) { if (pNode == NULL) {
//TODO: free memory and return error // TODO: free memory and return error
} }
memset(pNode, 0, sizeof(TqMetaList)); memset(pNode, 0, sizeof(STqMetaList));
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET); lseek(fileFd, pNode->handle.offset, SEEK_SET);
if(allocated < pNode->handle.serializedSize) { if (allocated < pNode->handle.serializedSize) {
void *ptr = realloc(serializedObj, pNode->handle.serializedSize); void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if(ptr == NULL) { if (ptr == NULL) {
//TODO: memory insufficient // TODO: memory insufficient
} }
serializedObj = ptr; serializedObj = ptr;
allocated = pNode->handle.serializedSize; allocated = pNode->handle.serializedSize;
} }
serializedObj->ssize = pNode->handle.serializedSize; serializedObj->ssize = pNode->handle.serializedSize;
if(read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
//TODO: read error // TODO: read error
} }
if(serializedObj->action == TQ_ACTION_INUSE) { if (serializedObj->action == TQ_ACTION_INUSE) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) { if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
} else { } else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN; pNode->handle.valueInUse = TQ_DELETE_TOKEN;
} }
} else if(serializedObj->action == TQ_ACTION_INTXN) { } else if (serializedObj->action == TQ_ACTION_INTXN) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) { if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn); pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn);
} else { } else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN; pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
} }
} else if(serializedObj->action == TQ_ACTION_INUSE_CONT) { } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) { if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
} else { } else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN; pNode->handle.valueInUse = TQ_DELETE_TOKEN;
} }
TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
if(ptr->ssize != sizeof(TqSerializedHead)) { if (ptr->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn); pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn);
} else { } else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN; pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
@ -197,22 +190,21 @@ TqMetaStore* tqStoreOpen(const char* path,
ASSERT(0); ASSERT(0);
} }
//put into list // put into list
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
TqMetaList* pBucketNode = pMeta->bucket[bucketKey]; STqMetaList* pBucketNode = pMeta->bucket[bucketKey];
if(pBucketNode == NULL) { if (pBucketNode == NULL) {
pMeta->bucket[bucketKey] = pNode; pMeta->bucket[bucketKey] = pNode;
} else if(pBucketNode->handle.key == pNode->handle.key) { } else if (pBucketNode->handle.key == pNode->handle.key) {
pNode->next = pBucketNode->next; pNode->next = pBucketNode->next;
pMeta->bucket[bucketKey] = pNode; pMeta->bucket[bucketKey] = pNode;
} else { } else {
while(pBucketNode->next && while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) {
pBucketNode->next->handle.key != pNode->handle.key) { pBucketNode = pBucketNode->next;
pBucketNode = pBucketNode->next;
} }
if(pBucketNode->next) { if (pBucketNode->next) {
ASSERT(pBucketNode->next->handle.key == pNode->handle.key); ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
TqMetaList *pNodeFound = pBucketNode->next; STqMetaList* pNodeFound = pBucketNode->next;
pNode->next = pNodeFound->next; pNode->next = pNodeFound->next;
pBucketNode->next = pNode; pBucketNode->next = pNode;
pBucketNode = pNodeFound; pBucketNode = pNodeFound;
@ -222,13 +214,11 @@ TqMetaStore* tqStoreOpen(const char* path,
pBucketNode = NULL; pBucketNode = NULL;
} }
} }
if(pBucketNode) { if (pBucketNode) {
if(pBucketNode->handle.valueInUse if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInUse); pMeta->pDeleter(pBucketNode->handle.valueInUse);
} }
if(pBucketNode->handle.valueInTxn if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInTxn); pMeta->pDeleter(pBucketNode->handle.valueInTxn);
} }
free(pBucketNode); free(pBucketNode);
@ -239,27 +229,25 @@ TqMetaStore* tqStoreOpen(const char* path,
return pMeta; return pMeta;
} }
int32_t tqStoreClose(TqMetaStore* pMeta) { int32_t tqStoreClose(STqMetaStore* pMeta) {
//commit data and idx // commit data and idx
tqStorePersist(pMeta); tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd); close(pMeta->fileFd);
close(pMeta->idxFd); close(pMeta->idxFd);
//free memory // free memory
for(int i = 0; i < TQ_BUCKET_SIZE; i++) { for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
TqMetaList* pNode = pMeta->bucket[i]; STqMetaList* pNode = pMeta->bucket[i];
while(pNode) { while (pNode) {
ASSERT(pNode->unpersistNext == NULL); ASSERT(pNode->unpersistNext == NULL);
ASSERT(pNode->unpersistPrev == NULL); ASSERT(pNode->unpersistPrev == NULL);
if(pNode->handle.valueInTxn if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
TqMetaList* next = pNode->next; STqMetaList* next = pNode->next;
free(pNode); free(pNode);
pNode = next; pNode = next;
} }
@ -270,23 +258,21 @@ int32_t tqStoreClose(TqMetaStore* pMeta) {
return 0; return 0;
} }
int32_t tqStoreDelete(TqMetaStore* pMeta) { int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd); close(pMeta->fileFd);
close(pMeta->idxFd); close(pMeta->idxFd);
//free memory // free memory
for(int i = 0; i < TQ_BUCKET_SIZE; i++) { for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
TqMetaList* pNode = pMeta->bucket[i]; STqMetaList* pNode = pMeta->bucket[i];
pMeta->bucket[i] = NULL; pMeta->bucket[i] = NULL;
while(pNode) { while (pNode) {
if(pNode->handle.valueInTxn if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
TqMetaList* next = pNode->next; STqMetaList* next = pNode->next;
free(pNode); free(pNode);
pNode = next; pNode = next;
} }
@ -298,26 +284,26 @@ int32_t tqStoreDelete(TqMetaStore* pMeta) {
return 0; return 0;
} }
//TODO: wrap in tfile // TODO: wrap in tfile
int32_t tqStorePersist(TqMetaStore* pMeta) { int32_t tqStorePersist(STqMetaStore* pMeta) {
TqIdxPageBuf idxBuf; STqIdxPageBuf idxBuf;
int64_t* bufPtr = (int64_t*)idxBuf.buffer; int64_t* bufPtr = (int64_t*)idxBuf.buffer;
TqMetaList *pHead = pMeta->unpersistHead; STqMetaList* pHead = pMeta->unpersistHead;
TqMetaList *pNode = pHead->unpersistNext; STqMetaList* pNode = pHead->unpersistNext;
TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead)); STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead));
if(pSHead == NULL) { if (pSHead == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
pSHead->ver = TQ_SVER; pSHead->ver = TQ_SVER;
pSHead->checksum = 0; pSHead->checksum = 0;
pSHead->ssize = sizeof(TqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
int allocatedSize = sizeof(TqSerializedHead); int allocatedSize = sizeof(STqSerializedHead);
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf); tqReadLastPage(pMeta->idxFd, &idxBuf);
if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) { if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END); lseek(pMeta->idxFd, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
@ -325,18 +311,18 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset); bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
} }
while(pHead != pNode) { while (pHead != pNode) {
int nBytes = 0; int nBytes = 0;
if(pNode->handle.valueInUse) { if (pNode->handle.valueInUse) {
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INUSE_CONT; pSHead->action = TQ_ACTION_INUSE_CONT;
} else { } else {
pSHead->action = TQ_ACTION_INUSE; pSHead->action = TQ_ACTION_INUSE;
} }
if(pNode->handle.valueInUse == TQ_DELETE_TOKEN) { if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(TqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
} else { } else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
} }
@ -344,10 +330,10 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
ASSERT(nBytes == pSHead->ssize); ASSERT(nBytes == pSHead->ssize);
} }
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INTXN; pSHead->action = TQ_ACTION_INTXN;
if(pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(TqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
} else { } else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
} }
@ -358,42 +344,39 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
pNode->handle.offset = offset; pNode->handle.offset = offset;
offset += nBytes; offset += nBytes;
//write idx file // write idx file
//TODO: endian check and convert // TODO: endian check and convert
*(bufPtr++) = pNode->handle.key; *(bufPtr++) = pNode->handle.key;
*(bufPtr++) = pNode->handle.offset; *(bufPtr++) = pNode->handle.offset;
*(bufPtr++) = (int64_t)nBytes; *(bufPtr++) = (int64_t)nBytes;
idxBuf.head.writeOffset += TQ_IDX_SIZE; idxBuf.head.writeOffset += TQ_IDX_SIZE;
if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
//TODO: handle error with tfile // TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE); ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
bufPtr = (int64_t*)&idxBuf.buffer; bufPtr = (int64_t*)&idxBuf.buffer;
} }
//remove from unpersist list // remove from unpersist list
pHead->unpersistNext = pNode->unpersistNext; pHead->unpersistNext = pNode->unpersistNext;
pHead->unpersistNext->unpersistPrev = pHead; pHead->unpersistNext->unpersistPrev = pHead;
pNode->unpersistPrev = pNode->unpersistNext = NULL; pNode->unpersistPrev = pNode->unpersistNext = NULL;
pNode = pHead->unpersistNext; pNode = pHead->unpersistNext;
//remove from bucket // remove from bucket
if(pNode->handle.valueInUse == TQ_DELETE_TOKEN && if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) {
pNode->handle.valueInTxn == NULL int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
) { STqMetaList* pBucketHead = pMeta->bucket[bucketKey];
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; if (pBucketHead == pNode) {
TqMetaList* pBucketHead = pMeta->bucket[bucketKey];
if(pBucketHead == pNode) {
pMeta->bucket[bucketKey] = pNode->next; pMeta->bucket[bucketKey] = pNode->next;
} else { } else {
TqMetaList* pBucketNode = pBucketHead; STqMetaList* pBucketNode = pBucketHead;
while(pBucketNode->next != NULL while (pBucketNode->next != NULL && pBucketNode->next != pNode) {
&& pBucketNode->next != pNode) { pBucketNode = pBucketNode->next;
pBucketNode = pBucketNode->next;
} }
//impossible for pBucket->next == NULL // impossible for pBucket->next == NULL
ASSERT(pBucketNode->next == pNode); ASSERT(pBucketNode->next == pNode);
pBucketNode->next = pNode->next; pBucketNode->next = pNode->next;
} }
@ -401,46 +384,45 @@ int32_t tqStorePersist(TqMetaStore* pMeta) {
} }
} }
//write left bytes // write left bytes
free(pSHead); free(pSHead);
//TODO: write new version in tfile // TODO: write new version in tfile
if((char*)bufPtr != idxBuf.buffer) { if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
//TODO: handle error in tfile // TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset); ASSERT(nBytes == idxBuf.head.writeOffset);
} }
//TODO: using fsync in tfile // TODO: using fsync in tfile
fsync(pMeta->idxFd); fsync(pMeta->idxFd);
fsync(pMeta->fileFd); fsync(pMeta->fileFd);
return 0; return 0;
} }
static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value) { static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
//TODO: think about thread safety // TODO: think about thread safety
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
//change pointer ownership // change pointer ownership
pNode->handle.valueInUse = value; pNode->handle.valueInUse = value;
return 0; return 0;
} else { } else {
pNode = pNode->next; pNode = pNode->next;
} }
} }
TqMetaList *pNewNode = malloc(sizeof(TqMetaList)); STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if(pNewNode == NULL) { if (pNewNode == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
memset(pNewNode, 0, sizeof(TqMetaList)); memset(pNewNode, 0, sizeof(STqMetaList));
pNewNode->handle.key = key; pNewNode->handle.key = key;
pNewNode->handle.valueInUse = value; pNewNode->handle.valueInUse = value;
//put into unpersist list // put into unpersist list
pNewNode->unpersistPrev = pMeta->unpersistHead; pNewNode->unpersistPrev = pMeta->unpersistHead;
pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode;
@ -448,13 +430,12 @@ static int32_t tqHandlePutCommitted(TqMetaStore* pMeta, int64_t key, void* value
return 0; return 0;
} }
void* tqHandleGet(TqMetaStore* pMeta, int64_t key) { void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInUse != NULL if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
return pNode->handle.valueInUse; return pNode->handle.valueInUse;
} else { } else {
return NULL; return NULL;
@ -466,13 +447,12 @@ void* tqHandleGet(TqMetaStore* pMeta, int64_t key) {
return NULL; return NULL;
} }
void* tqHandleTouchGet(TqMetaStore* pMeta, int64_t key) { void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInUse != NULL if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
tqLinkUnpersist(pMeta, pNode); tqLinkUnpersist(pMeta, pNode);
return pNode->handle.valueInUse; return pNode->handle.valueInUse;
} else { } else {
@ -485,17 +465,17 @@ void* tqHandleTouchGet(TqMetaStore* pMeta, int64_t key) {
return NULL; return NULL;
} }
static inline int32_t tqHandlePutImpl(TqMetaStore* pMeta, int64_t key, void* value) { static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
//TODO: think about thread safety // TODO: think about thread safety
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
if(TqDupIntxnReject(pMeta->tqConfigFlag)) { if (TqDupIntxnReject(pMeta->tqConfigFlag)) {
return -2; return -2;
} }
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
} }
@ -506,12 +486,12 @@ static inline int32_t tqHandlePutImpl(TqMetaStore* pMeta, int64_t key, void* val
pNode = pNode->next; pNode = pNode->next;
} }
} }
TqMetaList *pNewNode = malloc(sizeof(TqMetaList)); STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if(pNewNode == NULL) { if (pNewNode == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
memset(pNewNode, 0, sizeof(TqMetaList)); memset(pNewNode, 0, sizeof(STqMetaList));
pNewNode->handle.key = key; pNewNode->handle.key = key;
pNewNode->handle.valueInTxn = value; pNewNode->handle.valueInTxn = value;
pNewNode->next = pMeta->bucket[bucketKey]; pNewNode->next = pMeta->bucket[bucketKey];
@ -520,27 +500,24 @@ static inline int32_t tqHandlePutImpl(TqMetaStore* pMeta, int64_t key, void* val
return 0; return 0;
} }
int32_t tqHandleMovePut(TqMetaStore* pMeta, int64_t key, void* value) { int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); }
return tqHandlePutImpl(pMeta, key, value);
}
int32_t tqHandleCopyPut(TqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
void *vmem = malloc(vsize); void* vmem = malloc(vsize);
if(vmem == NULL) { if (vmem == NULL) {
//TODO: memory error // TODO: memory error
return -1; return -1;
} }
memcpy(vmem, value, vsize); memcpy(vmem, value, vsize);
return tqHandlePutImpl(pMeta, key, vmem); return tqHandlePutImpl(pMeta, key, vmem);
} }
static void* tqHandleGetUncommitted(TqMetaStore* pMeta, int64_t key) { static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInTxn != NULL if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
return pNode->handle.valueInTxn; return pNode->handle.valueInTxn;
} else { } else {
return NULL; return NULL;
@ -552,16 +529,15 @@ static void* tqHandleGetUncommitted(TqMetaStore* pMeta, int64_t key) {
return NULL; return NULL;
} }
int32_t tqHandleCommit(TqMetaStore* pMeta, int64_t key) { int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInTxn == NULL) { if (pNode->handle.valueInTxn == NULL) {
return -1; return -1;
} }
if(pNode->handle.valueInUse if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse); pMeta->pDeleter(pNode->handle.valueInUse);
} }
pNode->handle.valueInUse = pNode->handle.valueInTxn; pNode->handle.valueInUse = pNode->handle.valueInTxn;
@ -575,13 +551,13 @@ int32_t tqHandleCommit(TqMetaStore* pMeta, int64_t key) {
return -2; return -2;
} }
int32_t tqHandleAbort(TqMetaStore* pMeta, int64_t key) { int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.key == key) { if (pNode->handle.key == key) {
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
pNode->handle.valueInTxn = NULL; pNode->handle.valueInTxn = NULL;
@ -596,14 +572,15 @@ int32_t tqHandleAbort(TqMetaStore* pMeta, int64_t key) {
return -2; return -2;
} }
int32_t tqHandleDel(TqMetaStore* pMeta, int64_t key) { int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK; int64_t bucketKey = key & TQ_BUCKET_MASK;
TqMetaList* pNode = pMeta->bucket[bucketKey]; STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) { while (pNode) {
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if(pNode->handle.valueInTxn) { if (pNode->handle.valueInTxn) {
pMeta->pDeleter(pNode->handle.valueInTxn); pMeta->pDeleter(pNode->handle.valueInTxn);
} }
pNode->handle.valueInTxn = TQ_DELETE_TOKEN; pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
tqLinkUnpersist(pMeta, pNode); tqLinkUnpersist(pMeta, pNode);
return 0; return 0;
@ -611,11 +588,9 @@ int32_t tqHandleDel(TqMetaStore* pMeta, int64_t key) {
pNode = pNode->next; pNode = pNode->next;
} }
} }
//no such key // no such key
return -1; return -1;
} }
//TODO: clean deleted idx and data from persistent file // TODO: clean deleted idx and data from persistent file
int32_t tqStoreCompact(TqMetaStore *pMeta) { int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; }
return 0;
}

10
source/dnode/vnode/tq/test/tqMetaTest.cpp
View File

@ -9,17 +9,17 @@ struct Foo {
int32_t a; int32_t a;
}; };
int FooSerializer(const void* pObj, TqSerializedHead** ppHead) { int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
Foo* foo = (Foo*) pObj; Foo* foo = (Foo*) pObj;
if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) { if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
*ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t)); *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
} }
*(int32_t*)(*ppHead)->content = foo->a; *(int32_t*)(*ppHead)->content = foo->a;
return (*ppHead)->ssize; return (*ppHead)->ssize;
} }
const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) { const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
if(*ppObj == NULL) { if(*ppObj == NULL) {
*ppObj = realloc(*ppObj, sizeof(int32_t)); *ppObj = realloc(*ppObj, sizeof(int32_t));
} }

6
source/dnode/vnode/tsdb/inc/tsdbDef.h
View File

@ -17,6 +17,10 @@
#define _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_
#include "mallocator.h" #include "mallocator.h"
#include "taosmsg.h"
#include "tdlist.h"
#include "thash.h"
#include "tskiplist.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsdbMemTable.h" #include "tsdbMemTable.h"
@ -29,6 +33,8 @@ extern "C" {
struct STsdb { struct STsdb {
char * path; char * path;
STsdbCfg options; STsdbCfg options;
STsdbMemTable * mem;
STsdbMemTable * imem;
SMemAllocatorFactory *pmaf; SMemAllocatorFactory *pmaf;
}; };

9
source/dnode/vnode/tsdb/inc/tsdbMemTable.h
View File

@ -22,10 +22,11 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SMemTable { typedef struct STsdbMemTable STsdbMemTable;
/* TODO */
SMemAllocator *pma; STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF);
} SMemTable; void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable);
int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

20
src/inc/tp.h → source/dnode/vnode/tsdb/src/tsdbCommit.c
View File

@ -13,19 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_TP #include "tsdbDef.h"
#define TDENGINE_TP
#ifdef __cplusplus int tsdbCommit(STsdb *pTsdb) {
extern "C" { // TODO
#endif return 0;
}
int32_t tpInit();
void tpCleanUp();
void tpUpdateTs(int32_t vgId, int64_t *seq, void *pMsg);
#ifdef __cplusplus
}
#endif
#endif

82
source/dnode/vnode/tsdb/src/tsdbMemTable.c
View File

@ -11,4 +11,84 @@
* *
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tsdbDef.h"
#if 1
typedef struct STbData {
TD_SLIST_NODE(STbData);
SSubmitMsg *pMsg;
} STbData;
#else
typedef struct STbData {
TD_SLIST_NODE(STbData);
uint64_t uid; // TODO: change here as tb_uid_t
TSKEY keyMin;
TSKEY keyMax;
uint64_t nRows;
SSkipList *pData; // Here need a container, may not use the SL
T_REF_DECLARE()
} STbData;
#endif
struct STsdbMemTable {
T_REF_DECLARE()
SRWLatch latch;
TSKEY keyMin;
TSKEY keyMax;
uint64_t nRow;
SMemAllocator *pMA;
// Container
TD_SLIST(STbData) list;
};
STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) {
STsdbMemTable *pMemTable;
SMemAllocator *pMA;
pMA = (*pMAF->create)(pMAF);
ASSERT(pMA != NULL);
pMemTable = (STsdbMemTable *)((*pMA->malloc)(pMA, sizeof(*pMemTable)));
if (pMemTable == NULL) {
(*pMAF->destroy)(pMAF, pMA);
return NULL;
}
T_REF_INIT_VAL(pMemTable, 1);
taosInitRWLatch(&(pMemTable->latch));
pMemTable->keyMin = TSKEY_MAX;
pMemTable->keyMax = TSKEY_MIN;
pMemTable->nRow = 0;
pMemTable->pMA = pMA;
tSListInit(&(pMemTable->list));
// TODO
return pMemTable;
}
void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) {
SMemAllocator *pMA = pMemTable->pMA;
if (pMA->free) {
// TODO
ASSERT(0);
}
(*pMAF->destroy)(pMAF, pMA);
}
int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) {
SMemAllocator *pMA = pMemTable->pMA;
STbData * pTbData = (STbData *)((*pMA->malloc)(pMA, sizeof(*pTbData)));
if (pTbData == NULL) {
// TODO
}
tSListPush(&(pMemTable->list), pTbData);
return 0;
}
/* ------------------------ STATIC METHODS ------------------------ */

25
src/inc/tadmin.h → source/dnode/vnode/tsdb/src/tsdbWrite.c
View File

@ -13,20 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_ADMIN_H #include "tsdbDef.h"
#define TDENGINE_ADMIN_H
#ifdef __cplusplus int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg) {
extern "C" { // Check if mem is there. If not, create one.
#endif pTsdb->mem = tsdbNewMemTable(pTsdb->pmaf);
if (pTsdb->mem == NULL) {
struct HttpServer; return -1;
}
void adminInitHandle(struct HttpServer* pServer); return tsdbInsertDataToMemTable(pTsdb->mem, pMsg);
void opInitHandle(struct HttpServer* pServer); }
#ifdef __cplusplus
}
#endif
#endif

3
source/libs/CMakeLists.txt
View File

@ -9,4 +9,5 @@ add_subdirectory(cache)
add_subdirectory(catalog) add_subdirectory(catalog)
add_subdirectory(executor) add_subdirectory(executor)
add_subdirectory(planner) add_subdirectory(planner)
add_subdirectory(function) add_subdirectory(function)
add_subdirectory(query)

6
source/libs/catalog/CMakeLists.txt
View File

@ -8,5 +8,7 @@ target_include_directories(
target_link_libraries( target_link_libraries(
catalog catalog
PRIVATE os util common transport PRIVATE os util common transport query
) )
ADD_SUBDIRECTORY(test)

54
source/libs/catalog/inc/catalogInt.h
View File

@ -21,14 +21,62 @@ extern "C" {
#endif #endif
#include "catalog.h" #include "catalog.h"
#include "common.h"
#include "tlog.h"
#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6
#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
#define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000
#define CTG_DEFAULT_INVALID_VERSION (-1)
typedef struct SVgroupListCache {
int32_t vgroupVersion;
SHashObj *cache; // key:vgId, value:SVgroupInfo
} SVgroupListCache;
typedef struct SDBVgroupCache {
SHashObj *cache; //key:dbname, value:SDBVgroupInfo
} SDBVgroupCache;
typedef struct STableMetaCache {
SHashObj *cache; //key:fulltablename, value:STableMeta
SHashObj *stableCache; //key:suid, value:STableMeta*
} STableMetaCache;
typedef struct SCatalog { typedef struct SCatalog {
void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata SVgroupListCache vgroupCache;
SHashObj *pData; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node SDBVgroupCache dbCache;
STableMetaCache tableCache;
} SCatalog; } SCatalog;
typedef struct SCatalogMgmt {
void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata
SHashObj *pCluster; // items cached for each cluster, the hash key is the cluster-id got from mgmt node
SCatalogCfg cfg;
} SCatalogMgmt;
typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
extern int32_t ctgDebugFlag;
#define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgInfo(...) do { if (ctgDebugFlag & DEBUG_INFO) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgDebug(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0)
#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /*_TD_CATALOG_INT_H_*/ #endif /*_TD_CATALOG_INT_H_*/

653
source/libs/catalog/src/catalog.c
View File

@ -14,11 +14,656 @@
*/ */
#include "catalogInt.h" #include "catalogInt.h"
#include "trpc.h"
#include "query.h"
#include "tname.h"
struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { SCatalogMgmt ctgMgmt = {0};
return (struct SCatalog*) 0x1;
int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo *dbInfo, int32_t *exist) {
if (NULL == pCatalog->dbCache.cache) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
if (NULL == info) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
if (dbInfo) {
*dbInfo = *info;
}
*exist = 1;
return TSDB_CODE_SUCCESS;
} }
int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
return 0;
int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) {
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen);
if (code) {
return code;
}
SRpcMsg rpcMsg = {
.msgType = TSDB_MSG_TYPE_USE_DB,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
code = queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
return code;
}
return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) {
if (NULL == pCatalog->tableCache.cache) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName);
STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName));
if (NULL == tbMeta) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
if (tbMeta->tableType == TSDB_CHILD_TABLE) {
STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid));
if (NULL == stbMeta || NULL == *stbMeta) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
if ((*stbMeta)->suid != tbMeta->suid) {
ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema);
*pTableMeta = calloc(1, metaSize);
if (NULL == *pTableMeta) {
ctgError("calloc size[%d] failed", metaSize);
return TSDB_CODE_CTG_MEM_ERROR;
}
memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta));
memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta));
} else {
int32_t metaSize = sizeof(STableMeta) + (tbMeta->tableInfo.numOfTags + tbMeta->tableInfo.numOfColumns) * sizeof(SSchema);
*pTableMeta = calloc(1, metaSize);
if (NULL == *pTableMeta) {
ctgError("calloc size[%d] failed", metaSize);
return TSDB_CODE_CTG_MEM_ERROR;
}
memcpy(*pTableMeta, tbMeta, metaSize);
}
*exist = 1;
return TSDB_CODE_SUCCESS;
}
void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
epSet->inUse = 0;
epSet->numOfEps = vgroupInfo->numOfEps;
for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) {
memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i]));
memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i]));
}
}
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName};
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen);
if (code) {
return code;
}
SRpcMsg rpcMsg = {
.msgType = TSDB_MSG_TYPE_TABLE_META,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
SEpSet epSet;
ctgGenEpSet(&epSet, vgroupInfo);
rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code);
return rpcRsp.code;
}
code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
return code;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
switch (hashMethod) {
default:
*fp = MurmurHash3_32;
break;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray* vgroupList) {
SHashObj *vgroupHash = NULL;
SVgroupInfo *vgInfo = NULL;
void *pIter = taosHashIterate(dbInfo->vgInfo, NULL);
while (pIter) {
vgInfo = pIter;
if (NULL == taosArrayPush(vgroupList, vgInfo)) {
ctgError("taosArrayPush failed");
break;
}
pIter = taosHashIterate(dbInfo->vgInfo, pIter);
vgInfo = NULL;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
int32_t vgNum = taosHashGetSize(dbInfo->vgInfo);
if (vgNum <= 0) {
ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum);
return TSDB_CODE_TSC_DB_NOT_SELECTED;
}
tableNameHashFp fp = NULL;
SVgroupInfo *vgInfo = NULL;
CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
void *pIter = taosHashIterate(dbInfo->vgInfo, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
break;
}
pIter = taosHashIterate(dbInfo->vgInfo, pIter);
vgInfo = NULL;
}
if (NULL == vgInfo) {
ctgError("no hash range found for hashvalue[%u]", hashValue);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
*pVgroup = *vgInfo;
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
SDBVgroupInfo dbInfo = {0};
int32_t code = 0;
int32_t vgId = 0;
CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo));
if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) {
ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo);
return TSDB_CODE_TSC_DB_NOT_SELECTED;
}
CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup));
return code;
}
STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) {
assert(pChild != NULL);
int32_t total = pChild->numOfColumns + pChild->numOfTags;
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
pTableMeta->tableType = TSDB_SUPER_TABLE;
pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
pTableMeta->tableInfo.precision = pChild->precision;
pTableMeta->uid = pChild->suid;
pTableMeta->tversion = pChild->tversion;
pTableMeta->sversion = pChild->sversion;
memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total);
int32_t num = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < num; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
return pTableMeta;
}
int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) {
if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
int32_t exist = 0;
if (!forceUpdate) {
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist));
if (exist) {
return TSDB_CODE_SUCCESS;
}
}
CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName));
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist));
if (0 == exist) {
ctgError("get table meta from cache failed, but fetch succeed");
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) {
if (output->metaNum != 1 && output->metaNum != 2) {
ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
if (NULL == output->tbMeta) {
ctgError("no valid table meta got from meta rsp");
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
if (NULL == pCatalog->tableCache.cache) {
pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.cache) {
ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
return TSDB_CODE_CTG_MEM_ERROR;
}
}
if (NULL == pCatalog->tableCache.cache) {
pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.cache) {
ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
return TSDB_CODE_CTG_MEM_ERROR;
}
pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.stableCache) {
ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
return TSDB_CODE_CTG_MEM_ERROR;
}
}
if (output->metaNum == 2) {
if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) {
ctgError("push ctable[%s] to table cache failed", output->ctbFname);
goto error_exit;
}
if (TSDB_SUPER_TABLE != output->tbMeta->tableType) {
ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE);
goto error_exit;
}
}
if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, sizeof(*output->tbMeta)) != 0) {
ctgError("push table[%s] to table cache failed", output->tbFname);
goto error_exit;
}
if (TSDB_SUPER_TABLE == output->tbMeta->tableType) {
if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) {
ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid);
goto error_exit;
}
}
return TSDB_CODE_SUCCESS;
error_exit:
if (pCatalog->vgroupCache.cache) {
taosHashCleanup(pCatalog->vgroupCache.cache);
pCatalog->vgroupCache.cache = NULL;
}
pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t catalogInit(SCatalogCfg *cfg) {
ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == ctgMgmt.pCluster) {
CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER);
}
if (cfg) {
memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg));
} else {
ctgMgmt.cfg.enableVgroupCache = true;
ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER;
}
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) {
if (NULL == clusterId || NULL == catalogHandle) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
if (NULL == ctgMgmt.pCluster) {
ctgError("cluster cache are not ready");
return TSDB_CODE_CTG_NOT_READY;
}
size_t clen = strlen(clusterId);
SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen);
if (clusterCtg) {
*catalogHandle = clusterCtg;
return TSDB_CODE_SUCCESS;
}
clusterCtg = calloc(1, sizeof(*clusterCtg));
if (NULL == clusterCtg) {
ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg));
return TSDB_CODE_CTG_MEM_ERROR;
}
clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) {
ctgError("put cluster %s cache to hash failed", clusterId);
tfree(clusterCtg);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
*catalogHandle = clusterCtg;
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) {
if (NULL == pCatalog || NULL == dbName || NULL == version) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
if (NULL == pCatalog->dbCache.cache) {
*version = CTG_DEFAULT_INVALID_VERSION;
return TSDB_CODE_SUCCESS;
}
SDBVgroupInfo * dbInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
if (NULL == dbInfo) {
*version = CTG_DEFAULT_INVALID_VERSION;
return TSDB_CODE_SUCCESS;
}
*version = dbInfo->vgVersion;
return TSDB_CODE_SUCCESS;
}
int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
if (dbInfo->vgVersion < 0) {
if (pCatalog->dbCache.cache) {
taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName));
}
ctgWarn("remove db [%s] from cache", dbName);
return TSDB_CODE_SUCCESS;
}
if (NULL == pCatalog->dbCache.cache) {
pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->dbCache.cache) {
ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
return TSDB_CODE_CTG_MEM_ERROR;
}
} else {
SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
if (oldInfo && oldInfo->vgInfo) {
taosHashCleanup(oldInfo->vgInfo);
oldInfo->vgInfo = NULL;
}
}
if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) {
ctgError("push to vgroup hash cache failed");
return TSDB_CODE_CTG_MEM_ERROR;
}
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
int32_t exist = 0;
int32_t code = 0;
if (0 == forceUpdate) {
CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist));
if (exist) {
return TSDB_CODE_SUCCESS;
}
}
SUseDbOutput DbOut = {0};
SBuildUseDBInput input = {0};
strncpy(input.db, dbName, sizeof(input.db));
input.db[sizeof(input.db) - 1] = 0;
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
CTG_ERR_RET(catalogUpdateDBVgroupCache(pCatalog, dbName, &DbOut.dbVgroup));
if (dbInfo) {
*dbInfo = DbOut.dbVgroup;
}
return code;
}
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, false, pTableMeta);
}
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) {
if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
SVgroupInfo vgroupInfo = {0};
CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo));
STableMetaOutput output = {0};
CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output));
CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output));
tfree(output.tbMeta);
return TSDB_CODE_SUCCESS;
}
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta);
}
int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
STableMeta *tbMeta = NULL;
int32_t code = 0;
SVgroupInfo vgroupInfo = {0};
SDBVgroupInfo dbVgroup = {0};
CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta));
CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup));
if (tbMeta->tableType == TSDB_SUPER_TABLE) {
CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList));
} else {
int32_t vgId = tbMeta->vgId;
if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) {
ctgError("vgId[%d] not found in vgroup list", vgId);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) {
ctgError("push vgroupInfo to array failed");
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
}
_return:
tfree(tbMeta);
return code;
}
int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
int32_t code = 0;
if (pReq->pTableName) {
char dbName[TSDB_FULL_DB_NAME_LEN];
int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName);
if (tbNum > 0) {
pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES);
if (NULL == pRsp->pTableMeta) {
ctgError("taosArrayInit num[%d] failed", tbNum);
return TSDB_CODE_CTG_MEM_ERROR;
}
}
for (int32_t i = 0; i < tbNum; ++i) {
SName *name = taosArrayGet(pReq->pTableName, i);
STableMeta *pTableMeta = NULL;
snprintf(dbName, sizeof(dbName), "%s.%s", name->acctId, name->dbname);
CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta));
if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) {
ctgError("taosArrayPush failed, idx:%d", i);
tfree(pTableMeta);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
}
}
}
return TSDB_CODE_SUCCESS;
_return:
if (pRsp->pTableMeta) {
int32_t aSize = taosArrayGetSize(pRsp->pTableMeta);
for (int32_t i = 0; i < aSize; ++i) {
STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i);
tfree(pMeta);
}
taosArrayDestroy(pRsp->pTableMeta);
}
return code;
}
void catalogDestroy(void) {
if (ctgMgmt.pCluster) {
taosHashCleanup(ctgMgmt.pCluster); //TBD
ctgMgmt.pCluster = NULL;
}
}

18
source/libs/catalog/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build catalog unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
catalogTest
PUBLIC os util common catalog transport gtest query
)
TARGET_INCLUDE_DIRECTORIES(
catalogTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc"
)

152
source/libs/catalog/test/catalogTests.cpp
View File

@ -0,0 +1,152 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "catalog.h"
namespace {
}
TEST(testCase, normalCase) {
char *clusterId = "cluster1";
struct SCatalog* pCtg = NULL;
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(clusterId, &pCtg);
ASSERT_EQ(code, 0);
}
/*
TEST(testCase, normalCase) {
SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)");
ASSERT_EQ(info1.valid, true);
char msg[128] = {0};
SMsgBuf buf;
buf.len = 128;
buf.buf = msg;
SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0);
int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
ASSERT_EQ(code, 0);
SCatalogReq req = {0};
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
ASSERT_EQ(ret, 0);
ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
SQueryStmtInfo* pQueryInfo = createQueryInfo();
setTableMetaInfo(pQueryInfo, &req);
SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0);
ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
ASSERT_EQ(ret, 0);
SArray* pExprList = pQueryInfo->exprList[0];
int32_t num = tsCompatibleModel? 2:1;
ASSERT_EQ(taosArrayGetSize(pExprList), num);
SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1);
ASSERT_EQ(p1->base.pColumns->uid, 110);
ASSERT_EQ(p1->base.numOfParams, 1);
ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)");
ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP);
ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)");
ASSERT_EQ(p1->base.interBytes, 16);
ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE);
ASSERT_STREQ(p1->pExpr->_function.functionName, "top");
tExprNode* pParam = p1->pExpr->_function.pChild[0];
ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE);
ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3);
ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2);
struct SQueryPlanNode* n = nullptr;
code = createQueryPlan(pQueryInfo, &n);
char* str = NULL;
queryPlanToString(n, &str);
printf("%s\n", str);
destroyQueryInfo(pQueryInfo);
qParserClearupMetaRequestInfo(&req);
destroySqlInfo(&info1);
}
TEST(testCase, displayPlan) {
generateLogicplan("select count(*) from `t.1abc`");
generateLogicplan("select count(*)+ 22 from `t.1abc`");
generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30");
generateLogicplan("select count(*) from `t.1abc` group by a");
generateLogicplan("select count(A+B) from `t.1abc` group by a");
generateLogicplan("select count(length(a)+b) from `t.1abc` group by a");
generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)");
generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s) order by 1 desc ");
generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`");
generateLogicplan("select count(*), min(a) + 99 from `t.1abc`");
generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`");
generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20");
generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)");
generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)");
// order by + group by column + limit offset
generateLogicplan("select top(a, 20) k from `t.1abc` order by k asc limit 3 offset 1");
// fill
generateLogicplan("select min(a) from `t.1abc` where ts>now and ts<now+2h interval(1s) fill(linear)");
// union + union all
// join
// Aggregate(count(*) [count(*) #5056], sum(a) [sum(a) #5057], avg(b) [avg(b) #5058], min(a+b) [min(a+b) #5060])
// Projection(cols: [a+b #5059]) filters:(nil)
// Projection(cols: [ts #0], [a #1], [b #2]) filters:(nil)
// TableScan(t.1abc #110) time_range: -9223372036854775808 - 9223372036854775807
}
*/
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

4
source/libs/executor/src/executorimpl.c
View File

@ -4831,8 +4831,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
STableBlockDist tableBlockDist = {0}; STableBlockDist tableBlockDist = {0};
tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables;
int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS;
if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
++numRowSteps; ++numRowSteps;
} }
tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));

4
source/libs/index/inc/index_fst.h
View File

@ -30,6 +30,7 @@ extern "C" {
typedef struct Fst Fst; typedef struct Fst Fst;
typedef struct FstNode FstNode; typedef struct FstNode FstNode;
typedef struct StreamWithState StreamWithState;
typedef enum { Included, Excluded, Unbounded} FstBound; typedef enum { Included, Excluded, Unbounded} FstBound;
@ -283,6 +284,9 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null);
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx);
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
// into stream to expand later
StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb);
bool fstVerify(Fst *fst); bool fstVerify(Fst *fst);

22
source/libs/index/inc/index_fst_automation.h
View File

@ -20,6 +20,8 @@ extern "C" {
#endif #endif
#include "index_fst_util.h" #include "index_fst_util.h"
typedef struct AutomationCtx AutomationCtx; typedef struct AutomationCtx AutomationCtx;
typedef enum AutomationType { typedef enum AutomationType {
@ -38,18 +40,30 @@ typedef struct Complement {
// automation // automation
typedef struct AutomationCtx { typedef struct AutomationCtx {
AutomationType type; AutomationType type;
void *data; void *stdata;
char *data;
} AutomationCtx; } AutomationCtx;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType;
typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
typedef struct StartWithStateValue { typedef struct StartWithStateValue {
StartWithStateKind kind; StartWithStateKind kind;
void *value; ValueType type;
union {
int val;
char *ptr;
SArray *arr;
// add more type
} ;
} StartWithStateValue; } StartWithStateValue;
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val);
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv);
void startWithStateValueDestroy(void *sv);
typedef struct AutomationFunc { typedef struct AutomationFunc {
void* (*start)(AutomationCtx *ctx) ; void* (*start)(AutomationCtx *ctx) ;
bool (*isMatch)(AutomationCtx *ctx, void *); bool (*isMatch)(AutomationCtx *ctx, void *);
@ -59,7 +73,7 @@ typedef struct AutomationFunc {
void* (*acceptEof)(AutomationCtx *ct, void *state); void* (*acceptEof)(AutomationCtx *ct, void *state);
} AutomationFunc; } AutomationFunc;
AutomationCtx *automCtxCreate(void *data, AutomationType type); AutomationCtx *automCtxCreate(void *data, AutomationType atype);
void automCtxDestroy(AutomationCtx *ctx); void automCtxDestroy(AutomationCtx *ctx);
extern AutomationFunc automFuncs[]; extern AutomationFunc automFuncs[];

32
source/libs/index/src/index_fst.c
View File

@ -17,6 +17,7 @@
#include "tcoding.h" #include "tcoding.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "indexInt.h" #include "indexInt.h"
#include "index_fst_automation.h"
static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
@ -1093,6 +1094,10 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) {
return fstStreamBuilderCreate(fst, ctx); return fstStreamBuilderCreate(fst, ctx);
} }
StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb) {
if (sb == NULL) { return NULL; }
return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
}
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) {
return fstStreamBuilderCreate(fst, ctx); return fstStreamBuilderCreate(fst, ctx);
} }
@ -1118,7 +1123,7 @@ CompiledAddr fstGetRootAddr(Fst *fst) {
Output fstEmptyFinalOutput(Fst *fst, bool *null) { Output fstEmptyFinalOutput(Fst *fst, bool *null) {
Output res = 0; Output res = 0;
FstNode *node = fst->root; FstNode *node = fstGetRoot(fst);
if (FST_NODE_IS_FINAL(node)) { if (FST_NODE_IS_FINAL(node)) {
*null = false; *null = false;
res = FST_NODE_FINAL_OUTPUT(node); res = FST_NODE_FINAL_OUTPUT(node);
@ -1175,7 +1180,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) {
return bound->type == Included ? true : false; return bound->type == Excluded? false : true;
} }
void fstBoundDestroy(FstBoundWithData *bound) { void fstBoundDestroy(FstBoundWithData *bound) {
@ -1322,6 +1327,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
return swsResultCreate(&s, output, callback(start)); return swsResultCreate(&s, output, callback(start));
} }
} }
SArray *nodes = taosArrayInit(8, sizeof(FstNode *));
while (taosArrayGetSize(sws->stack) > 0) { while (taosArrayGetSize(sws->stack) > 0) {
StreamState *p = (StreamState *)taosArrayPop(sws->stack); StreamState *p = (StreamState *)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) {
@ -1337,8 +1343,8 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp);
void* tState = callback(nextState); void* tState = callback(nextState);
bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState);
//bool isMatch = sws->aut->isMatch(nextState);
FstNode *nextNode = fstGetNode(sws->fst, trn.addr); FstNode *nextNode = fstGetNode(sws->fst, trn.addr);
taosArrayPush(nodes, &nextNode);
taosArrayPush(sws->inp, &(trn.inp)); taosArrayPush(sws->inp, &(trn.inp));
if (FST_NODE_IS_FINAL(nextNode)) { if (FST_NODE_IS_FINAL(nextNode)) {
@ -1354,26 +1360,35 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState};
taosArrayPush(sws->stack, &s2); taosArrayPush(sws->stack, &s2);
uint8_t *buf = (uint8_t *)malloc(taosArrayGetSize(sws->inp) * sizeof(uint8_t));
for (uint32_t i = 0; i < taosArrayGetSize(sws->inp); i++) { size_t isz = taosArrayGetSize(sws->inp);
uint8_t *t = (uint8_t *)taosArrayGet(sws->inp, i); uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t));
buf[i] = *t; for (uint32_t i = 0; i < isz; i++) {
buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i);
} }
FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp));
if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { if (fstBoundWithDataExceededBy(sws->endAt, &slice)) {
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState));
free(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
return NULL; return NULL;
} }
if (FST_NODE_IS_FINAL(nextNode) && isMatch) { if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
StreamWithStateResult *result = swsResultCreate(&slice, fOutput , tState); StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState);
free(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
return result; return result;
} }
free(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
} }
for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
FstNode** node = (FstNode **)taosArrayGet(nodes, i);
fstNodeDestroy(*node);
}
taosArrayDestroy(nodes);
return NULL; return NULL;
} }
@ -1392,6 +1407,7 @@ void swsResultDestroy(StreamWithStateResult *result) {
if (NULL == result) { return; } if (NULL == result) { return; }
fstSliceDestroy(&result->data); fstSliceDestroy(&result->data);
startWithStateValueDestroy(result->state);
free(result); free(result);
} }

102
source/libs/index/src/index_fst_automation.c
View File

@ -16,21 +16,81 @@
#include "index_fst_automation.h" #include "index_fst_automation.h"
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; }
nsv->kind = kind;
nsv->type = ty;
if (ty == FST_INT) {
nsv->val = *(int *)val;
} else if (ty == FST_CHAR) {
size_t len = strlen((char *)val);
nsv->ptr = (char *)calloc(1, len + 1);
memcpy(nsv->ptr, val, len);
} else if (ty == FST_ARRAY) {
//TODO,
//nsv->arr = taosArrayFromList()
}
return nsv;
}
void startWithStateValueDestroy(void *val) {
StartWithStateValue *sv = (StartWithStateValue *)val;
if (sv == NULL) { return; }
if (sv->type == FST_INT) {
//
} else if (sv->type == FST_CHAR) {
free(sv->ptr);
} else if (sv->type == FST_ARRAY) {
taosArrayDestroy(sv->arr);
}
free(sv);
}
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; }
nsv->kind = sv->kind;
nsv->type= sv->type;
if (nsv->type == FST_INT) {
nsv->val = sv->val;
} else if (nsv->type == FST_CHAR) {
size_t len = strlen(sv->ptr);
nsv->ptr = (char *)calloc(1, len + 1);
memcpy(nsv->ptr, sv->ptr, len);
} else if (nsv->type == FST_ARRAY) {
}
return nsv;
}
// prefix query, impl later // prefix query, impl later
static void* prefixStart(AutomationCtx *ctx) { static void* prefixStart(AutomationCtx *ctx) {
StartWithStateValue *data = (StartWithStateValue *)(ctx->data); StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata);
return data; return startWithStateValueDump(data);
}; };
static bool prefixIsMatch(AutomationCtx *ctx, void *data) { static bool prefixIsMatch(AutomationCtx *ctx, void *sv) {
return true; StartWithStateValue* ssv = (StartWithStateValue *)sv;
return ssv->val == strlen(ctx->data);
} }
static bool prefixCanMatch(AutomationCtx *ctx, void *data) { static bool prefixCanMatch(AutomationCtx *ctx, void *sv) {
return true; StartWithStateValue* ssv = (StartWithStateValue *)sv;
return ssv->val >= 0;
} }
static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) {
return true; return true;
} }
static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
StartWithStateValue* ssv = (StartWithStateValue *)state;
if (ssv == NULL || ctx == NULL) {return NULL;}
char *data = ctx->data;
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1;
return startWithStateValueCreate(Running, FST_INT, &val);
}
return NULL; return NULL;
} }
static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { static void* prefixAcceptEof(AutomationCtx *ctx, void *state) {
@ -79,28 +139,34 @@ AutomationFunc automFuncs[] = {{
// add more search type // add more search type
}; };
AutomationCtx* automCtxCreate(void *data, AutomationType type) { AutomationCtx* automCtxCreate(void *data,AutomationType atype) {
AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) { return NULL; }
if (type == AUTOMATION_PREFIX) { StartWithStateValue *sv = NULL;
StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue)); if (atype == AUTOMATION_PREFIX) {
swsv->kind = Done; int val = 0;
swsv->value = NULL; sv = startWithStateValueCreate(Running, FST_INT, &val);
ctx->data = (void *)swsv; ctx->stdata = (void *)sv;
} else if (type == AUTMMATION_MATCH) { } else if (atype == AUTMMATION_MATCH) {
} else { } else {
// add more search type // add more search type
} }
ctx->type = type; char* src = (char *)data;
size_t len = strlen(src);
char* dst = (char *)malloc(len * sizeof(char) + 1);
memcpy(dst, src, len);
dst[len] = 0;
ctx->data = dst;
ctx->type = atype;
ctx->stdata = (void *)sv;
return ctx; return ctx;
} }
void automCtxDestroy(AutomationCtx *ctx) { void automCtxDestroy(AutomationCtx *ctx) {
if (ctx->type == AUTOMATION_PREFIX) { startWithStateValueDestroy(ctx->stdata);
free(ctx->data); free(ctx->data);
} else if (ctx->type == AUTMMATION_MATCH) {
}
free(ctx); free(ctx);
} }

55
source/libs/index/test/indexTests.cpp
View File

@ -59,9 +59,22 @@ class FstReadMemory {
return ok; return ok;
} }
// add later // add later
bool Search(const std::string &key, std::vector<uint64_t> &result) { bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) {
FstStreamBuilder *sb = fstSearch(_fst, ctx);
StreamWithState *st = streamBuilderIntoStream(sb);
StreamWithStateResult *rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
result.push_back((uint64_t)(rt->out.out));
}
return true; return true;
} }
bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector<uint64_t> &result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() { ~FstReadMemory() {
fstCountingWriterDestroy(_w); fstCountingWriterDestroy(_w);
@ -186,11 +199,43 @@ void checkFstPerf() {
printf("success to init fst read"); printf("success to init fst read");
} }
Performance_fstReadRecords(m); Performance_fstReadRecords(m);
delete m; delete m;
} }
void checkFstPrefixSearch() {
FstWriter *fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
std::string key("ab");
for (int i = 0; i < count; i++) {
key[1] = key[1] + i;
fw->Put(key, i);
}
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
delete fw;
FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
// prefix search
std::vector<uint64_t> result;
AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == count);
for (int i = 0; i < result.size(); i++) {
assert(result[i] == i); // check result
}
free(ctx);
delete m;
}
void validateFst() { void validateFst() {
int val = 100; int val = 100;
int count = 100; int count = 100;
@ -209,6 +254,8 @@ void validateFst() {
FstReadMemory *m = new FstReadMemory(1024 * 64); FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init() == false) { if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl; std::cout << "init readMemory failed" << std::endl;
delete m;
return;
} }
{ {
@ -230,10 +277,12 @@ void validateFst() {
} }
} }
delete m; delete m;
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {
checkFstPerf(); checkFstPerf();
//checkFstPrefixSearch();
return 1; return 1;
} }

4
source/libs/parser/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries( target_link_libraries(
parser parser
PRIVATE os util common catalog function transport PRIVATE os util common catalog function transport query
) )
ADD_SUBDIRECTORY(test) ADD_SUBDIRECTORY(test)

4
source/libs/parser/inc/parserInt.h
View File

@ -88,13 +88,13 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf);
* @param msgBufLen * @param msgBufLen
* @return * @return
*/ */
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen); int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen);
/** /**
* Destroy the meta data request structure. * Destroy the meta data request structure.
* @param pMetaInfo * @param pMetaInfo
*/ */
void qParserClearupMetaRequestInfo(SMetaReq* pMetaInfo); void qParserClearupMetaRequestInfo(SCatalogReq* pMetaInfo);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/libs/parser/src/astValidate.c
View File

@ -4029,18 +4029,18 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
} }
SMetaReq req = {0}; SCatalogReq req = {0};
SMetaData data = {0}; SMetaData data = {0};
// TODO: check if the qnode info has been cached already // TODO: check if the qnode info has been cached already
req.qNodeEpset = true; req.qNodeRequired = true;
code = qParserExtractRequestedMetaInfo(pInfo, &req, msgBuf, msgBufLen); code = qParserExtractRequestedMetaInfo(pInfo, &req, msgBuf, msgBufLen);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
// load the meta data from catalog // load the meta data from catalog
code = catalogGetMetaData(pCatalog, &req, &data); code = catalogGetAllMeta(pCatalog, NULL, NULL, &req, &data);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }

36
source/libs/parser/src/insertParser.c
View File

@ -71,8 +71,7 @@ typedef struct SInsertParseContext {
const char* pSql; const char* pSql;
SMsgBuf msg; SMsgBuf msg;
struct SCatalog* pCatalog; struct SCatalog* pCatalog;
SMetaData meta; // need release STableMeta* pTableMeta;
const STableMeta* pTableMeta;
SHashObj* pTableBlockHashObj; // data block for each table. need release SHashObj* pTableBlockHashObj; // data block for each table. need release
int32_t totalNum; int32_t totalNum;
SInsertStmtInfo* pOutput; SInsertStmtInfo* pOutput;
@ -165,29 +164,29 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullDbName, char* tableName) {
if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z);
} }
SName name = {0}; char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false);
strcpy(name.dbname, pCxt->pComCxt->pDbname); if (NULL != p) { // db.table
strncpy(name.tname, pStname->z, pStname->n); strcpy(fullDbName, pCxt->pComCxt->pAcctId);
taosArrayPush(tableNameList, &name); fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0];
strncpy(fullDbName, pStname->z, p - pStname->z);
strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1);
} else {
snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname);
strncpy(tableName, pStname->z, pStname->n);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SMetaReq* pMetaReq) {
pMetaReq->pTableName = taosArrayInit(4, sizeof(SName));
return buildTableName(pCxt, pStname, pMetaReq->pTableName);
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
SMetaReq req; char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0};
CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); char tableName[TSDB_TABLE_NAME_LEN] = {0};
CHECK_CODE(catalogGetMetaData(pCxt->pCatalog, &req, &pCxt->meta)); CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName));
pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -861,7 +860,7 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
.pComCxt = pContext, .pComCxt = pContext,
.pSql = pContext->pSql, .pSql = pContext->pSql,
.msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
.pCatalog = getCatalogHandle(pContext->pEpSet), .pCatalog = NULL,
.pTableMeta = NULL, .pTableMeta = NULL,
.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false),
.totalNum = 0, .totalNum = 0,
@ -872,6 +871,7 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog));
CHECK_CODE(skipInsertInto(&context)); CHECK_CODE(skipInsertInto(&context));
CHECK_CODE(parseInsertBody(&context)); CHECK_CODE(parseInsertBody(&context));

10
source/libs/parser/src/parser.c
View File

@ -51,9 +51,9 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ
// do nothing // do nothing
} }
} else { } else {
struct SCatalog* pCatalog = getCatalogHandle(NULL); struct SCatalog* pCatalog = NULL;
int32_t code = catalogGetHandle(NULL, &pCatalog);
int32_t code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
*pOutput = pQueryInfo; *pOutput = pQueryInfo;
} }
@ -147,7 +147,7 @@ static void freePtrElem(void* p) {
tfree(*(char**)p); tfree(*(char**)p);
} }
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen) { int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen};
@ -204,7 +204,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet
return code; return code;
} }
void qParserClearupMetaRequestInfo(SMetaReq* pMetaReq) { void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) {
if (pMetaReq == NULL) { if (pMetaReq == NULL) {
return; return;
} }

40
source/libs/parser/src/parserUtil.c
View File

@ -1458,23 +1458,6 @@ void* vgroupInfoClear(SVgroupsInfo *vgroupList) {
return NULL; return NULL;
} }
char* serializeTagData(STagData* pTagData, char* pMsg) {
int32_t n = (int32_t) strlen(pTagData->name);
*(int32_t*) pMsg = htonl(n);
pMsg += sizeof(n);
memcpy(pMsg, pTagData->name, n);
pMsg += n;
*(int32_t*)pMsg = htonl(pTagData->dataLen);
pMsg += sizeof(int32_t);
memcpy(pMsg, pTagData->data, pTagData->dataLen);
pMsg += pTagData->dataLen;
return pMsg;
}
int32_t copyTagData(STagData* dst, const STagData* src) { int32_t copyTagData(STagData* dst, const STagData* src) {
dst->dataLen = src->dataLen; dst->dataLen = src->dataLen;
tstrncpy(dst->name, src->name, tListLen(dst->name)); tstrncpy(dst->name, src->name, tListLen(dst->name));
@ -1491,29 +1474,6 @@ int32_t copyTagData(STagData* dst, const STagData* src) {
return 0; return 0;
} }
STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
assert(pChild != NULL);
int32_t total = pChild->numOfColumns + pChild->numOfTags;
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
pTableMeta->tableType = TSDB_SUPER_TABLE;
pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
pTableMeta->tableInfo.precision = pChild->precision;
pTableMeta->uid = pChild->suid;
pTableMeta->tversion = pChild->tversion;
pTableMeta->sversion = pChild->sversion;
memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total);
int32_t num = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < num; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
return pTableMeta;
}
uint32_t getTableMetaSize(const STableMeta* pTableMeta) { uint32_t getTableMetaSize(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL); assert(pTableMeta != NULL);

11
source/libs/parser/test/CMakeLists.txt
View File

@ -6,13 +6,16 @@ SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(parserTest ${SOURCE_LIST}) ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner
)
TARGET_INCLUDE_DIRECTORIES( TARGET_INCLUDE_DIRECTORIES(
parserTest parserTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/" PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc"
) )
TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner query
)
TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc)

21
source/libs/parser/test/insertTest.cpp
View File

@ -27,6 +27,27 @@ namespace {
} }
} }
extern "C" {
#include <execinfo.h>
void *__real_malloc(size_t);
void *__wrap_malloc(size_t c) {
// printf("My MALLOC called: %d\n", c);
// void *array[32];
// int size = backtrace(array, 32);
// char **symbols = backtrace_symbols(array, size);
// for (int i = 0; i < size; ++i) {
// cout << symbols[i] << endl;
// }
// free(symbols);
return __real_malloc(c);
}
}
// syntax: // syntax:
// INSERT INTO // INSERT INTO
// tb_name // tb_name

2
source/libs/parser/test/mockCatalog.cpp
View File

@ -48,6 +48,6 @@ struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
return mockCatalogService->getCatalogHandle(pMgmtEps); return mockCatalogService->getCatalogHandle(pMgmtEps);
} }
int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) {
return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData);
} }

Some files were not shown because too many files have changed in this diff Show More