diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index ca250ab169..09c73145a1 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -70,11 +70,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_DB, "drop-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_USE_DB, "use-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_DB, "alter-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_DB, "sync-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_TOPIC, "create-topic" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_TOPIC, "drop-topic" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_TOPIC, "alter-topic" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_FUNCTION, "alter-function" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) @@ -86,8 +84,6 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW_RETRIEVE, "retrieve" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC, "retrieve-func" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE, "compact-vnode" ) // message from client to qnode // message from client to dnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) @@ -163,7 +159,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, TSDB_MGMT_TABLE_VGROUP, - TSDB_MGMT_TABLE_METRIC, + TSDB_MGMT_TABLE_STABLE, TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, @@ -276,14 +272,12 @@ typedef struct { SShellSubmitRspBlock failedBlocks[]; } SShellSubmitRspMsg; -//#if 0 typedef struct SSchema { - uint8_t type; - char name[TSDB_COL_NAME_LEN]; - int16_t colId; + int8_t type; + int32_t colId; int32_t bytes; + char name[TSDB_COL_NAME_LEN]; } SSchema; -//#endif typedef struct { int32_t contLen; @@ -571,7 +565,6 @@ typedef struct { char db[TSDB_FULL_DB_NAME_LEN]; int32_t cacheBlockSize; // MB int32_t totalBlocks; - int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -580,58 +573,93 @@ typedef struct { int32_t maxRowsPerFileBlock; int32_t commitTime; int32_t fsyncPeriod; + int8_t walLevel; int8_t precision; // time resolution int8_t compression; - int8_t walLevel; int8_t replications; int8_t quorum; - int8_t ignoreExist; int8_t update; int8_t cacheLastRow; - int8_t dbType; - int16_t partitions; - int8_t reserve[5]; -} SCreateDbMsg, SAlterDbMsg; + int8_t ignoreExist; + int32_t reserve[8]; +} SCreateDbMsg; + +typedef struct { + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t totalBlocks; + int32_t daysToKeep0; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t fsyncPeriod; + int8_t walLevel; + int8_t quorum; + int8_t cacheLastRow; + int32_t reserve[8]; +} SAlterDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; + int32_t reserve[8]; +} SDropDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; + int32_t reserve[8]; +} SUseDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int32_t reserve[8]; +} SSyncDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int32_t reserve[8]; +} SCompactDbMsg; typedef struct { char name[TSDB_FUNC_NAME_LEN]; - char path[PATH_MAX]; - int32_t funcType; + int8_t funcType; + int8_t scriptType; + int8_t align; int8_t outputType; - int16_t outputLen; + int32_t outputLen; int32_t bufSize; - int32_t codeLen; - char code[]; + int64_t sigature; + int32_t commentSize; + int32_t codeSize; + char pCont[]; } SCreateFuncMsg; typedef struct { - int32_t num; - char name[]; + char name[TSDB_FUNC_NAME_LEN]; +} SDropFuncMsg; + +typedef struct { + int32_t numOfFuncs; + char pFuncNames[]; } SRetrieveFuncMsg; typedef struct { char name[TSDB_FUNC_NAME_LEN]; - int32_t funcType; - int8_t resType; - int16_t resBytes; + int8_t funcType; + int8_t scriptType; + int8_t align; + int8_t outputType; + int32_t outputLen; int32_t bufSize; - int32_t len; - char content[]; -} SFunctionInfoMsg; + int64_t sigature; + int32_t commentSize; + int32_t codeSize; + char pCont[]; +} SFuncInfo; typedef struct { - int32_t num; - char content[]; -} SUdfFuncMsg; - -typedef struct { - char name[TSDB_FUNC_NAME_LEN]; -} SDropFuncMsg; - -typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; - int8_t ignoreNotExists; -} SDropDbMsg, SUseDbMsg, SSyncDbMsg; + int32_t numOfFuncs; + char pFuncInfos[]; +} SRetrieveFuncRsp; typedef struct { int32_t statusInterval; @@ -736,13 +764,17 @@ typedef struct { } SAuthVnodeMsg; typedef struct { - char tableFname[TSDB_TABLE_FNAME_LEN]; - int16_t createFlag; - char tags[]; + char name[TSDB_TABLE_FNAME_LEN]; +} SStableInfoMsg; + +typedef struct { + char tableFname[TSDB_TABLE_FNAME_LEN]; + int8_t createFlag; + char tags[]; } STableInfoMsg; typedef struct { - uint8_t metaClone; // create local clone of the cached table meta + int8_t metaClone; // create local clone of the cached table meta int32_t numOfVgroups; int32_t numOfTables; int32_t numOfUdfs; @@ -760,25 +792,24 @@ typedef struct { } SVgroupMsg; typedef struct { - int32_t numOfVgroups; + int32_t numOfVgroups; SVgroupMsg vgroups[]; } SVgroupsMsg, SVgroupsInfo; -typedef struct STableMetaMsg { - int32_t contLen; +typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; // table id - int8_t numOfTags; + char stableFname[TSDB_TABLE_FNAME_LEN]; + int32_t numOfTags; + int32_t numOfColumns; int8_t precision; int8_t tableType; - int16_t numOfColumns; - int16_t sversion; - int16_t tversion; - int32_t tid; - int64_t uid; + int8_t update; + int32_t sversion; + int32_t tversion; + uint64_t tuid; + uint64_t suid; SVgroupMsg vgroup; - char sTableName[TSDB_TABLE_FNAME_LEN]; - int64_t suid; - SSchema schema[]; + SSchema pSchema[]; } STableMetaMsg; typedef struct SMultiTableMeta { @@ -823,14 +854,31 @@ typedef struct SShowRsp { typedef struct { char ep[TSDB_EP_LEN]; // end point, hostname:port -} SCreateDnodeMsg, SDropDnodeMsg; +} SCreateDnodeMsg; + +typedef struct { + int32_t dnodeId; +} SDropDnodeMsg; + +typedef struct { + int32_t dnodeId; + char config[128]; +} SCfgDnodeMsg; + +typedef struct { + int32_t dnodeId; +} SCreateMnodeMsg, SDropMnodeMsg; typedef struct { int32_t dnodeId; + int8_t align[3]; int8_t replica; - int8_t reserved[3]; SReplica replicas[TSDB_MAX_REPLICA]; -} SCreateMnodeMsg, SAlterMnodeMsg, SDropMnodeMsg; +} SCreateMnodeInMsg, SAlterMnodeInMsg; + +typedef struct { + int32_t dnodeId; +} SDropMnodeInMsg; typedef struct { int32_t dnodeId; @@ -843,11 +891,6 @@ typedef struct { int32_t vgId; } SConfigVnodeMsg; -typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port - char config[64]; -} SCfgDnodeMsg; - typedef struct { char sql[TSDB_SHOW_SQL_LEN]; int32_t queryId; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 8bd3db0217..d1f7d9a958 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -65,6 +65,16 @@ extern "C" { dataPos += valLen; \ } +#define SDB_GET_RESERVE(pRaw, pRow, dataPos, valLen) \ + { \ + char val[valLen] = {0}; \ + if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ + sdbFreeRow(pRow); \ + return NULL; \ + } \ + dataPos += valLen; \ + } + #define SDB_SET_INT64(pRaw, dataPos, val) \ { \ if (sdbSetRawInt64(pRaw, dataPos, val) != 0) { \ @@ -110,6 +120,16 @@ extern "C" { dataPos += valLen; \ } +#define SDB_SET_RESERVE(pRaw, dataPos, valLen) \ + { \ + char val[valLen] = {0}; \ + if (sdbSetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ + sdbFreeRaw(pRaw); \ + return NULL; \ + } \ + dataPos += valLen; \ + } + #define SDB_SET_DATALEN(pRaw, dataLen) \ { \ if (sdbSetRawDataLen(pRaw, dataLen) != 0) { \ @@ -258,7 +278,7 @@ int32_t sdbDeploy(SSdb *pSdb); int32_t sdbReadFile(SSdb *pSdb); /** - * @brief Parse and write raw data to sdb. + * @brief Parse and write raw data to sdb, then free the pRaw object * * @param pSdb The sdb object. * @param pRaw The raw data. @@ -266,6 +286,15 @@ int32_t sdbReadFile(SSdb *pSdb); */ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw); +/** + * @brief Parse and write raw data to sdb. + * + * @param pSdb The sdb object. + * @param pRaw The raw data. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw); + /** * @brief Acquire a row from sdb * diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index b514648bbd..f84297670b 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -55,12 +55,14 @@ typedef struct { uint32_t signature; uint32_t cksumHead; uint32_t cksumBody; - //char cont[]; + char cont[]; } SWalHead; typedef struct { int32_t vgId; int32_t fsyncPeriod; // millisecond + int32_t rollPeriod; + int64_t segSize; EWalType walLevel; // wal level } SWalCfg; @@ -87,36 +89,41 @@ typedef struct SWal { // cfg int32_t vgId; int32_t fsyncPeriod; // millisecond - int32_t fsyncSeq; int32_t rollPeriod; // second int64_t segSize; + int64_t retentionSize; + int32_t retentionPeriod; EWalType level; + //total size + int64_t totSize; + //fsync seq + int32_t fsyncSeq; //reference int64_t refId; - //current tfd - int64_t curLogTfd; - int64_t curIdxTfd; + //write tfd + int64_t writeLogTfd; + int64_t writeIdxTfd; + //read tfd + int64_t readLogTfd; + int64_t readIdxTfd; //current version int64_t curVersion; - int64_t curLogOffset; - //current file version - int64_t curFileFirstVersion; - int64_t curFileLastVersion; - //wal fileset version + //wal lifecycle int64_t firstVersion; int64_t snapshotVersion; + int64_t commitVersion; int64_t lastVersion; - int64_t lastFileName; //roll status int64_t lastRollSeq; - int64_t lastFileWriteSize; + //file set + int32_t writeCur; + int32_t readCur; + SArray* fileInfoSet; //ctl int32_t curStatus; pthread_mutex_t mutex; //path char path[WAL_PATH_LEN]; - //file set - SArray* fileSet; //reusable write head SWalHead head; } SWal; // WAL HANDLE @@ -133,7 +140,7 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write -int64_t walWrite(SWal *, int64_t index, uint8_t msgType, void *body, int32_t bodyLen); +int64_t walWrite(SWal *, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen); void walFsync(SWal *, bool force); // apis for lifecycle management diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f6cefa96df..36301466f8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -183,6 +183,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") +// mnode-table #define TSDB_CODE_MND_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) //"Table already exists") #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") #define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") @@ -200,32 +201,41 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") #define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) //"Exceed max row bytes") -#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) //"Invalid func name") -#define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) //"Invalid func length") -#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) //"Invalid func code") -#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) //"Func already exists") -#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func") -#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize") +#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370) +#define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371) +#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0372) +#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0373) +#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0374) +#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0375) +#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0376) #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") -#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") -#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") -#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") -#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) //"Invalid database name") -#define TSDB_CODE_MND_MONITOR_DB_FORBIDDEN TAOS_DEF_ERROR_CODE(0, 0x0384) //"Cannot delete monitor database") +#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) +#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) +#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0383) #define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account") -#define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) //"Database not available") -#define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) //"Database unsynced") - -#define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) //"Invalid database option: days out of range") -#define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) //"Invalid database option: keep >= keep1 >= keep0 >= days") - -#define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) //"Invalid topic name) -#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x0393) //"Invalid topic option) -#define TSDB_CODE_MND_INVALID_TOPIC_PARTITONS TAOS_DEF_ERROR_CODE(0, 0x0394) //"Invalid topic partitons num, valid range: [1, 1000]) -#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) //"Topic already exists) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/include/util/tarray.h b/include/util/tarray.h index 5807c980e0..25862a7119 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -146,6 +146,13 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData); */ void taosArraySet(SArray* pArray, size_t index, void* pData); +/** + * remove some data entry from front + * @param pArray + * @param cnt + */ +void taosArrayPopFrontBatch(SArray* pArray, size_t cnt); + /** * remove data entry of the given index * @param pArray diff --git a/include/util/tdef.h b/include/util/tdef.h index 5be7df2c14..278c22512c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -163,9 +163,14 @@ do { \ #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 + +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_COMMENT_LEN 4096 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_TYPE_SCALAR 1 +#define TSDB_FUNC_TYPE_AGGREGATE 2 + #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_FULL_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) #define TSDB_COL_NAME_LEN 65 @@ -245,12 +250,6 @@ do { \ #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 10000000 -#define TSDB_DEFAULT_TABLES 1000000 -#define TSDB_TABLES_STEP 1000 -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default - #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 #define TSDB_DEFAULT_DAYS_PER_FILE 10 @@ -259,18 +258,26 @@ do { \ #define TSDB_MAX_KEEP 365000 // data in db to be reserved. #define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 #define TSDB_MIN_MIN_ROW_FBLOCK 10 #define TSDB_MAX_MIN_ROW_FBLOCK 1000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 #define TSDB_MIN_MAX_ROW_FBLOCK 200 #define TSDB_MAX_MAX_ROW_FBLOCK 10000 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 #define TSDB_MIN_COMMIT_TIME 30 #define TSDB_MAX_COMMIT_TIME 40960 #define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second + +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 + #define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO #define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI @@ -279,9 +286,13 @@ do { \ #define TSDB_MAX_COMP_LEVEL 2 #define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 + +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 #define TSDB_MIN_DB_UPDATE 0 #define TSDB_MAX_DB_UPDATE 2 @@ -291,22 +302,6 @@ do { \ #define TSDB_MAX_DB_CACHE_LAST_ROW 3 #define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second - -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 - -#define TSDB_MIN_DB_PARTITON_OPTION 0 -#define TSDB_MAX_DB_PARTITON_OPTION 1000 -#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 - -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 - #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 @@ -320,6 +315,11 @@ do { \ #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode + +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default + + + /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag diff --git a/include/util/tfile.h b/include/util/tfile.h index 3d0e2177ac..af4c19e7d1 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -16,6 +16,8 @@ #ifndef _TD_UTIL_FILE_H #define _TD_UTIL_FILE_H +#include "os.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/include/util/tmd5.h b/include/util/tmd5.h index 329f4acf11..f8114ad57b 100644 --- a/include/util/tmd5.h +++ b/include/util/tmd5.h @@ -30,10 +30,10 @@ typedef struct { uint32_t buf[4]; /* scratch buffer */ uint8_t in[64]; /* input buffer */ uint8_t digest[16]; /* actual digest after MD5Final call */ -} MD5_CTX; +} T_MD5_CTX; -void MD5Init(MD5_CTX *mdContext); -void MD5Update(MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen); -void MD5Final(MD5_CTX *mdContext); +void tMD5Init(T_MD5_CTX *mdContext); +void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen); +void tMD5Final(T_MD5_CTX *mdContext); #endif /*_TD_UTIL_MD5_H*/ diff --git a/include/util/tutil.h b/include/util/tutil.h index 189cc2b728..5f6746c92c 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -21,22 +21,22 @@ extern "C" { #endif #include "os.h" -#include "tmd5.h" #include "tcrc32c.h" #include "tdef.h" +#include "tmd5.h" int32_t strdequote(char *src); -int32_t strndequote(char *dst, const char* z, int32_t len); +int32_t strndequote(char *dst, const char *z, int32_t len); int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); -char * strnchr(char *haystack, char needle, int32_t len, bool skipquote); -char ** strsplit(char *src, const char *delim, int32_t *num); -char * strtolower(char *dst, const char *src); -char * strntolower(char *dst, const char *src, int32_t n); -char * strntolower_s(char *dst, const char *src, int32_t n); +char *strnchr(char *haystack, char needle, int32_t len, bool skipquote); +char **strsplit(char *src, const char *delim, int32_t *num); +char *strtolower(char *dst, const char *src); +char *strntolower(char *dst, const char *src, int32_t n); +char *strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); -char * strbetween(char *string, char *begin, char *end); -char * paGetToken(char *src, char **token, int32_t *tokenLen); +char *strbetween(char *string, char *begin, char *end); +char *paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); @@ -47,19 +47,19 @@ void taosIp2String(uint32_t ip, char *str); void taosIpPort2String(uint32_t ip, uint16_t port, char *str); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, inBuf, (unsigned int)inLen); - MD5Final(&context); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, inBuf, (unsigned int)inLen); + tMD5Final(&context); memcpy(target, context.digest, tListLen(context.digest)); } static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, inBuf, (unsigned int)len); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, inBuf, (unsigned int)len); + tMD5Final(&context); - MD5Final(&context); sprintf(target, "%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x", context.digest[0], context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], @@ -70,4 +70,4 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar } #endif -#endif /*_TD_UTIL_UTIL_H*/ +#endif /*_TD_UTIL_UTIL_H*/ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2e68fb2742..8821390b82 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -143,13 +143,9 @@ int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int16_t tsPartitons = TSDB_DEFAULT_DB_PARTITON_OPTION; int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; int32_t tsMaxVgroupsPerDb = 0; -int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; -int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; -int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; // tsdb config @@ -727,37 +723,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // database configs - cfg.option = "maxTablesPerVnode"; - cfg.ptr = &tsMaxTablePerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES; - cfg.maxValue = TSDB_MAX_TABLES; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minTablesPerVnode"; - cfg.ptr = &tsMinTablePerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES; - cfg.maxValue = TSDB_MAX_TABLES; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tableIncStepPerVnode"; - cfg.ptr = &tsTableIncStepPerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES; - cfg.maxValue = TSDB_MAX_TABLES; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "cache"; cfg.ptr = &tsCacheBlockSize; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -868,16 +833,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "partitions"; - cfg.ptr = &tsPartitons; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_PARTITON_OPTION; - cfg.maxValue = TSDB_MAX_DB_PARTITON_OPTION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "quorum"; cfg.ptr = &tsQuorum; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 374a2f2b2c..d3336fb079 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -366,7 +366,7 @@ static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); } -static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SCreateMnodeMsg *pMsg) { +static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SCreateMnodeInMsg *pMsg) { dndInitMnodeOption(pDnode, pOption); pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); @@ -488,8 +488,8 @@ static int32_t dndDropMnode(SDnode *pDnode) { return 0; } -static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { - SCreateMnodeMsg *pMsg = pRpcMsg->pCont; +static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { + SCreateMnodeInMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); for (int32_t i = 0; i < pMsg->replica; ++i) { pMsg->replicas[i].id = htonl(pMsg->replicas[i].id); @@ -500,7 +500,7 @@ static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { } static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + SCreateMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -516,7 +516,7 @@ static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + SAlterMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -531,7 +531,8 @@ static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + SDropMnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 07bbe6c0f6..2b8418b12d 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -65,11 +65,9 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_USE_DB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_DB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_DB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg; @@ -81,8 +79,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dndProcessMnodeWriteMsg; // message from client to dnode pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index e7aa103996..b3b6818a5c 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,4 +1,5 @@ -add_subdirectory(acct) -add_subdirectory(cluster) -add_subdirectory(profile) -add_subdirectory(show) +# add_subdirectory(acct) +# add_subdirectory(cluster) +# add_subdirectory(profile) +# add_subdirectory(show) +add_subdirectory(user) diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index d7f9bca5ed..4e71a8016c 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -93,21 +93,21 @@ TEST_F(DndTestCluster, ShowCluster) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "id"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "name"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 14e8dda457..a4632547a2 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -152,49 +152,49 @@ TEST_F(DndTestProfile, SConnectMsg_03) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "program"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "pid"); - pSchema = &pMeta->schema[4]; + pSchema = &pMeta->pSchema[4]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "ip:port"); - pSchema = &pMeta->schema[5]; + pSchema = &pMeta->pSchema[5]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "login_time"); - pSchema = &pMeta->schema[6]; + pSchema = &pMeta->pSchema[6]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); @@ -493,28 +493,28 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "queryId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); @@ -680,21 +680,21 @@ TEST_F(DndTestProfile, SKillStreamMsg_03) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "streamId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index e8c8fa261c..266f5858e4 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -154,49 +154,49 @@ TEST_F(DndTestShow, SShowMsg_04) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "program"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "pid"); - pSchema = &pMeta->schema[4]; + pSchema = &pMeta->pSchema[4]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "ip:port"); - pSchema = &pMeta->schema[5]; + pSchema = &pMeta->pSchema[5]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "login_time"); - pSchema = &pMeta->schema[6]; + pSchema = &pMeta->pSchema[6]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 8a84733fa3..9d4cf21432 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -33,7 +33,6 @@ void initLog(const char* path) { sDebugFlag = 0; tsdbDebugFlag = 0; cqDebugFlag = 0; - debugFlag = 0; char temp[PATH_MAX]; snprintf(temp, PATH_MAX, "%s/taosdlog", path); diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt new file mode 100644 index 0000000000..b5f02e41f4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestUser "") + +target_sources(dndTestUser + PRIVATE + "user.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestUser + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestUser + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestUser + COMMAND dndTestUser +) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp new file mode 100644 index 0000000000..36f801fec2 --- /dev/null +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestUser : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} + + static void SetUpTestSuite() { + const char* user = "root"; + const char* pass = "taosdata"; + const char* path = "/tmp/dndTestUser"; + const char* fqdn = "localhost"; + uint16_t port = 9524; + + pServer = createServer(path, fqdn, port); + ASSERT(pServer); + pClient = createClient(user, pass, fqdn, port); + } + + static void TearDownTestSuite() { + dropServer(pServer); + dropClient(pClient); + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; +}; + +SServer* DndTestUser::pServer; +SClient* DndTestUser::pClient; +int32_t DndTestUser::connId; + +#if 0 +TEST_F(DndTestUser, ShowUser) { + int32_t showId = 0; + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, "show users"); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 4); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + + pSchema = &pMeta->pSchema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "name"); + + pSchema = &pMeta->pSchema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "privilege"); + + pSchema = &pMeta->pSchema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "create_time"); + + pSchema = &pMeta->pSchema[3]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "account"); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, 2); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + int64_t int64Val = 0; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } + + //--- privilege --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += 10; + EXPECT_STREQ(strVal, "super"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += 10; + EXPECT_STREQ(strVal, "writable"); + } + + //--- create_time --- + { + int64Val = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(int64Val, 0); + + int64Val = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(int64Val, 0); + } + + //--- account --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + } +} +#endif + +TEST_F(DndTestUser, CreateUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- create user --- + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 3); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "u1"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +TEST_F(DndTestUser, AlterUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- drop user --- + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 3); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "u1"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +TEST_F(DndTestUser, DropUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- drop user --- + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + strcpy(pReq->user, "u1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 2); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7da92c84bb..0f19a03cdb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -47,7 +47,6 @@ typedef struct SAcctObj SAcctObj; typedef struct SUserObj SUserObj; typedef struct SDbObj SDbObj; typedef struct SVgObj SVgObj; -typedef struct SSTableObj SSTableObj; typedef struct SFuncObj SFuncObj; typedef struct SOperObj SOperObj; @@ -195,7 +194,6 @@ typedef struct SUserObj { typedef struct { int32_t cacheBlockSize; int32_t totalBlocks; - int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -204,102 +202,91 @@ typedef struct { int32_t maxRowsPerFileBlock; int32_t commitTime; int32_t fsyncPeriod; + int8_t walLevel; int8_t precision; int8_t compression; - int8_t walLevel; int8_t replications; int8_t quorum; int8_t update; int8_t cacheLastRow; - int8_t dbType; - int16_t partitions; } SDbCfg; typedef struct SDbObj { - char name[TSDB_FULL_DB_NAME_LEN]; - char acct[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - SDbCfg cfg; - int64_t uid; - int8_t status; - int32_t numOfVgroups; - int32_t numOfTables; - int32_t numOfSuperTables; - int32_t vgListSize; - int32_t vgListIndex; - SVgObj **vgList; - SAcctObj *pAcct; + char name[TSDB_FULL_DB_NAME_LEN]; + char acct[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + SDbCfg cfg; } SDbObj; typedef struct { int32_t dnodeId; - int8_t role; - SDnodeObj *pDnode; + ESyncState role; } SVnodeGid; typedef struct SVgObj { - uint32_t vgId; - int32_t numOfVnodes; + int32_t vgId; int64_t createdTime; int64_t updateTime; - int32_t lbDnodeId; - int32_t lbTime; + int32_t version; char dbName[TSDB_FULL_DB_NAME_LEN]; - int8_t inUse; - int8_t accessState; - int8_t status; - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; - int32_t vgCfgVersion; - int8_t compact; int32_t numOfTables; + int32_t numOfTimeSeries; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; - SDbObj *pDb; + int8_t compact; + int8_t replica; + SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; -typedef struct SSTableObj { - char tableId[TSDB_TABLE_NAME_LEN]; - uint64_t uid; +typedef struct SStableObj { + char name[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; int64_t createdTime; int64_t updateTime; - int32_t numOfColumns; // used by normal table + uint64_t uid; + int32_t version; + int32_t numOfColumns; int32_t numOfTags; - SSchema *schema; -} SSTableObj; + SRWLatch lock; + SSchema *columnSchema; + SSchema *tagSchema; +} SStableObj; typedef struct SFuncObj { char name[TSDB_FUNC_NAME_LEN]; - char path[128]; - int32_t contLen; - char cont[TSDB_FUNC_CODE_LEN]; - int32_t funcType; - int32_t bufSize; int64_t createdTime; - uint8_t resType; - int16_t resBytes; - int64_t sig; - int16_t type; + int8_t funcType; + int8_t scriptType; + int8_t align; + int8_t outputType; + int32_t outputLen; + int32_t bufSize; + int64_t sigature; + int32_t commentSize; + int32_t codeSize; + char *pComment; + char *pCode; + char pData[]; } SFuncObj; -typedef struct SShowObj SShowObj; -typedef struct SShowObj { - int8_t type; - int8_t maxReplica; - int16_t numOfColumns; - int32_t id; - int32_t rowSize; - int32_t numOfRows; - int32_t numOfReads; - uint16_t payloadLen; - void *pIter; - void *pVgIter; - SMnode *pMnode; - char db[TSDB_FULL_DB_NAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; - char payload[]; +typedef struct { + int32_t id; + int8_t type; + int8_t replica; + int16_t numOfColumns; + int32_t rowSize; + int32_t numOfRows; + int32_t numOfReads; + int32_t payloadLen; + void *pIter; + SMnode *pMnode; + char db[TSDB_FULL_DB_NAME_LEN]; + int16_t offset[TSDB_MAX_COLUMNS]; + int32_t bytes[TSDB_MAX_COLUMNS]; + char payload[]; } SShowObj; typedef struct SMnodeMsg { @@ -318,6 +305,11 @@ typedef struct SMnodeMsg { void *pCont; } SMnodeMsg; +typedef struct { + int32_t id; + void *rpcHandle; +} STransMsg; + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 9bb1ab7acd..04ceb2820a 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -26,6 +26,8 @@ int32_t mndInitDnode(SMnode *pMnode); void mndCleanupDnode(SMnode *pMnode); SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); +SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); +int32_t mndGetDnodeSize(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8d20356e39..b44d6570fe 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -20,6 +20,7 @@ #include "sdb.h" #include "tcache.h" #include "tqueue.h" +#include "ttime.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index 64bbb13b60..906d11aec2 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -26,6 +26,7 @@ int32_t mndInitMnode(SMnode *pMnode); void mndCleanupMnode(SMnode *pMnode); bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); +char *mndGetRoleStr(int32_t role); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndOper.h b/source/dnode/mnode/impl/inc/mndOper.h deleted file mode 100644 index 5ad5059a0f..0000000000 --- a/source/dnode/mnode/impl/inc/mndOper.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_MND_OPER_H_ -#define _TD_MND_OPER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t mndInitOper(SMnode *pMnode); -void mndCleanupOper(SMnode *pMnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_MND_OPER_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index e7a3fcd45f..37fb9159dc 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -28,6 +28,7 @@ void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); +char *mndShowStr(int32_t showType); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index 3e45b0adb1..02ba725be1 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -25,7 +25,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); bool mndIsMaster(SMnode *pMnode); -int32_t mndSyncPropose(SSdbRaw *pRaw, void *pData); +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 9559a96255..878337e4be 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -33,13 +33,10 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg); int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg); -int32_t mndTransPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)); -int32_t mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, void *pData, int32_t code); +int32_t mndTransPrepare(STrans *pTrans); +void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); int32_t mndTransExecute(SSdb *pSdb, int32_t tranId); -SSdbRaw *mndTransActionEncode(STrans *pTrans); -SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index c75bdb5949..21aee0f0a4 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -24,6 +24,8 @@ extern "C" { int32_t mndInitVgroup(SMnode *pMnode); void mndCleanupVgroup(SMnode *pMnode); +SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); +void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index e3d37cd9f9..a365ad7326 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -24,7 +24,7 @@ static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct); +static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); @@ -65,7 +65,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("acct:%s, will be created while deploy sdb", acctObj.acct); + mDebug("acct:%s, will be created while deploy sdb", acctObj.acct); return sdbWrite(pMnode->pSdb, pRaw); } @@ -131,15 +131,15 @@ static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { return 0; } -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct) { - mTrace("acct:%s, perform update action", pSrcAcct->acct); +static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { + mTrace("acct:%s, perform update action", pOldAcct->acct); - memcpy(pSrcAcct->acct, pDstAcct->acct, TSDB_USER_LEN); - pSrcAcct->createdTime = pDstAcct->createdTime; - pSrcAcct->updateTime = pDstAcct->updateTime; - pSrcAcct->acctId = pDstAcct->acctId; - pSrcAcct->status = pDstAcct->status; - pSrcAcct->cfg = pDstAcct->cfg; + memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); + pOldAcct->createdTime = pNewAcct->createdTime; + pOldAcct->updateTime = pNewAcct->updateTime; + pOldAcct->acctId = pNewAcct->acctId; + pOldAcct->status = pNewAcct->status; + pOldAcct->cfg = pNewAcct->cfg; return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 9fa7a06630..c5c71d8751 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -24,7 +24,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); -static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pSrcCluster, SClusterObj *pDstCluster); +static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster); static int32_t mndCreateDefaultCluster(SMnode *pMnode); static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); @@ -107,8 +107,8 @@ static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster) { return 0; } -static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pSrcCluster, SClusterObj *pDstCluster) { - mTrace("cluster:%d, perform update action", pSrcCluster->id); +static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster) { + mTrace("cluster:%d, perform update action", pOldCluster->id); return 0; } @@ -132,13 +132,13 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("cluster:%d, will be created while deploy sdb", clusterObj.id); + mDebug("cluster:%d, will be created while deploy sdb", clusterObj.id); return sdbWrite(pMnode->pSdb, pRaw); } static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f01a43dfef..7477213890 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,16 +15,146 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndTrans.h" +#include "mndUser.h" -static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg); +#define TSDB_DB_VER_NUM 1 +#define TSDB_DB_RESERVE_SIZE 64 + +static SSdbRaw *mndDbActionEncode(SDbObj *pDb); +static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); +static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); +static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); +static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb); +static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg); +static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mnodeProcessUseMsg); - return 0; + SSdbTable table = {.sdbType = SDB_DB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DB, mndProcessCreateDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_DB, mndProcessAlterDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DB, mndProcessDropDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mndProcessUseDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SYNC_DB, mndProcessSyncDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_COMPACT_DB, mndProcessCompactDbMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); + + return sdbSetTable(pMnode->pSdb, table); } void mndCleanupDb(SMnode *pMnode) {} +static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj)); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pDb->name, TSDB_FULL_DB_NAME_LEN) + SDB_SET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN) + SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) + SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) + SDB_SET_INT64(pRaw, dataPos, pDb->uid) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRowsPerFileBlock) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRowsPerFileBlock) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.quorum) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_DB_VER_NUM) { + mError("failed to decode db since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); + SDbObj *pDb = sdbGetRowObj(pRow); + if (pDb == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->name, TSDB_FULL_DB_NAME_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->acct, TSDB_USER_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep0) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep1) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep2) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRowsPerFileBlock) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRowsPerFileBlock) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.commitTime) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.fsyncPeriod) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.precision) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.compression) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.replications) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.quorum) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.update) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.cacheLastRow) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_DB_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) { + mTrace("db:%s, perform insert action", pDb->name); + return 0; +} + +static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { + mTrace("db:%s, perform delete action", pDb->name); + return 0; +} + +static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { + mTrace("db:%s, perform update action", pOldDb->name); + memcpy(pOldDb, pNewDb, sizeof(SDbObj)); + return 0; +} + SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { SSdb *pSdb = pMnode->pSdb; return sdbAcquire(pSdb, SDB_DB, db); @@ -35,14 +165,459 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { +static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + terrno = TSDB_CODE_MND_INVALID_DB_CACHE_SIZE; + return -1; + } + + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { + terrno = TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS; + return -1; + } + + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + terrno = TSDB_CODE_MND_INVALID_DB_DAYS; + return -1; + } + + if (pCfg->daysToKeep0 < pCfg->daysPerFile) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + return -1; + } + + if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + return -1; + } + + if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + return -1; + } + + if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + return -1; + } + + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + return -1; + } + + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + terrno = TSDB_CODE_MND_INVALID_DB_MAX_ROWS; + return -1; + } + + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { + terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + return -1; + } + + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { + terrno = TSDB_CODE_MND_INVALID_DB_COMMIT_TIME; + return -1; + } + + if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { + terrno = TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD; + return -1; + } + + if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { + terrno = TSDB_CODE_MND_INVALID_DB_WAL_LEVEL; + return -1; + } + + if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { + terrno = TSDB_CODE_MND_INVALID_DB_PRECISION; + return -1; + } + + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { + terrno = TSDB_CODE_MND_INVALID_DB_COMP; + return -1; + } + + if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { + terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + return -1; + } + + if (pCfg->replications > mndGetDnodeSize(pMnode)) { + terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + return -1; + } + + if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { + terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + return -1; + } + + if (pCfg->quorum > pCfg->replications) { + terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + return -1; + } + + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { + terrno = TSDB_CODE_MND_INVALID_DB_UPDATE; + return -1; + } + + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { + terrno = TSDB_CODE_MND_INVALID_DB_CACHE_LAST; + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +static void mndSetDefaultDbCfg(SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; + if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; + if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; + if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = TSDB_DEFAULT_KEEP; + if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = TSDB_DEFAULT_KEEP; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; + if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; + if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; + if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; + if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; + if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA_OPTION; + if (pCfg->quorum < 0) pCfg->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION; + if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION; + if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; +} + +static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, char *acct) { + SDbObj dbObj = {0}; + tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); + tstrncpy(dbObj.acct, acct, TSDB_USER_LEN); + dbObj.createdTime = taosGetTimestampMs(); + dbObj.updateTime = dbObj.createdTime; + dbObj.uid = 1234; + dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, + .totalBlocks = pCreate->totalBlocks, + .daysPerFile = pCreate->daysPerFile, + .daysToKeep0 = pCreate->daysToKeep0, + .daysToKeep1 = pCreate->daysToKeep1, + .daysToKeep2 = pCreate->daysToKeep2, + .minRowsPerFileBlock = pCreate->minRowsPerFileBlock, + .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .fsyncPeriod = pCreate->fsyncPeriod, + .commitTime = pCreate->commitTime, + .precision = pCreate->precision, + .compression = pCreate->compression, + .walLevel = pCreate->walLevel, + .replications = pCreate->replications, + .quorum = pCreate->quorum, + .update = pCreate->update, + .cacheLastRow = pCreate->cacheLastRow}; + + mndSetDefaultDbCfg(&dbObj.cfg); + + if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); + + SSdbRaw *pRedoRaw = mndDbActionEncode(&dbObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndDbActionEncode(&dbObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); + pCreate->daysPerFile = htonl(pCreate->daysPerFile); + pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); + pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); + pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); + pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + pCreate->commitTime = htonl(pCreate->commitTime); + pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + + mDebug("db:%s, start to create", pCreate->db); + + SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); + if (pDb != NULL) { + sdbRelease(pMnode->pSdb, pDb); + if (pCreate->ignoreExist) { + mDebug("db:%s, already exist, ignore exist is set", pCreate->db); + return 0; + } else { + terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + } + + SUserObj *pOperUser = mndAcquireUser(pMnode, pMsg->user); + if (pOperUser == NULL) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser->acct); + mndReleaseUser(pMnode, pOperUser); + + if (code != 0) { + terrno = code; + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mnodeSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { + bool changed = false; + + if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { + pDb->cfg.totalBlocks = pAlter->totalBlocks; + changed = true; + } + + if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { + pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; + changed = true; + } + + if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { + pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; + changed = true; + } + + if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { + pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; + changed = true; + } + + if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { + pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; + changed = true; + } + + if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) { + pDb->cfg.walLevel = pAlter->walLevel; + changed = true; + } + + if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { + pDb->cfg.quorum = pAlter->quorum; + changed = true; + } + + if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { + pDb->cfg.cacheLastRow = pAlter->cacheLastRow; + changed = true; + } + + if (!changed) { + terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; + return -1; + } + + return 0; +} + +static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); + return terrno; + } + + mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); + + SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; + pAlter->totalBlocks = htonl(pAlter->totalBlocks); + pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); + pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); + pAlter->daysToKeep2 = htonl(pAlter->daysToKeep2); + pAlter->fsyncPeriod = htonl(pAlter->fsyncPeriod); + + mDebug("db:%s, start to alter", pAlter->db); + + SDbObj *pDb = mndAcquireDb(pMnode, pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter since %s", pAlter->db, terrstr()); + return TSDB_CODE_MND_DB_NOT_EXIST; + } + + SDbObj dbObj = {0}; + memcpy(&dbObj, pDb, sizeof(SDbObj)); + + int32_t code = mnodeSetDbCfgFromAlterDbMsg(&dbObj, pAlter); + if (code != 0) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); + return code; + } + + code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("db:%s, failed to drop since %s", pDb->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); + + SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropDbMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("db:%s, start to drop", pDrop->db); + + SDbObj *pDb = mndAcquireDb(pMnode, pDrop->db); + if (pDb == NULL) { + if (pDrop->ignoreNotExists) { + mDebug("db:%s, not exist, ignore not exist is set", pDrop->db); + return TSDB_CODE_SUCCESS; + } else { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); + return -1; + } + } + + int32_t code = mndDropDb(pMnode, pMsg, pDb); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + terrno = code; + mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SUseDbMsg *pUse = pMsg->rpcMsg.pCont; - strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb != NULL) { + strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); mndReleaseDb(pMnode, pDb); return 0; } else { @@ -50,3 +625,275 @@ static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { return -1; } } + +static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; + + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb == NULL) { + mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr()); + return -1; + } else { + mndReleaseDb(pMnode, pDb); + return 0; + } +} + +static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; + + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb == NULL) { + mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr()); + return -1; + } else { + mndReleaseDb(pMnode, pDb); + return 0; + } +} + +static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "replica"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "quorum"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "days"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "keep0,keep1,keep2"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "cache(MB)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "blocks"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "minrows"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "maxrows"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "wallevel"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "fsync"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "comp"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "cachelast"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "precision"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "update"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +char *mnodeGetDbStr(char *src) { + char *pos = strstr(src, TS_PATH_DELIMITER); + if (pos != NULL) ++pos; + + return pos; +} + +static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + char *pWrite; + int32_t cols = 0; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *name = mnodeGetDbStr(pDb->name); + if (name != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + } else { + STR_TO_VARSTR(pWrite, "NULL"); + } + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pDb->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.replications; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.quorum; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.daysPerFile; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char tmp[128] = {0}; + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { + sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); + } else { + sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2); + } + STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, strlen(tmp)); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.totalBlocks; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.walLevel; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.fsyncPeriod; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.compression; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.cacheLastRow; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *prec = NULL; + switch (pDb->cfg.precision) { + case TSDB_TIME_PRECISION_MILLI: + prec = TSDB_TIME_PRECISION_MILLI_STR; + break; + case TSDB_TIME_PRECISION_MICRO: + prec = TSDB_TIME_PRECISION_MICRO_STR; + break; + case TSDB_TIME_PRECISION_NANO: + prec = TSDB_TIME_PRECISION_NANO_STR; + break; + default: + assert(false); + break; + } + STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.update; + cols++; + + numOfRows++; + sdbRelease(pSdb, pDb); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 241b006293..dc9f54888f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -16,12 +16,17 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" #include "mndMnode.h" +#include "mndShow.h" #include "mndTrans.h" #include "ttime.h" +#include "tutil.h" -#define SDB_DNODE_VER 1 +#define TSDB_DNODE_VER 1 +#define TSDB_CONFIG_OPTION_LEN 16 +#define TSDB_CONIIG_VALUE_LEN 48 +#define TSDB_CONFIG_NUMBER 8 -static char *offlineReason[] = { +static const char *offlineReason[] = { "", "status msg timeout", "status not received", @@ -36,8 +41,74 @@ static char *offlineReason[] = { "unknown", }; +static const char *dnodeStatus[] = {"offline", "ready", "creating", "dropping"}; + +static int32_t mndCreateDefaultDnode(SMnode *pMnode); +static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); +static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); +static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); +static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode); + +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg); + +static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); +static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); + +int32_t mndInitDnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_DNODE, + .keyType = SDB_KEY_INT32, + .deployFp = (SdbDeployFp)mndCreateDefaultDnode, + .encodeFp = (SdbEncodeFp)mndDnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndDnodeActionDecode, + .insertFp = (SdbInsertFp)mndDnodeActionInsert, + .updateFp = (SdbUpdateFp)mndDnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndDnodeActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DNODE, mndProcessCreateDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DNODE, mndProcessDropDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE, mndProcessConfigDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP, mndProcessConfigDnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_STATUS, mndProcessStatusMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndGetConfigMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndRetrieveConfigs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndCancelGetNextConfig); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DNODE, mndGetDnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DNODE, mndRetrieveDnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DNODE, mndCancelGetNextDnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupDnode(SMnode *pMnode) {} + +static int32_t mndCreateDefaultDnode(SMnode *pMnode) { + SDnodeObj dnodeObj = {0}; + dnodeObj.id = 1; + dnodeObj.createdTime = taosGetTimestampMs(); + dnodeObj.updateTime = dnodeObj.createdTime; + dnodeObj.port = pMnode->replicas[0].port; + memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); + + SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); + return sdbWrite(pMnode->pSdb, pRaw); +} + static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, SDB_DNODE_VER, sizeof(SDnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj)); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -55,7 +126,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_DNODE_VER) { + if (sver != TSDB_DNODE_VER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode dnode since %s", terrstr()); return NULL; @@ -101,30 +172,30 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { return 0; } -static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pSrcDnode, SDnodeObj *pDstDnode) { - mTrace("dnode:%d, perform update action", pSrcDnode->id); - pSrcDnode->id = pDstDnode->id; - pSrcDnode->createdTime = pDstDnode->createdTime; - pSrcDnode->updateTime = pDstDnode->updateTime; - pSrcDnode->port = pDstDnode->port; - memcpy(pSrcDnode->fqdn, pDstDnode->fqdn, TSDB_FQDN_LEN); +static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { + mTrace("dnode:%d, perform update action", pOldDnode->id); + pOldDnode->id = pNewDnode->id; + pOldDnode->createdTime = pNewDnode->createdTime; + pOldDnode->updateTime = pNewDnode->updateTime; + pOldDnode->port = pNewDnode->port; + memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN); return 0; } -static int32_t mndCreateDefaultDnode(SMnode *pMnode) { - SDnodeObj dnodeObj = {0}; - dnodeObj.id = 1; - dnodeObj.createdTime = taosGetTimestampMs(); - dnodeObj.updateTime = dnodeObj.createdTime; - dnodeObj.port = pMnode->replicas[0].port; - memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); +SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_DNODE, &dnodeId); +} - SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); +void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pDnode); +} - mTrace("dnode:%d, will be created while deploy sdb", dnodeObj.id); - return sdbWrite(pMnode->pSdb, pRaw); +SEpSet mndGetDnodeEpset(SDnodeObj *pDnode) { + SEpSet epSet = {.inUse = 0, .numOfEps = 1, .port[0] = pDnode->port}; + memcpy(epSet.fqdn[0], pDnode->fqdn, TSDB_FQDN_LEN); + return epSet; } static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { @@ -145,7 +216,7 @@ static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { return NULL; } -static int32_t mndGetDnodeSize(SMnode *pMnode) { +int32_t mndGetDnodeSize(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; return sdbGetSize(pSdb, SDB_DNODE); } @@ -317,38 +388,385 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { + SDnodeObj dnodeObj = {0}; + dnodeObj.id = 1; // todo + dnodeObj.createdTime = taosGetTimestampMs(); + dnodeObj.updateTime = dnodeObj.createdTime; + taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); -static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return 0; } + if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return terrno; + } -static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return -1; + } + mDebug("trans:%d, used to create dnode:%s", pTrans->id, pCreate->ep); -int32_t mndInitDnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DNODE, - .keyType = SDB_KEY_INT32, - .deployFp = (SdbDeployFp)mndCreateDefaultDnode, - .encodeFp = (SdbEncodeFp)mndDnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndDnodeActionDecode, - .insertFp = (SdbInsertFp)mndDnodeActionInsert, - .updateFp = (SdbUpdateFp)mndDnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndDnodeActionDelete}; + SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DNODE, mndProcessCreateDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DNODE, mndProcessDropDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE, mndProcessConfigDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_STATUS, mndProcessStatusMsg); + SSdbRaw *pUndoRaw = mndDnodeActionEncode(&dnodeObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - return sdbSetTable(pMnode->pSdb, table); + SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; } -void mndCleanupDnode(SMnode *pMnode) {} +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; -SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) { + mDebug("dnode:%s, start to create", pCreate->ep); + + if (pCreate->ep[0] == 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep); + if (pDnode != NULL) { + mError("dnode:%d, already exist", pDnode->id); + sdbRelease(pMnode->pSdb, pDnode); + terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; + return -1; + } + + int32_t code = mndCreateDnode(pMnode, pMsg, pCreate); + + if (code != 0) { + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%d", pTrans->id, pDnode->id); + + SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndDnodeActionEncode(pDnode); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("dnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDrop->dnodeId); + if (pDnode == NULL) { + mError("dnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropDnode(pMnode, pMsg, pDnode); + + if (code != 0) { + mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pDnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCfgDnodeMsg *pCfg = pMsg->rpcMsg.pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); + if (pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("dnode:%d, failed to cfg since %s ", pCfg->dnodeId, terrstr()); + return -1; + } + + SEpSet epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + SCfgDnodeMsg *pCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); + pCfgDnode->dnodeId = htonl(pCfg->dnodeId); + memcpy(pCfgDnode->config, pCfg->config, 128); + + SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, .pCont = pCfgDnode, .contLen = sizeof(SCfgDnodeMsg)}; + + mInfo("dnode:%d, is configured", pCfg->dnodeId); + mndSendMsgToDnode(pMnode, &epSet, &rpcMsg); + + return 0; +} + +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { mInfo("cfg dnode rsp is received"); } + +static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = TSDB_CONFIG_NUMBER; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pIter = NULL; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; + char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONIIG_VALUE_LEN + 1] = {0}; + char *pWrite; + int32_t cols = 0; + + cfgOpts[numOfRows] = "statusInterval"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", pMnode->cfg.statusInterval); + numOfRows++; + + cfgOpts[numOfRows] = "timezone"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.timezone); + numOfRows++; + + cfgOpts[numOfRows] = "locale"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.locale); + numOfRows++; + + cfgOpts[numOfRows] = "charset"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.charset); + numOfRows++; + + for (int32_t i = 0; i < numOfRows; i++) { + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfgOpts[i], TSDB_CONFIG_OPTION_LEN); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfgVals[i], TSDB_CONIIG_VALUE_LEN); + cols++; + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} + +static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "end point"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "cores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "offline reason"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SDnodeObj *pDnode = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->ep, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->numOfVnodes; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->numOfCores; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + const char *status = dnodeStatus[pDnode->status]; + STR_TO_VARSTR(pWrite, status); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pDnode->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + cols++; + + numOfRows++; + sdbRelease(pSdb, pDnode); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_DNODE, &dnodeId); -} - -void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pDnode); -} + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d859da029f..d2d538b973 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -14,8 +14,498 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndFunc.h" +#include "mndShow.h" +#include "mndSync.h" +#include "mndTrans.h" -int32_t mndInitFunc(SMnode *pMnode) { return 0; } -void mndCleanupFunc(SMnode *pMnode) {} \ No newline at end of file +#define SDB_FUNC_VER 1 + +static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); +static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); +static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); +static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); +static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg); +static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); + +int32_t mndInitFunc(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_FUNC, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndFuncActionEncode, + .decodeFp = (SdbDecodeFp)mndFuncActionDecode, + .insertFp = (SdbInsertFp)mndFuncActionInsert, + .updateFp = (SdbUpdateFp)mndFuncActionUpdate, + .deleteFp = (SdbDeleteFp)mndFuncActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_FUNCTION, mndProcessCreateFuncMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_FUNCTION, mndProcessDropFuncMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupFunc(SMnode *pMnode) {} + +static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { + int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj); + SSdbRaw *pRaw = sdbAllocRaw(SDB_FUNC, SDB_FUNC_VER, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pFunc->createdTime) + SDB_SET_INT8(pRaw, dataPos, pFunc->funcType) + SDB_SET_INT8(pRaw, dataPos, pFunc->scriptType) + SDB_SET_INT8(pRaw, dataPos, pFunc->align) + SDB_SET_INT8(pRaw, dataPos, pFunc->outputType) + SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen) + SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize) + SDB_SET_INT64(pRaw, dataPos, pFunc->sigature) + SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize) + SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize) + SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize) + SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != SDB_FUNC_VER) { + mError("failed to decode func since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + int32_t size = sizeof(SFuncObj) + TSDB_FUNC_COMMENT_LEN + TSDB_FUNC_CODE_LEN; + SSdbRow *pRow = sdbAllocRow(size); + SFuncObj *pFunc = sdbGetRowObj(pRow); + if (pFunc == NULL) return NULL; + char *tmp = (char *)pFunc + sizeof(SFuncObj); + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pFunc->createdTime) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->funcType) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->scriptType) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->align) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->outputType) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->outputLen) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->bufSize) + SDB_GET_INT64(pRaw, pRow, dataPos, &pFunc->sigature) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->commentSize) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->codeSize) + SDB_GET_BINARY(pRaw, pRow, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize) + pFunc->pComment = pFunc->pData; + pFunc->pCode = (pFunc->pData + pFunc->commentSize); + + return pRow; +} + +static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { + mTrace("func:%s, perform insert action", pFunc->name); + return 0; +} + +static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { + mTrace("func:%s, perform delete action", pFunc->name); + return 0; +} + +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) { + mTrace("func:%s, perform update action", pOldFunc->name); + return 0; +} + +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pCreate) { + SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); + pFunc->createdTime = taosGetTimestampMs(); + pFunc->funcType = pCreate->funcType; + pFunc->scriptType = pCreate->scriptType; + pFunc->outputType = pCreate->outputType; + pFunc->outputLen = pCreate->outputLen; + pFunc->bufSize = pCreate->bufSize; + pFunc->sigature = pCreate->sigature; + pFunc->commentSize = pCreate->commentSize; + pFunc->codeSize = pCreate->codeSize; + pFunc->pComment = pFunc->pData; + memcpy(pFunc->pComment, pCreate->pCont, pCreate->commentSize); + pFunc->pCode = pFunc->pData + pCreate->commentSize; + memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + free(pFunc); + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + free(pFunc); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + free(pFunc); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + free(pFunc); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + free(pFunc); + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); + + SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + + SCreateFuncMsg *pCreate = pMsg->rpcMsg.pCont; + pCreate->outputLen = htonl(pCreate->outputLen); + pCreate->bufSize = htonl(pCreate->bufSize); + pCreate->sigature = htobe64(pCreate->sigature); + pCreate->commentSize = htonl(pCreate->commentSize); + pCreate->codeSize = htonl(pCreate->codeSize); + + mDebug("func:%s, start to create", pCreate->name); + + SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pCreate->name); + if (pFunc != NULL) { + sdbRelease(pMnode->pSdb, pFunc); + terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->commentSize <= 0 || pCreate->commentSize > TSDB_FUNC_COMMENT_LEN) { + terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->codeSize <= 0 || pCreate->codeSize > TSDB_FUNC_CODE_LEN) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->pCont[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->bufSize < 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + int32_t code = mndCreateFunc(pMnode, pMsg, pCreate); + + if (code != 0) { + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropFuncMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("func:%s, start to drop", pDrop->name); + + if (pDrop->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pDrop->name); + if (pFunc == NULL) { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + int32_t code = mndDropFunc(pMnode, pMsg, pFunc); + + if (code != 0) { + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + + SRetrieveFuncMsg *pRetrieve = pMsg->rpcMsg.pCont; + pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); + + int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; + + SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); + pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); + char *pOutput = pRetrieveRsp->pFuncInfos; + + for (int32_t i = 0; i < pRetrieve->numOfFuncs; ++i) { + char funcName[TSDB_FUNC_NAME_LEN] = {0}; + memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); + + SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, funcName); + if (pFunc == NULL) { + terrno = TSDB_CODE_MND_INVALID_FUNC; + mError("func:%s, failed to retrieve since %s", funcName, terrstr()); + return -1; + } + + SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; + + strncpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); + pFuncInfo->funcType = pFunc->funcType; + pFuncInfo->scriptType = pFunc->scriptType; + pFuncInfo->outputType = pFunc->outputType; + pFuncInfo->outputLen = htonl(pFunc->outputLen); + pFuncInfo->bufSize = htonl(pFunc->bufSize); + pFuncInfo->sigature = htobe64(pFunc->sigature); + pFuncInfo->commentSize = htonl(pFunc->commentSize); + pFuncInfo->codeSize = htonl(pFunc->codeSize); + memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize); + + pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; + } + + pMsg->pCont = pRetrieveRsp; + pMsg->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + + return 0; +} + +static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "comment"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "aggregate"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "outputtype"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "code_len"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "bufsize"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, "show funcs"); + + return 0; +} + +static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) { + char *msg = "unknown"; + if (type >= sizeof(tDataTypes) / sizeof(tDataTypes[0])) { + return msg; + } + + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t bytes = len > 0 ? (int)(len - VARSTR_HEADER_SIZE) : len; + + snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes / 4 : bytes); + buf[buflen - 1] = 0; + + return buf; + } + + return tDataTypes[type].name; +} + +static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SFuncObj *pFunc = NULL; + int32_t cols = 0; + char *pWrite; + char buf[TSDB_TYPE_STR_MAX_LEN]; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_FUNC, pShow->pIter, (void **)&pFunc); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->name, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->pComment, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE ? 1 : 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), + pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pFunc->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pFunc->codeSize; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pFunc->bufSize; + cols++; + + numOfRows++; + sdbRelease(pSdb, pFunc); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index eabcce04c5..8a05eb02e8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -14,10 +14,88 @@ */ #define _DEFAULT_SOURCE +#include "mndMnode.h" +#include "mndDnode.h" +#include "mndShow.h" #include "mndTrans.h" #define SDB_MNODE_VER 1 +static int32_t mndCreateDefaultMnode(SMnode *pMnode); +static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj); +static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pMnodeObj); +static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj); +static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode); +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); + +int32_t mndInitMnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_MNODE, + .keyType = SDB_KEY_INT32, + .deployFp = (SdbDeployFp)mndCreateDefaultMnode, + .encodeFp = (SdbEncodeFp)mndMnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndMnodeActionDecode, + .insertFp = (SdbInsertFp)mndMnodeActionInsert, + .updateFp = (SdbUpdateFp)mndMnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndMnodeActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP, mndProcessCreateMnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE_IN_RSP, mndProcessDropMnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndGetMnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndRetrieveMnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndCancelGetNextMnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupMnode(SMnode *pMnode) {} + +static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_MNODE, &mnodeId); +} + +static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pMnodeObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pMnodeObj); +} + +char *mndGetRoleStr(int32_t showType) { + switch (showType) { + case TAOS_SYNC_STATE_FOLLOWER: + return "unsynced"; + case TAOS_SYNC_STATE_CANDIDATE: + return "slave"; + case TAOS_SYNC_STATE_LEADER: + return "master"; + default: + return "undefined"; + } +} + +static int32_t mndCreateDefaultMnode(SMnode *pMnode) { + SMnodeObj mnodeObj = {0}; + mnodeObj.id = 1; + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; + + SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mDebug("mnode:%d, will be created while deploy sdb", mnodeObj.id); + return sdbWrite(pMnode->pSdb, pRaw); +} + static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) { SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj)); if (pRaw == NULL) return NULL; @@ -81,50 +159,14 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj) { return 0; } -static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pSrcMnode, SMnodeObj *pDstMnode) { - mTrace("mnode:%d, perform update action", pSrcMnode->id); - pSrcMnode->id = pDstMnode->id; - pSrcMnode->createdTime = pDstMnode->createdTime; - pSrcMnode->updateTime = pDstMnode->updateTime; +static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode) { + mTrace("mnode:%d, perform update action", pOldMnode->id); + pOldMnode->id = pNewMnode->id; + pOldMnode->createdTime = pNewMnode->createdTime; + pOldMnode->updateTime = pNewMnode->updateTime; return 0; } -static int32_t mndCreateDefaultMnode(SMnode *pMnode) { - SMnodeObj mnodeObj = {0}; - mnodeObj.id = 1; - mnodeObj.createdTime = taosGetTimestampMs(); - mnodeObj.updateTime = mnodeObj.createdTime; - - SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mTrace("mnode:%d, will be created while deploy sdb", mnodeObj.id); - return sdbWrite(pMnode->pSdb, pRaw); -} - -static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; } - -int32_t mndInitMnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_MNODE, - .keyType = SDB_KEY_INT32, - .deployFp = (SdbDeployFp)mndCreateDefaultMnode, - .encodeFp = (SdbEncodeFp)mndMnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndMnodeActionDecode, - .insertFp = (SdbInsertFp)mndMnodeActionInsert, - .updateFp = (SdbUpdateFp)mndMnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndMnodeActionDelete}; - - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeMsg); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupMnode(SMnode *pMnode) {} - bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { SSdb *pSdb = pMnode->pSdb; @@ -157,4 +199,271 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { pEpSet->numOfEps++; } -} \ No newline at end of file +} + +static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateMnodeMsg *pCreate) { + SMnodeObj mnodeObj = {0}; + mnodeObj.id = 1; // todo + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("dnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + mDebug("trans:%d, used to create dnode:%d", pTrans->id, pCreate->dnodeId); + + SSdbRaw *pRedoRaw = mndMnodeActionEncode(&mnodeObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndMnodeActionEncode(&mnodeObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndMnodeActionEncode(&mnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("mnode:%d, start to create", pCreate->dnodeId); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("mnode:%d, dnode not exist", pDnode->id); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + mndReleaseDnode(pMnode, pDnode); + + SMnodeObj *pMnodeObj = mndAcquireMnode(pMnode, pCreate->dnodeId); + if (pMnodeObj != NULL) { + mError("mnode:%d, mnode already exist", pMnodeObj->id); + terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; + return -1; + } + + int32_t code = mndCreateMnode(pMnode, pMsg, pCreate); + + if (code != 0) { + mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pMnodeObj) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("mnode:%d, failed to drop since %s", pMnodeObj->id, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%d", pTrans->id, pMnodeObj->id); + + SSdbRaw *pRedoRaw = mndMnodeActionEncode(pMnodeObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndMnodeActionEncode(pMnodeObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndMnodeActionEncode(pMnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("mnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("mnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SMnodeObj *pMnodeObj = mndAcquireMnode(pMnode, pDrop->dnodeId); + if (pMnodeObj == NULL) { + mError("mnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropMnode(pMnode, pMsg, pMnodeObj); + + if (code != 0) { + mError("mnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "end point"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "role"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "role time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SMnodeObj *pMnodeObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pMnodeObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pMnodeObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pMnodeObj->id); + if (pDnode != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->ep, pShow->bytes[cols]); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols]); + } + mndReleaseDnode(pMnode, pDnode); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *roles = mndGetRoleStr(pMnodeObj->role); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pMnodeObj->roleTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pMnodeObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pMnodeObj); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndOper.c b/source/dnode/mnode/impl/src/mndOper.c deleted file mode 100644 index 4416a1fc2d..0000000000 --- a/source/dnode/mnode/impl/src/mndOper.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" - -int32_t mndInitOper(SMnode *pMnode) { return 0; } -void mndCleanupOper(SMnode *pMnode) {} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ad7a6322bb..b429879789 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -467,7 +467,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -595,7 +595,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -811,7 +811,7 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 747ea39237..cfecaf2775 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -20,7 +20,6 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static char *mndShowStr(int32_t showType); static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); static bool mndCheckRetrieveFinished(SShowObj *pShow); @@ -88,10 +87,6 @@ static void mndFreeShowObj(SShowObj *pShow) { ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; if (freeFp != NULL) { - if (pShow->pVgIter != NULL) { - // only used in 'show vnodes "ep"' - (*freeFp)(pMnode, pShow->pVgIter); - } if (pShow->pIter != NULL) { (*freeFp)(pMnode, pShow->pIter); } @@ -259,7 +254,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { return TSDB_CODE_SUCCESS; } -static char *mndShowStr(int32_t showType) { +char *mndShowStr(int32_t showType) { switch (showType) { case TSDB_MGMT_TABLE_ACCT: return "show accounts"; @@ -275,7 +270,7 @@ static char *mndShowStr(int32_t showType) { return "show mnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; - case TSDB_MGMT_TABLE_METRIC: + case TSDB_MGMT_TABLE_STABLE: return "show stables"; case TSDB_MGMT_TABLE_MODULE: return "show modules"; diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index e54bb17451..b57b05f299 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -14,8 +14,414 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndStable.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndDb.h" +#include "tname.h" -int32_t mndInitStable(SMnode *pMnode) { return 0; } -void mndCleanupStable(SMnode *pMnode) {} \ No newline at end of file +#define TSDB_STABLE_VER_NUM 1 +#define TSDB_STABLE_RESERVE_SIZE 64 + +static SSdbRaw *mndStableActionEncode(SStableObj *pStb); +static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw); +static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb); +static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb); +static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb); +static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextStable(SMnode *pMnode, void *pIter); + +int32_t mndInitStable(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_STABLE, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStableActionEncode, + .decodeFp = (SdbDecodeFp)mndStableActionDecode, + .insertFp = (SdbInsertFp)mndStableActionInsert, + .updateFp = (SdbUpdateFp)mndStableActionUpdate, + .deleteFp = (SdbDeleteFp)mndStableActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE, mndProcessCreateStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE, mndProcessAlterStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE, mndProcessDropStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP, mndProcessCreateStableInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP, mndProcessAlterStableInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE_IN_RSP, mndProcessDropStableInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStableMetaMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndGetStableMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndRetrieveStables); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndCancelGetNextStable); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupStable(SMnode *pMnode) {} + +static SSdbRaw *mndStableActionEncode(SStableObj *pStb) { + int32_t size = sizeof(SStableObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pStb->createdTime) + SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) + SDB_SET_INT64(pRaw, dataPos, pStb->uid) + SDB_SET_INT64(pRaw, dataPos, pStb->version) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->columnSchema[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type); + SDB_SET_INT32(pRaw, dataPos, pSchema->colId); + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pStb->tagSchema[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type); + SDB_SET_INT32(pRaw, dataPos, pSchema->colId); + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_STABLE_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_STABLE_VER_NUM) { + mError("failed to decode stable since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + int32_t size = sizeof(SStableObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + SStableObj *pStb = sdbGetRowObj(pRow); + if (pStb == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) + + pStb->columnSchema = calloc(pStb->numOfColumns, sizeof(SSchema)); + pStb->tagSchema = calloc(pStb->numOfTags, sizeof(SSchema)); + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->columnSchema[i]; + SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); + SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pStb->tagSchema[i]; + SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); + SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STABLE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb) { + mTrace("stable:%s, perform insert action", pStb->name); + return 0; +} + +static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb) { + mTrace("stable:%s, perform delete action", pStb->name); + return 0; +} + +static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb) { + mTrace("stable:%s, perform update action", pOldStb->name); + atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime); + atomic_exchange_32(&pOldStb->version, pNewStb->version); + + taosWLockLatch(&pOldStb->lock); + int32_t numOfTags = pNewStb->numOfTags; + int32_t tagSize = numOfTags * sizeof(SSchema); + int32_t numOfColumns = pNewStb->numOfColumns; + int32_t columnSize = numOfColumns * sizeof(SSchema); + + if (pOldStb->numOfTags < numOfTags) { + pOldStb->tagSchema = malloc(tagSize); + } + if (pOldStb->numOfColumns < numOfColumns) { + pOldStb->columnSchema = malloc(columnSize); + } + + memcpy(pOldStb->tagSchema, pNewStb->tagSchema, tagSize); + memcpy(pOldStb->columnSchema, pNewStb->columnSchema, columnSize); + taosWUnLockLatch(&pOldStb->lock); + return 0; +} + +SStableObj *mndAcquireStb(SMnode *pMnode, char *stbName) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_STABLE, stbName); +} + +void mndReleaseStb(SMnode *pMnode, SStableObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pStb); +} + +static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg) { return 0; } + +static SDbObj *mndGetDbByStbName(SMnode *pMnode, char *stbName) { + SName name = {0}; + tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SStableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + + mDebug("stable:%s, start to retrieve meta", pInfo->name); + + SDbObj *pDb = mndGetDbByStbName(pMnode, pInfo->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stable:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + return -1; + } + + SStableObj *pStb = mndAcquireStb(pMnode, pInfo->name); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; + mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); + return -1; + } + + int32_t contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + STableMetaMsg *pMeta = rpcMallocCont(contLen); + if (pMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); + return -1; + } + + memcpy(pMeta->stableFname, pStb->name, TSDB_TABLE_FNAME_LEN); + pMeta->numOfTags = htonl(pStb->numOfTags); + pMeta->numOfColumns = htonl(pStb->numOfColumns); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->update = pDb->cfg.update; + pMeta->sversion = htonl(pStb->version); + pMeta->suid = htonl(pStb->uid); + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pColumn = &pStb->columnSchema[i]; + memcpy(pSchema->name, pColumn->name, TSDB_COL_NAME_LEN); + pSchema->type = pColumn->type; + pSchema->colId = htonl(pColumn->colId); + pSchema->bytes = htonl(pColumn->bytes); + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns]; + SSchema *pTag = &pStb->tagSchema[i]; + memcpy(pSchema->name, pTag->name, TSDB_COL_NAME_LEN); + pSchema->type = pTag->type; + pSchema->colId = htons(pTag->colId); + pSchema->bytes = htonl(pTag->bytes); + } + + pMsg->pCont = pMeta; + pMsg->contLen = contLen; + + mDebug("stable:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); + return 0; +} + +static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfStables = 0; + void *pIter = NULL; + while (1) { + SStableObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); + if (pIter == NULL) break; + + if (strcmp(pStb->db, dbName) == 0) { + numOfStables++; + } + + sdbRelease(pSdb, pStb); + } + + *pNumOfStables = numOfStables; + return 0; +} + +static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfStables(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "tags"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static void mnodeExtractTableName(char* tableId, char* name) { + int pos = -1; + int num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; + } + + if (num == 2) { + strcpy(name, tableId + pos + 1); + } +} + +static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SStableObj *pStb = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; + + tstrncpy(prefix, pShow->db, 64); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = (int32_t)strlen(prefix); + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStb); + if (pShow->pIter == NULL) break; + + if (strncmp(pStb->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pStb); + continue; + } + + cols = 0; + + char stableName[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(stableName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, stableName); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pStb->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pStb->numOfColumns; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pStb->numOfTags; + cols++; + + numOfRows++; + sdbRelease(pSdb, pStb); + } + + pShow->numOfReads += numOfRows; + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextStable(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 7231915311..6e7ee662f8 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -21,9 +21,15 @@ int32_t mndInitSync(SMnode *pMnode) { return 0; } void mndCleanupSync(SMnode *pMnode) {} -int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, void *pData) { - mndTransApply(pMnode, pData, pData, 0); - free(pData); +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg) { + int32_t code = 0; + + int32_t len = sdbGetRawTotalSize(pRaw); + SSdbRaw *pReceived = calloc(1, len); + memcpy(pReceived, pRaw, len); + mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); + + mndTransApply(pMnode, pReceived, pMsg, code); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 663ae76506..5beb1b10e3 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -189,12 +189,12 @@ static void mndSendTelemetryReport(SMnode* pMnode) { char buf[128] = {0}; uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { - mTrace("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); + mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); return; } SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { - mTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); + mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); return; } @@ -228,7 +228,7 @@ static void mndSendTelemetryReport(SMnode* pMnode) { // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mTrace("failed to receive response since %s", strerror(errno)); + mDebug("failed to receive response since %s", strerror(errno)); } taosCloseSocket(fd); @@ -297,7 +297,7 @@ int32_t mndInitTelem(SMnode* pMnode) { int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); pthread_attr_destroy(&attr); if (code != 0) { - mTrace("failed to create telemetry thread since :%s", strerror(code)); + mDebug("failed to create telemetry thread since :%s", strerror(code)); } mInfo("mnd telemetry is initialized"); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7eab16895c..3a53472d45 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,13 +15,33 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "trpc.h" +#include "mndSync.h" #define SDB_TRANS_VER 1 #define TRN_DEFAULT_ARRAY_SIZE 8 -SSdbRaw *mndTransActionEncode(STrans *pTrans) { - int32_t rawDataLen = 10 * sizeof(int32_t); +static SSdbRaw *mndTransActionEncode(STrans *pTrans); +static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); +static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); +static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); + +int32_t mndInitTrans(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_TRANS, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndTransActionEncode, + .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .insertFp = (SdbInsertFp)mndTransActionInsert, + .updateFp = (SdbUpdateFp)mndTransActionUpdate, + .deleteFp = (SdbDeleteFp)mndTransActionDelete}; + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupTrans(SMnode *pMnode) {} + +static SSdbRaw *mndTransActionEncode(STrans *pTrans) { + int32_t rawDataLen = 16 * sizeof(int32_t); int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -29,23 +49,23 @@ SSdbRaw *mndTransActionEncode(STrans *pTrans) { int32_t undoActionNum = taosArrayGetSize(pTrans->undoActions); for (int32_t i = 0; i < redoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->redoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); rawDataLen += sdbGetRawTotalSize(pTmp); } for (int32_t i = 0; i < undoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->undoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); rawDataLen += sdbGetRawTotalSize(pTmp); } for (int32_t i = 0; i < commitLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->commitLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); rawDataLen += sdbGetRawTotalSize(pTmp); } SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, SDB_TRANS_VER, rawDataLen); if (pRaw == NULL) { - mError("trn:%d, failed to alloc raw since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; } @@ -60,31 +80,33 @@ SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, undoActionNum) for (int32_t i = 0; i < redoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->redoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); SDB_SET_INT32(pRaw, dataPos, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } for (int32_t i = 0; i < undoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->undoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); SDB_SET_INT32(pRaw, dataPos, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } for (int32_t i = 0; i < commitLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->commitLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); SDB_SET_INT32(pRaw, dataPos, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } - mDebug("trn:%d, is encoded as raw:%p, len:%d", pTrans->id, pRaw, dataPos); + mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos); return pRaw; } -SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { +static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) { mError("failed to get soft ver from raw:%p since %s", pRaw, terrstr()); @@ -97,8 +119,8 @@ SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - SSdbRow *pRow = sdbAllocRow(sizeof(STrans)); - STrans *pTrans = sdbGetRowObj(pRow); + SSdbRow *pRow = sdbAllocRow(sizeof(STrans)); + STrans *pTrans = sdbGetRowObj(pRow); if (pTrans == NULL) { mError("failed to alloc trans from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -112,9 +134,9 @@ SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mDebug("trn:%d, failed to create array while parsed from raw:%p", pTrans->id, pRaw); - return NULL; + mDebug("trans:%d, failed to create array while parsed from raw:%p", pTrans->id, pRaw); + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; } int32_t redoLogNum = 0; @@ -133,85 +155,118 @@ SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &redoActionNum) SDB_GET_INT32(pRaw, pRow, dataPos, &undoActionNum) - int32_t code = 0; for (int32_t i = 0; i < redoLogNum; ++i) { int32_t dataLen = 0; SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) char *pData = malloc(dataLen); SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); - void *ret = taosArrayPush(pTrans->redoLogs, pData); + void *ret = taosArrayPush(pTrans->redoLogs, &pData); if (ret == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; break; } } + for (int32_t i = 0; i < undoLogNum; ++i) { + int32_t dataLen = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) + + char *pData = malloc(dataLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->undoLogs, &pData); + if (ret == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + break; + } + } + + for (int32_t i = 0; i < commitLogNum; ++i) { + int32_t dataLen = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) + + char *pData = malloc(dataLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->commitLogs, &pData); + if (ret == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + break; + } + } + +TRANS_DECODE_OVER: if (code != 0) { - terrno = code; - mError("trn:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, tstrerror(errno)); mndTransDrop(pTrans); + terrno = code; return NULL; } - mDebug("trn:%d, is parsed from raw:%p", pTrans->id, pRaw); + mTrace("trans:%d, decode from raw:%p", pTrans->id, pRaw); return pRow; } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { + mTrace("trans:%d, perform insert action, stage:%d", pTrans->id, pTrans->stage); + SArray *pArray = pTrans->redoLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); return code; } } - - mDebug("trn:%d, write to sdb", pTrans->id); return 0; } static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { - SArray *pArray = pTrans->redoLogs; + mTrace("trans:%d, perform delete action, stage:%d", pTrans->id, pTrans->stage); + + SArray *pArray = pTrans->undoLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); return code; } } - mDebug("trn:%d, delete from sdb", pTrans->id); return 0; } -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pTrans, STrans *pDstTrans) { - assert(true); - SArray *pArray = pTrans->redoLogs; +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { + mTrace("trans:%d, perform update action, stage:%d", pOldTrans->id, pNewTrans->stage); + + SArray *pArray = pOldTrans->commitLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pOldTrans->id, pRaw, terrstr()); return code; } } - pTrans->stage = pDstTrans->stage; - mDebug("trn:%d, update in sdb", pTrans->id); + pOldTrans->stage = pNewTrans->stage; return 0; } -static int32_t trnGenerateTransId() { return 1; } +static int32_t trnGenerateTransId() { + static int32_t tmp = 0; + return ++tmp; +} STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { STrans *pTrans = calloc(1, sizeof(STrans)); @@ -224,6 +279,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { pTrans->id = trnGenerateTransId(); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; + pTrans->pMnode = pMnode; pTrans->rpcHandle = rpcHandle; pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); @@ -238,13 +294,13 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - mDebug("trn:%d, is created, %p", pTrans->id, pTrans); + mDebug("trans:%d, data:%p is created", pTrans->id, pTrans); return pTrans; } static void trnDropArray(SArray *pArray) { for (int32_t i = 0; i < pArray->size; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); tfree(pRaw); } @@ -258,13 +314,13 @@ void mndTransDrop(STrans *pTrans) { trnDropArray(pTrans->redoActions); trnDropArray(pTrans->undoActions); - mDebug("trn:%d, is dropped, %p", pTrans->id, pTrans); + mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); tfree(pTrans); } void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { pTrans->rpcHandle = rpcHandle; - mTrace("trn:%d, set rpc handle:%p", pTrans->id, rpcHandle); + mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle); } static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { @@ -273,7 +329,7 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { return -1; } - void *ptr = taosArrayPush(pArray, pRaw); + void *ptr = taosArrayPush(pArray, &pRaw); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -284,92 +340,92 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw); - mTrace("trn:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw); - mTrace("trn:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw); - mTrace("trn:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { int32_t code = mndTransAppendArray(pTrans->redoActions, pMsg); - mTrace("trn:%d, msg:%p append to redo actions", pTrans->id, pMsg); + mTrace("trans:%d, msg:%p append to redo actions", pTrans->id, pMsg); return code; } int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { int32_t code = mndTransAppendArray(pTrans->undoActions, pMsg); - mTrace("trn:%d, msg:%p append to undo actions", pTrans->id, pMsg); + mTrace("trans:%d, msg:%p append to undo actions", pTrans->id, pMsg); return code; } -int32_t mndInitTrans(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TRANS, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, - .insertFp = (SdbInsertFp)mndTransActionInsert, - .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupTrans(SMnode *pMnode) {} - -int32_t mndTransPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)) { - if (syncfp == NULL) return -1; +int32_t mndTransPrepare(STrans *pTrans) { + mDebug("trans:%d, prepare transaction", pTrans->id); SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { - mError("trn:%d, failed to decode trans since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); return -1; } sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); - if (sdbWrite(pTrans->pMnode->pSdb, pRaw) != 0) { - mError("trn:%d, failed to write trans since %s", pTrans->id, terrstr()); + if (sdbWriteNotFree(pTrans->pMnode->pSdb, pRaw) != 0) { + mError("trans:%d, failed to write trans since %s", pTrans->id, terrstr()); return -1; } - if ((*syncfp)(pRaw, pTrans->rpcHandle) != 0) { - mError("trn:%d, failed to sync trans since %s", pTrans->id, terrstr()); + STransMsg *pMsg = calloc(1, sizeof(STransMsg)); + pMsg->id = pTrans->id; + pMsg->rpcHandle = pTrans->rpcHandle; + + mDebug("trans:%d, start sync, RPC:%p pMsg:%p", pTrans->id, pTrans->rpcHandle, pMsg); + if (mndSyncPropose(pTrans->pMnode, pRaw, pMsg) != 0) { + mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); + free(pMsg); + sdbFreeRaw(pRaw); return -1; } + sdbFreeRaw(pRaw); return 0; } -static void trnSendRpcRsp(void *rpcHandle, int32_t code) { - if (rpcHandle != NULL) { - SRpcMsg rspMsg = {.handle = rpcHandle, .code = terrno}; +static void trnSendRpcRsp(STransMsg *pMsg, int32_t code) { + mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x pMsg:%p", pMsg->id, pMsg->rpcHandle, code & 0xFFFF, pMsg); + if (pMsg->rpcHandle != NULL) { + SRpcMsg rspMsg = {.handle = pMsg->rpcHandle, .code = code}; rpcSendResponse(&rspMsg); } + + free(pMsg); } -int32_t mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, void *pData, int32_t code) { - if (code != 0) { - trnSendRpcRsp(pData, terrno); - return 0; +void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) { + if (code == 0) { + mDebug("trans:%d, commit transaction", pMsg->id); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + if (sdbWrite(pMnode->pSdb, pRaw) != 0) { + code = terrno; + mError("trans:%d, failed to write sdb while commit since %s", pMsg->id, terrstr()); + } + trnSendRpcRsp(pMsg, code); + } else { + mDebug("trans:%d, rollback transaction", pMsg->id); + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + if (sdbWrite(pMnode->pSdb, pRaw) != 0) { + mError("trans:%d, failed to write sdb while rollback since %s", pMsg->id, terrstr()); + } + trnSendRpcRsp(pMsg, code); } - - if (sdbWrite(pMnode->pSdb, pData) != 0) { - code = terrno; - trnSendRpcRsp(pData, code); - terrno = code; - return -1; - } - - return 0; } static int32_t trnExecuteArray(SMnode *pMnode, SArray *pArray) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c5162d8595..b77b0d4314 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -27,11 +27,14 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); -static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser); +static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser); static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg); static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg); static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg); +static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_USER, @@ -47,6 +50,9 @@ int32_t mndInitUser(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_USER, mndProcessAlterUserMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_USER, mndProcessDropUserMsg); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_USER, mndGetUserMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser); return sdbSetTable(pMnode->pSdb, table); } @@ -70,7 +76,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("user:%s, will be created while deploy sdb", userObj.user); + mDebug("user:%s, will be created while deploy sdb", userObj.user); return sdbWrite(pMnode->pSdb, pRaw); } @@ -162,16 +168,16 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { return 0; } -static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser) { - mTrace("user:%s, perform update action", pSrcUser->user); - memcpy(pSrcUser->user, pDstUser->user, TSDB_USER_LEN); - memcpy(pSrcUser->pass, pDstUser->pass, TSDB_PASSWORD_LEN); - memcpy(pSrcUser->acct, pDstUser->acct, TSDB_USER_LEN); - pSrcUser->createdTime = pDstUser->createdTime; - pSrcUser->updateTime = pDstUser->updateTime; - pSrcUser->superAuth = pDstUser->superAuth; - pSrcUser->readAuth = pDstUser->readAuth; - pSrcUser->writeAuth = pDstUser->writeAuth; +static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { + mTrace("user:%s, perform update action", pOldUser->user); + memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); + memcpy(pOldUser->pass, pNewUser->pass, TSDB_KEY_LEN); + memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); + pOldUser->createdTime = pNewUser->createdTime; + pOldUser->updateTime = pNewUser->updateTime; + pOldUser->superAuth = pNewUser->superAuth; + pOldUser->readAuth = pNewUser->readAuth; + pOldUser->writeAuth = pNewUser->writeAuth; return 0; } @@ -197,11 +203,15 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, userObj.writeAuth = 1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); - if (pTrans == NULL) return -1; + if (pTrans == NULL) { + mError("user:%s, failed to create since %s", user, terrstr()); + return -1; + } + mDebug("trans:%d, used to create user:%s", pTrans->id, user); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("failed to append redo log since %s", terrstr()); + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } @@ -209,7 +219,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("failed to append undo log since %s", terrstr()); + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } @@ -217,13 +227,90 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("failed to append commit log since %s", terrstr()); + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans, mndSyncPropose) != 0) { + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(pOldUser); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to drop since %s", pUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndUserActionEncode(pUser); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } @@ -236,6 +323,8 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; + mDebug("user:%s, start to create", pCreate->user); + if (pCreate->user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; mError("user:%s, failed to create since %s", pCreate->user, terrstr()); @@ -275,13 +364,185 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { } static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; - mError("failed to process alter user msg since %s", terrstr()); - return -1; + SMnode *pMnode = pMsg->pMnode; + SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; + + mDebug("user:%s, start to alter", pAlter->user); + + if (pAlter->user[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + if (pAlter->pass[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + SUserObj *pUser = sdbAcquire(pMnode->pSdb, SDB_USER, pAlter->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + SUserObj *pOperUser = sdbAcquire(pMnode->pSdb, SDB_USER, pMsg->user); + if (pOperUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + SUserObj newUser = {0}; + memcpy(&newUser, pUser, sizeof(SUserObj)); + memset(pUser->pass, 0, sizeof(pUser->pass)); + taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + + int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg); + sdbRelease(pMnode->pSdb, pOperUser); + + if (code != 0) { + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; - mError("failed to process drop user msg since %s", terrstr()); - return -1; + SMnode *pMnode = pMsg->pMnode; + SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("user:%s, start to drop", pDrop->user); + + if (pDrop->user[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + SUserObj *pUser = sdbAcquire(pMnode->pSdb, SDB_USER, pDrop->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + SUserObj *pOperUser = sdbAcquire(pMnode->pSdb, SDB_USER, pMsg->user); + if (pOperUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + int32_t code = mndDropUser(pMnode, pMsg, pUser); + sdbRelease(pMnode->pSdb, pOperUser); + + if (code != 0) { + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "privilege"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "account"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SUserObj *pUser = NULL; + int32_t cols = 0; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pUser->superAuth) { + const char *src = "super"; + STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); + } else if (pUser->writeAuth) { + const char *src = "writable"; + STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); + } else { + const char *src = "readable"; + STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); + } + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pUser->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->acct, pShow->bytes[cols]); + cols++; + + numOfRows++; + sdbRelease(pSdb, pUser); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b98468b63e..a0af5f7f5e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -14,8 +14,372 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndVgroup.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndTrans.h" -int32_t mndInitVgroup(SMnode *pMnode) { return 0; } -void mndCleanupVgroup(SMnode *pMnode) {} \ No newline at end of file +#define TSDB_VGROUP_VER_NUM 1 +#define TSDB_VGROUP_RESERVE_SIZE 64 + +static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup); + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg); + +static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); +static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); + +int32_t mndInitVgroup(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_VGROUP, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionDelete, + .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP, mndProcessCreateVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP, mndProcessAlterVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_VNODE_IN_RSP, mndProcessDropVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP, mndProcessSyncVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP, mndProcessCompactVnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndGetVgroupMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndGetVnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndRetrieveVnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndCancelGetNextVnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupVgroup(SMnode *pMnode) {} + +static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SDbObj)); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica) + for (int8_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId) + SDB_SET_INT8(pRaw, dataPos, pVgid->role) + } + SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_VGROUP_VER_NUM) { + mError("failed to decode vgroup since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); + SVgObj *pVgroup = sdbGetRowObj(pRow); + if (pVgroup == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->vgId) + SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->updateTime) + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->version) + SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_GET_INT8(pRaw, pRow, dataPos, &pVgroup->replica) + for (int8_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgid->dnodeId) + SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pVgid->role) + } + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_VGROUP_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) { + mTrace("vgId:%d, perform insert action", pVgroup->vgId); + return 0; +} + +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) { + mTrace("vgId:%d, perform delete action", pVgroup->vgId); + return 0; +} + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { + mTrace("vgId:%d, perform update action", pOldVgroup->vgId); + pOldVgroup->vgId = pNewVgroup->vgId; + pOldVgroup->createdTime = pNewVgroup->createdTime; + pOldVgroup->updateTime = pNewVgroup->updateTime; + pOldVgroup->version = pNewVgroup->version; + memcpy(pOldVgroup->dbName, pNewVgroup->dbName, TSDB_FULL_DB_NAME_LEN); + pOldVgroup->replica = pNewVgroup->replica; + memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); + return 0; +} + +SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_VGROUP, &vgId); +} + +void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pVgroup); +} + +static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int8_t replica = 1; + int32_t numOfVgroups = 0; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (strcmp(pVgroup->dbName, dbName) == 0) { + replica = MAX(replica, pVgroup->replica); + numOfVgroups++; + } + + sdbRelease(pSdb, pVgroup); + } + + *pReplica = replica; + *pNumOfVgroups = numOfVgroups; + return 0; +} + +static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "vgId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "tables"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + for (int32_t i = 0; i < pShow->replica; ++i) { + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + } + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SVgObj *pVgroup = NULL; + int32_t cols = 0; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->vgId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->numOfTables; + cols++; + + for (int32_t i = 0; i < pShow->replica; ++i) { + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; + cols++; + + const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + cols++; + } + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pVgroup->compact; + cols++; + + sdbRelease(pSdb, pVgroup); + numOfRows++; + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} + +static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { + if (dnodeId == 0) { + return 0; + } + + return 0; +} + +static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "vgId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + int32_t dnodeId = 0; + if (pShow->payloadLen > 0) { + dnodeId = atoi(pShow->payload); + } + + pShow->replica = dnodeId; + pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SVgObj *pVgroup = NULL; + char *pWrite; + int32_t cols = 0; + int32_t dnodeId = pShow->replica; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); + if (pShow->pIter == NULL) break; + + for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + if (pVgid->dnodeId != dnodeId) continue; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(uint32_t *)pWrite = pVgroup->vgId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, mndGetRoleStr(pVgid->role)); + cols++; + numOfRows++; + } + + sdbRelease(pSdb, pVgroup); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 2fa0838ee8..3e0ac746fc 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -22,7 +22,6 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" -#include "mndOper.h" #include "mndProfile.h" #include "mndShow.h" #include "mndStable.h" @@ -333,7 +332,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); if (pMsg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to create msg since %s", terrstr()); + mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr()); return NULL; } @@ -341,7 +340,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { if (rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { mndCleanupMsg(pMsg); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - mError("failed to create msg since %s", terrstr()); + mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr()); return NULL; } memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); @@ -350,13 +349,13 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { pMsg->rpcMsg = *pRpcMsg; pMsg->createdTime = taosGetTimestampSec(); - mTrace("msg:%p, is created", pMsg); + mTrace("msg:%p, app:%p is created, RPC:%p", pMsg, pRpcMsg->ahandle, pRpcMsg->handle); return pMsg; } void mndCleanupMsg(SMnodeMsg *pMsg) { + mTrace("msg:%p, app:%p is destroyed, RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); taosFreeQitem(pMsg); - mTrace("msg:%p, is destroyed", pMsg); } void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { @@ -371,7 +370,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType % 2 == 1); - mTrace("msg:%p, app:%p will be processed", pMsg, ahandle); + mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, taosMsg[msgType]); if (isReq && !mndIsMaster(pMnode)) { code = TSDB_CODE_APP_NOT_READY; @@ -393,7 +392,10 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { } code = (*fp)(pMsg); - if (code != 0) { + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mTrace("msg:%p, app:%p in progressing", pMsg, ahandle); + return; + } else if (code != 0) { code = terrno; mError("msg:%p, app:%p failed to process since %s", pMsg, ahandle, terrstr()); goto PROCESS_RPC_END; diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index e492f28557..a160533bf2 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -71,7 +71,6 @@ typedef struct SSdb { } SSdb; int32_t sdbWriteFile(SSdb *pSdb); -int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw); #ifdef __cplusplus } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 68cb7be68b..1d4888c2eb 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -94,7 +94,7 @@ void sdbCleanup(SSdb *pSdb) { taosHashClear(hash); taosHashCleanup(hash); pSdb->hashObjs[i] = NULL; - mTrace("sdb table:%d is cleaned up", i); + mDebug("sdb table:%d is cleaned up", i); } free(pSdb); @@ -129,7 +129,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { pSdb->hashObjs[sdbType] = hash; taosInitRWLatch(&pSdb->locks[sdbType]); - mTrace("sdb table:%d is initialized", sdbType); + mDebug("sdb table:%d is initialized", sdbType); return 0; } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6f88f08b2c..af37e9e1d5 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -118,7 +118,7 @@ int32_t sdbReadFile(SSdb *pSdb) { break; } - code = sdbWriteRaw(pSdb, pRaw); + code = sdbWriteNotFree(pSdb, pRaw); if (code != 0) { mError("failed to read file:%s since %s", file, terrstr()); goto PARSE_SDB_DATA_ERROR; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index bdca5eaa98..8d8daf5ce5 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -52,11 +52,12 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SRWLatch *pLock = &pSdb->locks[pRow->type]; taosWLockLatch(pLock); - SSdbRow *pDstRow = taosHashGet(hash, pRow->pObj, keySize); - if (pDstRow != NULL) { + SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); + if (pOldRow != NULL) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return TSDB_CODE_SDB_OBJ_ALREADY_THERE; + terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; + return terrno; } pRow->refCount = 1; @@ -65,7 +66,8 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return TSDB_CODE_SDB_OBJ_ALREADY_THERE; + terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; + return terrno; } taosWUnLockLatch(pLock); @@ -78,35 +80,36 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return code; + terrno = code; + return terrno; } } return 0; } -static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { +static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pNewRow, int32_t keySize) { int32_t code = 0; - SRWLatch *pLock = &pSdb->locks[pRow->type]; + SRWLatch *pLock = &pSdb->locks[pNewRow->type]; taosRLockLatch(pLock); - SSdbRow **ppDstRow = taosHashGet(hash, pRow->pObj, keySize); - if (ppDstRow == NULL || *ppDstRow == NULL) { + SSdbRow **ppOldRow = taosHashGet(hash, pNewRow->pObj, keySize); + if (ppOldRow == NULL || *ppOldRow == NULL) { taosRUnLockLatch(pLock); - return sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); + return sdbInsertRow(pSdb, hash, pRaw, pNewRow, keySize); } - SSdbRow *pDstRow = *ppDstRow; + SSdbRow *pOldRow = *ppOldRow; - pRow->status = pRaw->status; + pOldRow->status = pRaw->status; taosRUnLockLatch(pLock); - SdbUpdateFp updateFp = pSdb->updateFps[pRow->type]; + SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; if (updateFp != NULL) { - code = (*updateFp)(pSdb, pRow->pObj, pDstRow->pObj); + code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - sdbFreeRow(pRow); + sdbFreeRow(pNewRow); return code; } @@ -116,29 +119,30 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SRWLatch *pLock = &pSdb->locks[pRow->type]; taosWLockLatch(pLock); - SSdbRow **ppDstRow = taosHashGet(hash, pRow->pObj, keySize); - if (ppDstRow == NULL || *ppDstRow == NULL) { + SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); + if (ppOldRow == NULL || *ppOldRow == NULL) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return TSDB_CODE_SDB_OBJ_NOT_THERE; + terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; + return terrno; } - SSdbRow *pDstRow = *ppDstRow; + SSdbRow *pOldRow = *ppOldRow; - pDstRow->status = pRaw->status; - taosHashRemove(hash, pDstRow->pObj, keySize); + pOldRow->status = pRaw->status; + taosHashRemove(hash, pOldRow->pObj, keySize); taosWUnLockLatch(pLock); - SdbDeleteFp deleteFp = pSdb->deleteFps[pDstRow->type]; + SdbDeleteFp deleteFp = pSdb->deleteFps[pOldRow->type]; if (deleteFp != NULL) { - code = (*deleteFp)(pSdb, pDstRow->pObj); + code = (*deleteFp)(pSdb, pOldRow->pObj); } - sdbRelease(pSdb, pDstRow->pObj); + sdbRelease(pSdb, pOldRow->pObj); sdbFreeRow(pRow); return code; } -int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw) { +int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { SHashObj *hash = sdbGetHash(pSdb, pRaw->type); if (hash == NULL) return terrno; @@ -170,7 +174,7 @@ int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw) { } int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { - int32_t code = sdbWriteRaw(pSdb, pRaw); + int32_t code = sdbWriteNotFree(pSdb, pRaw); sdbFreeRaw(pRaw); return code; } diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 7ed1a427f5..e37559808e 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -26,10 +26,15 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->type = type; pRaw->sver = sver; pRaw->dataLen = dataLen; + + mTrace("raw:%p, is created, len:%d", pRaw, dataLen); return pRaw; } -void sdbFreeRaw(SSdbRaw *pRaw) { free(pRaw); } +void sdbFreeRaw(SSdbRaw *pRaw) { + mTrace("raw:%p, is freed", pRaw); + free(pRaw); +} int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { if (pRaw == NULL) { diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 040b02d2b6..d6d267c4d4 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) \ No newline at end of file +#if(${BUILD_TEST}) +# add_subdirectory(test) +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 493fe4448b..df784181b7 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -45,7 +45,7 @@ static SKVRow createBasicTag() { tdInitKVRowBuilder(&rb); - for (int i = 10; i < 12; i++) { + for (int i = 0; i < 2; i++) { void *pVal = malloc(sizeof(VarDataLenT) + strlen("foo")); varDataLen(pVal) = strlen("foo"); memcpy(varDataVal(pVal), "foo", strlen("foo")); @@ -120,7 +120,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { { // Create some child tables - int ntables = 100000; + int ntables = 1000000; int batch = 10; for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 7bbe7ddc71..3c1ccc72dc 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -20,6 +20,11 @@ #include "tcoding.h" #include "thash.h" +typedef struct { + tb_uid_t uid; + int32_t sver; +} SSchemaKey; + struct SMetaDB { // DB DB *pTbDB; @@ -33,15 +38,25 @@ struct SMetaDB { DB_ENV *pEvn; }; +typedef int (*bdbIdxCbPtr)(DB *, const DBT *, const DBT *, DBT *); + static SMetaDB *metaNewDB(); static void metaFreeDB(SMetaDB *pDB); static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path); static void metaCloseBDBEnv(DB_ENV *pEnv); -static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName); +static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup); static void metaCloseBDBDb(DB *pDB); +static int metaOpenBDBIdx(DB **ppIdx, DB_ENV *pEnv, const char *pFName, DB *pDB, bdbIdxCbPtr cbf, bool isDup); +static void metaCloseBDBIdx(DB *pIdx); +static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); +static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); +static void metaClearTbCfg(STbCfg *pTbCfg); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) -#define metaOpenBDBIdx metaOpenBDBDb int metaOpenDB(SMeta *pMeta) { SMetaDB *pDB; @@ -61,43 +76,48 @@ int metaOpenDB(SMeta *pMeta) { } // Open DBs - if (metaOpenBDBDb(&(pDB->pTbDB), pDB->pEvn, "meta.db") < 0) { + if (metaOpenBDBDb(&(pDB->pTbDB), pDB->pEvn, "meta.db", false) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "meta.db") < 0) { + if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "meta.db", false) < 0) { metaCloseDB(pMeta); return -1; } // Open Indices - if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBIdx(&(pDB->pStbIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pStbIdx), pDB->pEvn, "stb.index", pDB->pTbDB, &metaStbIdxCb, false) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBIdx(&(pDB->pNtbIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pNtbIdx), pDB->pEvn, "ntb.index", pDB->pTbDB, &metaNtbIdxCb, false) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBIdx(&(pDB->pCtbIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pCtbIdx), pDB->pEvn, "ctb.index", pDB->pTbDB, &metaCtbIdxCb, true) < 0) { metaCloseDB(pMeta); return -1; } - // Associate Indices return 0; } void metaCloseDB(SMeta *pMeta) { if (pMeta->pDB) { + metaCloseBDBIdx(pMeta->pDB->pCtbIdx); + metaCloseBDBIdx(pMeta->pDB->pNtbIdx); + metaCloseBDBIdx(pMeta->pDB->pStbIdx); + metaCloseBDBIdx(pMeta->pDB->pNameIdx); + metaCloseBDBDb(pMeta->pDB->pSchemaDB); + metaCloseBDBDb(pMeta->pDB->pTbDB); metaCloseBDBEnv(pMeta->pDB->pEvn); metaFreeDB(pMeta->pDB); pMeta->pDB = NULL; @@ -105,7 +125,60 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - // TODO + tb_uid_t uid; + char buf[512]; + void * pBuf; + DBT key, value; + STSchema *pSchema = NULL; + + if (pTbCfg->type == META_SUPER_TABLE) { + uid = pTbCfg->stbCfg.suid; + } else { + uid = metaGenerateUid(pMeta); + } + + { + // save table info + pBuf = buf; + memset(&key, 0, sizeof(key)); + memset(&value, 0, sizeof(key)); + + key.data = &uid; + key.size = sizeof(uid); + + metaEncodeTbInfo(&pBuf, pTbCfg); + + value.data = buf; + value.size = POINTER_DISTANCE(pBuf, buf); + value.app_data = pTbCfg; + + pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key, &value, 0); + } + + // save schema + if (pTbCfg->type == META_SUPER_TABLE) { + pSchema = pTbCfg->stbCfg.pSchema; + } else if (pTbCfg->type == META_NORMAL_TABLE) { + pSchema = pTbCfg->ntbCfg.pSchema; + } + + if (pSchema) { + pBuf = buf; + memset(&key, 0, sizeof(key)); + memset(&value, 0, sizeof(key)); + SSchemaKey schemaKey = {uid, schemaVersion(pSchema)}; + + key.data = &schemaKey; + key.size = sizeof(schemaKey); + + tdEncodeSchema(&pBuf, pSchema); + + value.data = buf; + value.size = POINTER_DISTANCE(pBuf, buf); + + pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key, &value, 0); + } + return 0; } @@ -160,22 +233,32 @@ static void metaCloseBDBEnv(DB_ENV *pEnv) { } } -static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName) { +static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup) { int ret; DB *pDB; - ret = db_create(&((pDB)), (pEnv), 0); + ret = db_create(&(pDB), pEnv, 0); if (ret != 0) { BDB_PERR("Failed to create META DB", ret); return -1; } + if (isDup) { + ret = pDB->set_flags(pDB, DB_DUPSORT); + if (ret != 0) { + BDB_PERR("Failed to set DB flags", ret); + return -1; + } + } + ret = pDB->open(pDB, NULL, pFName, NULL, DB_BTREE, DB_CREATE, 0); if (ret) { BDB_PERR("Failed to open META DB", ret); return -1; } + *ppDB = pDB; + return 0; } @@ -185,139 +268,94 @@ static void metaCloseBDBDb(DB *pDB) { } } -#if 0 -typedef struct { - tb_uid_t uid; - int32_t sver; -} SSchemaKey; +static int metaOpenBDBIdx(DB **ppIdx, DB_ENV *pEnv, const char *pFName, DB *pDB, bdbIdxCbPtr cbf, bool isDup) { + DB *pIdx; + int ret; + if (metaOpenBDBDb(ppIdx, pEnv, pFName, isDup) < 0) { + return -1; + } -static SMetaDB *metaNewDB(); -static void metaFreeDB(SMetaDB *pDB); -static int metaCreateDBEnv(SMetaDB *pDB, const char *path); -static void metaDestroyDBEnv(SMetaDB *pDB); -static int metaEncodeSchemaKey(void **buf, SSchemaKey *pSchemaKey); -static void * metaDecodeSchemaKey(void *buf, SSchemaKey *pSchemaKey); -static int metaNameIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static int metaUidIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey); -static void metaPutSchema(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema); -static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); -static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); -static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg); - -#define META_ASSOCIATE_IDX(pDB, pIdx, cbf) \ - do { \ - int ret = (pDB)->associate((pDB), NULL, (pIdx), (cbf), 0); \ - if (ret != 0) { \ - P_ERROR("Failed to associate META DB", ret); \ - metaCloseDB(pMeta); \ - } \ - } while (0) - - -int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - char buf[512]; - void * pBuf; - DBT key = {0}; - DBT value = {0}; - SSchemaKey schemaKey; - tb_uid_t uid; - - if (pTbCfg->type == META_SUPER_TABLE) { - // Handle SUPER table - uid = pTbCfg->stbCfg.suid; - - // Same table info - metaSaveTbInfo(pMeta->pDB->pStbDB, uid, pTbCfg); - - // save schema - metaPutSchema(pMeta, uid, pTbCfg->stbCfg.pSchema); - - { - // Create a super table DB and corresponding index DB - DB *pStbDB; - DB *pStbIdxDB; - - META_OPEN_DB(pStbDB, pMeta->pDB->pEvn, "meta.db"); - - META_OPEN_DB(pStbIdxDB, pMeta->pDB->pEvn, "index.db"); - - // TODO META_ASSOCIATE_IDX(); - } - } else if (pTbCfg->type == META_CHILD_TABLE) { - // Handle CHILD table - uid = metaGenerateUid(pMeta); - - DB *pCTbDB = taosHashGet(pMeta->pDB->pCtbMap, &(pTbCfg->ctbCfg.suid), sizeof(pTbCfg->ctbCfg.suid)); - if (pCTbDB == NULL) { - ASSERT(0); - } - - metaSaveTbInfo(pCTbDB, uid, pTbCfg); - - } else if (pTbCfg->type == META_NORMAL_TABLE) { - // Handle NORMAL table - uid = metaGenerateUid(pMeta); - - metaSaveTbInfo(pMeta->pDB->pNtbDB, uid, pTbCfg); - - metaPutSchema(pMeta, uid, pTbCfg->stbCfg.pSchema); - } else { - ASSERT(0); + pIdx = *ppIdx; + ret = pDB->associate(pDB, NULL, pIdx, cbf, 0); + if (ret) { + BDB_PERR("Failed to associate META DB and Index", ret); } return 0; } -int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { - // TODO +static void metaCloseBDBIdx(DB *pIdx) { + if (pIdx) { + pIdx->close(pIdx, 0); + } } -/* ------------------------ STATIC METHODS ------------------------ */ -static int metaEncodeSchemaKey(void **buf, SSchemaKey *pSchemaKey) { - int tsize = 0; +static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - tsize += taosEncodeFixedU64(buf, pSchemaKey->uid); - tsize += taosEncodeFixedI32(buf, pSchemaKey->sver); + memset(pSKey, 0, sizeof(*pSKey)); - return tsize; -} + pSKey->data = pTbCfg->name; + pSKey->size = strlen(pTbCfg->name); -static void *metaDecodeSchemaKey(void *buf, SSchemaKey *pSchemaKey) { - buf = taosDecodeFixedU64(buf, &(pSchemaKey->uid)); - buf = taosDecodeFixedI32(buf, &(pSchemaKey->sver)); - - return buf; -} - -static int metaNameIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - // TODO return 0; } -static int metaUidIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey) { - // TODO - return 0; +static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); + + if (pTbCfg->type == META_SUPER_TABLE) { + memset(pSKey, 0, sizeof(*pSKey)); + pSKey->data = pKey->data; + pSKey->size = pKey->size; + + return 0; + } else { + return DB_DONOTINDEX; + } } -static void metaPutSchema(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema) { - SSchemaKey skey; - char buf[256]; - void * pBuf = buf; - DBT key = {0}; - DBT value = {0}; +static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - skey.uid = uid; - skey.sver = schemaVersion(pSchema); + if (pTbCfg->type == META_NORMAL_TABLE) { + memset(pSKey, 0, sizeof(*pSKey)); + pSKey->data = pKey->data; + pSKey->size = pKey->size; - key.data = &skey; - key.size = sizeof(skey); + return 0; + } else { + return DB_DONOTINDEX; + } +} - tdEncodeSchema(&pBuf, pSchema); - value.data = buf; - value.size = POINTER_DISTANCE(pBuf, buf); +static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); + DBT * pDbt; - pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key, &value, 0); + if (pTbCfg->type == META_CHILD_TABLE) { + pDbt = calloc(2, sizeof(DBT)); + + // First key is suid + pDbt[0].data = &(pTbCfg->ctbCfg.suid); + pDbt[0].size = sizeof(pTbCfg->ctbCfg.suid); + + // Second key is the first tag + void *pTagVal = tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, 0); + pDbt[1].data = varDataVal(pTagVal); + pDbt[1].size = varDataLen(pTagVal); + + // Set index key + memset(pSKey, 0, sizeof(*pSKey)); + pSKey->flags = DB_DBT_MULTIPLE | DB_DBT_APPMALLOC; + pSKey->data = pDbt; + pSKey->size = 2; + + return 0; + } else { + return DB_DONOTINDEX; + } } static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { @@ -326,6 +364,7 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeString(buf, pTbCfg->name); tsize += taosEncodeFixedU32(buf, pTbCfg->ttl); tsize += taosEncodeFixedU32(buf, pTbCfg->keep); + tsize += taosEncodeFixedU8(buf, pTbCfg->type); if (pTbCfg->type == META_SUPER_TABLE) { tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); @@ -341,10 +380,10 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { } static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { - // TODO buf = taosDecodeString(buf, &(pTbCfg->name)); buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl)); buf = taosDecodeFixedU32(buf, &(pTbCfg->keep)); + buf = taosDecodeFixedU8(buf, &(pTbCfg->type)); if (pTbCfg->type == META_SUPER_TABLE) { buf = tdDecodeSchema(buf, &(pTbCfg->stbCfg.pTagSchema)); @@ -358,22 +397,11 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { return buf; } -static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg) { - DBT key = {0}; - DBT value = {0}; - char buf[512]; - void *pBuf = buf; - - key.data = &uid; - key.size = sizeof(uid); - - metaEncodeTbInfo(&pBuf, pTbCfg); - - value.data = buf; - value.size = POINTER_DISTANCE(pBuf, buf); - - pDB->put(pDB, NULL, &key, &value, 0); - - return 0; -} -#endif \ No newline at end of file +static void metaClearTbCfg(STbCfg *pTbCfg) { + tfree(pTbCfg->name); + if (pTbCfg->type == META_SUPER_TABLE) { + tdFreeSchema(pTbCfg->stbCfg.pTagSchema); + } else if (pTbCfg->type == META_CHILD_TABLE) { + tfree(pTbCfg->ctbCfg.pTag); + } +} \ No newline at end of file diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 9e70b9a68d..36c9e2513f 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -343,7 +343,7 @@ bool isProjectionQueryOnSTable(SArray* pFunctionIdList, int32_t tableIndex) { // // if (functionId < 0) { // SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); -// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { +// if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { // return false; // } // diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 095c5a6bb0..2771a9ecf8 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -146,7 +146,7 @@ void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, i pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes, &pUdfInfo->init); } - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { pCtx->resultInfo->numOfRes = output; } else { pCtx->resultInfo->numOfRes += output; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 0cc95b738d..6a81e888d2 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -167,7 +167,10 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, break; } if (addPrefix != 0) { - fstBuilderNodeUnfinishedAddOutputPrefix(un, addPrefix); + if (i + 1 < ssz) { + FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); + fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); + } } } return i; @@ -294,15 +297,16 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil // any value greater than or equal to the number of transitions in // this node indicates an absent transition. uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); - for (uint8_t i = 0; i < 256; i++) { - index[i] = 255; - } + memset(index, 255, sizeof(uint8_t) * 256); + ///for (uint8_t i = 0; i < 256; i++) { + // index[i] = 255; + ///} for (size_t i = 0; i < sz; i++) { FstTransition *t = taosArrayGet(node->trans, i); index[t->inp] = i; - fstCountingWriterWrite(w, (char *)index, sizeof(index)); //fstPackDeltaIn(w, addr, t->addr, tSize); } + fstCountingWriterWrite(w, (char *)index, 256); free(index); } fstCountingWriterWrite(w, (char *)&packSizes, 1); @@ -475,6 +479,7 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { return 0; } FstSlice *slice = &node->data; + uint8_t *data = fstSliceData(slice, NULL); uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -482,7 +487,6 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { - (i * oSizes) - oSizes; - uint8_t *data = fstSliceData(slice, NULL); return unpackUint64(data + at, oSizes); } @@ -552,6 +556,7 @@ Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, Pack uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) + - 1 // pack size - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; @@ -581,18 +586,19 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { - 1 // pack size - node->nTrans; uint64_t end = start + node->nTrans; - uint64_t len = end - start; - int32_t dlen = 0; - uint8_t *data = fstSliceData(slice, &dlen); - for(int i = 0; i < len; i++) { - //uint8_t v = slice->data[slice->start + i]; - ////slice->data[slice->start + i]; + FstSlice t = fstSliceCopy(slice, start, end - 1); + int32_t len = 0; + uint8_t *data = fstSliceData(&t, &len); + int i = 0; + for(; i < len; i++) { uint8_t v = data[i]; - if (v == b) { + fstSliceDestroy(&t); return node->nTrans - i - 1; // bug } } + if (i == len) { *null = true; } + fstSliceDestroy(&t); } } @@ -628,7 +634,7 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { } else if (st.state == OneTrans) { FstSlice data = fstSliceCopy(slice, 0, addr); PackSizes sz = fstStateSizes(&st, &data); - n->data = fstSliceCopy(slice, 0, addr); + n->data = data; n->version = version; n->state = st; n->start = addr; @@ -772,7 +778,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { if (NULL == b) { return b; } - b->wrt = fstCountingWriterCreate(w, false); + b->wrt = fstCountingWriterCreate(w, false); b->unfinished = fstUnFinishedNodesCreate(); b->registry = fstRegistryCreate(10000, 2) ; b->last = fstSliceCreate(NULL, 0); @@ -797,6 +803,7 @@ void fstBuilderDestroy(FstBuilder *b) { fstCountingWriterDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); + fstSliceDestroy(&b->last); free(b); } @@ -845,8 +852,9 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { FstSlice *input = &bs; if (fstSliceIsEmpty(&b->last)) { + fstSliceDestroy(&b->last); // deep copy or not - b->last = fstSliceCopy(&bs, input->start, input->end); + b->last = fstSliceDeepCopy(&bs, input->start, input->end); } else { int comp = fstSliceCompare(&b->last, &bs); if (comp == 0 && ckDup) { @@ -855,20 +863,23 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { return OutOfOrdered; } // deep copy or not - b->last = fstSliceCopy(&bs, input->start, input->end); + fstSliceDestroy(&b->last); + b->last = fstSliceDeepCopy(&bs, input->start, input->end); } return Ordered; } void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { CompiledAddr addr = NONE_ADDRESS; while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { - FstBuilderNode *n = NULL; + FstBuilderNode *bn = NULL; if (addr == NONE_ADDRESS) { - n = fstUnFinishedNodesPopEmpty(b->unfinished); + bn = fstUnFinishedNodesPopEmpty(b->unfinished); } else { - n = fstUnFinishedNodesPopFreeze(b->unfinished, addr); + bn = fstUnFinishedNodesPopFreeze(b->unfinished, addr); } - addr = fstBuilderCompile(b, n); + addr = fstBuilderCompile(b, bn); + + fstBuilderNodeDestroy(bn); assert(addr != NONE_ADDRESS); //fstBuilderNodeDestroy(n); } @@ -903,6 +914,7 @@ void* fstBuilderInsertInner(FstBuilder *b) { fstBuilderCompileFrom(b, 0); FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); + fstBuilderNodeDestroy(rootNode); char buf64[8] = {0}; @@ -1005,8 +1017,7 @@ Fst* fstCreate(FstSlice *slice) { uint64_t fstLen; len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); - //TODO(validat root addr) - // + //TODO(validate root addr) Fst *fst= (Fst *)calloc(1, sizeof(Fst)); if (fst == NULL) { return NULL; } @@ -1020,7 +1031,11 @@ Fst* fstCreate(FstSlice *slice) { fst->meta->ty = type; fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - fst->data = slice; + + FstSlice *s = calloc(1, sizeof(FstSlice)); + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); + fst->data = s; + return fst; FST_CREAT_FAILED: @@ -1031,7 +1046,8 @@ FST_CREAT_FAILED: void fstDestroy(Fst *fst) { if (fst) { free(fst->meta); - fstNodeDestroy(fst->root); + fstSliceDestroy(fst->data); + free(fst->data); } free(fst); } @@ -1041,6 +1057,9 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { Output tOut = 0; int32_t len; uint8_t *data = fstSliceData(b, &len); + + SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); + taosArrayPush(nodes, &root); for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; @@ -1052,15 +1071,25 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { fstNodeGetTransitionAt(root, res, &trn); tOut += trn.out; root = fstGetNode(fst, trn.addr); + taosArrayPush(nodes, &root); + //fstNodeDestroy(root); } if (!FST_NODE_IS_FINAL(root)) { return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); } + + for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { + FstNode **node = (FstNode **)taosArrayGet(nodes, i); + fstNodeDestroy(*node); + } + taosArrayDestroy(nodes); + + fst->root = NULL; *out = tOut; - return false; + return true; } FstNode *fstGetRoot(Fst *fst) { @@ -1120,6 +1149,7 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { return b; } + bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { int comp = fstSliceCompare(slice, &bound->data); if (bound->type == Included) { @@ -1220,6 +1250,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); + fstNodeDestroy(node); } else { // This is a little tricky. We're in this case if the @@ -1372,7 +1403,9 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { } void fstStreamBuilderDestroy(FstStreamBuilder *b) { fstSliceDestroy(&b->min->data); + tfree(b->min); fstSliceDestroy(&b->max->data); + tfree(b->max); free(b); } FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 858861529d..928c3875b0 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -2,13 +2,79 @@ #include #include #include "index.h" +#include "tutil.h" #include "indexInt.h" #include "index_fst.h" #include "index_fst_util.h" #include "index_fst_counting_writer.h" +class FstWriter { + public: + FstWriter() { + _b = fstBuilderCreate(NULL, 0); + } + bool Put(const std::string &key, uint64_t val) { + FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); + bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); + return ok; + } + ~FstWriter() { + fstBuilderFinish(_b); + fstBuilderDestroy(_b); + } + private: + FstBuilder *_b; +}; +class FstReadMemory { + public: + FstReadMemory(size_t size) { + _w = fstCountingWriterCreate(NULL, true); + _size = size; + memset((void *)&_s, 0, sizeof(_s)); + } + bool init() { + char *buf = (char *)calloc(1, sizeof(char) * _size); + int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size); + if (nRead <= 0) { return false; } + _size = nRead; + _s = fstSliceCreate((uint8_t *)buf, _size); + _fst = fstCreate(&_s); + free(buf); + return _fst != NULL; + } + bool Get(const std::string &key, uint64_t *val) { + FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size()); + bool ok = fstGet(_fst, &skey, val); + fstSliceDestroy(&skey); + return ok; + } + bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Get(key, val); + int64_t e = taosGetTimestampUs(); + *elapse = e - s; + return ok; + } + // add later + bool Search(const std::string &key, std::vector &result) { + return true; + } + + ~FstReadMemory() { + fstCountingWriterDestroy(_w); + fstSliceDestroy(&_s); + } + + private: + FstCountingWriter *_w; + Fst *_fst; + FstSlice _s; + size_t _size; + +}; //TEST(IndexTest, index_create_test) { // SIndexOpts *opts = indexOptsCreate(); @@ -62,68 +128,104 @@ // // //} -int main(int argc, char** argv) { - // test write - FstBuilder *b = fstBuilderCreate(NULL, 0); - { - std::string str("aaa"); - FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - Output val = 1; - fstBuilderInsert(b, key, val); + + +int Performance_fstWriteRecords(FstWriter *b) { + std::string str("aa"); + int L = 100, M = 100, N = 10; + for (int i = 0; i < L; i++) { + str[0] = 'a' + i; + str.resize(2); + for(int j = 0; j < M; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < N; k++) { + str.push_back('a'); + b->Put(str, k); + printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str()); + } + } } + return L * M * N; +} - //std::string str1("bcd"); - //FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size()); - //Output val2 = 10; - // - - { - - for (size_t i = 1; i < 26; i++) { - std::string str("aaa"); - str[2] = 'a' + i ; - FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - Output val = 2; - fstBuilderInsert(b, key, val); - } - - } - fstBuilderFinish(b); - fstBuilderDestroy(b); +void Performance_fstReadRecords(FstReadMemory *m) { + std::string str("a"); + for (int i = 0; i < 50; i++) { + //std::string str("aa"); + str.push_back('a'); + uint64_t out, cost; + bool ok = m->GetWithTimeCostUs(str, &out, &cost); + if (ok == true) { + printf("success to get (%s, %" PRId64"), time cost: %" PRId64")\n", str.c_str(), out, cost); + } else { + printf("failed to get(%s)\n", str.c_str()); + } + } +} +void checkFstPerf() { + FstWriter *fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int num = Performance_fstWriteRecords(fw); + int64_t e = taosGetTimestampUs(); + printf("write %d record cost %" PRId64"us\n", num, e - s); + delete fw; - - char buf[64 * 1024] = {0}; - - FstSlice s; - - FstCountingWriter *w = fstCountingWriterCreate(NULL, true); - int nRead = fstCountingWriterRead(w, (uint8_t *)buf, sizeof(buf)); - assert(nRead <= sizeof(buf)); - s = fstSliceCreate((uint8_t *)buf, nRead); - fstCountingWriterDestroy(w); - - - // test reader - - - Fst *fst = fstCreate(&s); - { - std::string str("aaa"); - uint64_t out; - - - FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - bool ok = fstGet(fst, &key, &out); - if (ok == true) { - //indexInfo("Get key-value success, %s, %d", str.c_str(), out); + FstReadMemory *m = new FstReadMemory(1024 * 64); + if (m->init()) { + uint64_t val; + if(m->Get("aaaaaaa", &val)) { + std::cout << "succes to Get val: " << val << std::endl; } else { - //indexError("Get key-value failed, %s", str.c_str()); + std::cout << "failed to Get " << std::endl; + } + } +} + + +void validateFst() { + int val = 100; + int count = 100; + FstWriter *fw = new FstWriter; + // write + { + std::string key("ab"); + for (int i = 0; i < count; i++) { + key.push_back('a' + i); + fw->Put(key, val - i); } } - fstSliceDestroy(&s); - + delete fw; - + // read + FstReadMemory *m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + } + + { + std::string key("ab"); + uint64_t out; + if (m->Get(key, &out)) { + printf("success to get (%s, %" PRId64")\n", key.c_str(), out); + } else { + printf("failed to get(%s)\n", key.c_str()); + } + for (int i = 0; i < count; i++) { + key.push_back('a' + i); + if (m->Get(key, &out) ) { + assert(val - i == out); + printf("success to get (%s, %" PRId64")\n", key.c_str(), out); + } else { + printf("failed to get(%s)\n", key.c_str()); + } + } + } + delete m; + +} +int main(int argc, char** argv) { + checkFstPerf(); return 1; } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 04fb3beb54..ca281c9f1c 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -112,13 +112,13 @@ cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { } cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &X, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &X, 0); } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SToken token; tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index f328486556..3b7d1cbc29 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -972,13 +972,6 @@ void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { memset(&pDBInfo->precision, 0, sizeof(SToken)); } -void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) { - setDefaultCreateDbOption(pDBInfo); - - pDBInfo->dbType = TSDB_DB_TYPE_TOPIC; - pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION; -} - // prefix show db.tables; void tSetDbName(SToken *pCpxName, SToken *pDb) { pCpxName->type = pDb->type; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index e98443e2a1..3e83381a76 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1480,7 +1480,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { pTableMeta->tversion = pChild->tversion; pTableMeta->sversion = pChild->sversion; - memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); + memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); int32_t num = pTableMeta->tableInfo.numOfColumns; for(int32_t i = 0; i < num; ++i) { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 7158708989..2d6ec3f864 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2312,14 +2312,14 @@ static void yy_reduce( break; case 26: /* cmd ::= SHOW dbPrefix STABLES */ { - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &yymsp[-1].minor.yy0, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &yymsp[-1].minor.yy0, 0); } break; case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SToken token; tSetDbName(&token, &yymsp[-3].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= SHOW dbPrefix VGROUPS */ diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 56d5228c2d..ed8932ccd0 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -1523,14 +1523,14 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { } static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { - MD5_CTX context; + T_MD5_CTX context; int ret = -1; - MD5Init(&context); - MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - MD5Update(&context, (uint8_t *)pMsg, msgLen); - MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - MD5Final(&context); + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t *)pMsg, msgLen); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; @@ -1538,13 +1538,13 @@ static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { } static void rpcBuildAuthHead(void *pMsg, int msgLen, void *pAuth, void *pKey) { - MD5_CTX context; + T_MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - MD5Update(&context, (uint8_t *)pMsg, msgLen); - MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); - MD5Final(&context); + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t *)pMsg, msgLen); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); memcpy(pAuth, context.digest, sizeof(context.digest)); } diff --git a/source/libs/wal/CMakeLists.txt b/source/libs/wal/CMakeLists.txt index e5697415f1..4d2dd97c87 100644 --- a/source/libs/wal/CMakeLists.txt +++ b/source/libs/wal/CMakeLists.txt @@ -8,6 +8,11 @@ target_include_directories( target_link_libraries( wal + PUBLIC cjson PUBLIC os PUBLIC util ) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 285d7e2576..f56f240904 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -23,9 +23,73 @@ extern "C" { #endif -int walGetFile(SWal* pWal, int32_t version); +//meta section begin +typedef struct WalFileInfo { + int64_t firstVer; + int64_t lastVer; + int64_t createTs; + int64_t closeTs; + int64_t fileSize; +} WalFileInfo; + +static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { + WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; + WalFileInfo* pInfoRight = (WalFileInfo*)pRight; + return compareInt64Val(&pInfoLeft->firstVer, &pInfoRight->firstVer); +} + +static inline int64_t walGetLastFileSize(SWal* pWal) { + WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + return pInfo->fileSize; +} + +static inline int64_t walGetLastFileFirstVer(SWal* pWal) { + WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + return pInfo->firstVer; +} + +static inline int64_t walGetCurFileFirstVer(SWal* pWal) { + WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + return pInfo->firstVer; +} + +static inline int64_t walGetCurFileLastVer(SWal* pWal) { + WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + return pInfo->firstVer; +} + +static inline int64_t walGetCurFileOffset(SWal* pWal) { + WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + return pInfo->fileSize; +} + +static inline bool walCurFileClosed(SWal* pWal) { + return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; +} + +static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { + return (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); +} + +static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { + return sprintf(buf, "%s/%" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); +} + +static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { + return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); +} + +int walReadMeta(SWal* pWal); +int walWriteMeta(SWal* pWal); +int walRollFileInfo(SWal* pWal); + +char* walMetaSerialize(SWal* pWal); +int walMetaDeserialize(SWal* pWal, const char* bytes); +//meta section end int64_t walGetSeq(); +int walSeekVer(SWal *pWal, int64_t ver); +int walRoll(SWal *pWal); #ifdef __cplusplus } diff --git a/source/libs/wal/src/walIndex.c b/source/libs/wal/src/walIndex.c index 1aa64b34b5..bf51be4346 100644 --- a/source/libs/wal/src/walIndex.c +++ b/source/libs/wal/src/walIndex.c @@ -23,27 +23,27 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd = pWal->curIdxTfd; - int64_t logTfd = pWal->curLogTfd; + int64_t idxTfd = pWal->writeIdxTfd; + int64_t logTfd = pWal->writeLogTfd; //seek position - int64_t offset = (ver - pWal->curFileFirstVersion) * WAL_IDX_ENTRY_SIZE; + int64_t offset = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE; code = tfLseek(idxTfd, offset, SEEK_SET); if(code != 0) { - + return -1; } int64_t readBuf[2]; code = tfRead(idxTfd, readBuf, sizeof(readBuf)); if(code != 0) { - + return -1; } //TODO:deserialize ASSERT(readBuf[0] == ver); code = tfLseek(logTfd, readBuf[1], SEEK_CUR); if (code != 0) { - + return -1; } - pWal->curLogOffset = readBuf[1]; + /*pWal->curLogOffset = readBuf[1];*/ pWal->curVersion = ver; return code; } @@ -52,43 +52,43 @@ static int walChangeFile(SWal *pWal, int64_t ver) { int code = 0; int64_t idxTfd, logTfd; char fnameStr[WAL_FILE_LEN]; - code = tfClose(pWal->curLogTfd); + code = tfClose(pWal->writeLogTfd); if(code != 0) { //TODO } - code = tfClose(pWal->curIdxTfd); + code = tfClose(pWal->writeIdxTfd); if(code != 0) { //TODO } + WalFileInfo tmpInfo; + tmpInfo.firstVer = ver; //bsearch in fileSet - int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE); + WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); - int64_t fname = *pRet; - if(fname < pWal->lastFileName) { + int64_t fileFirstVer = pRet->firstVer; + //closed + if(taosArrayGetLast(pWal->fileInfoSet) != pRet) { pWal->curStatus &= ~WAL_CUR_FILE_WRITABLE; - pWal->curFileLastVersion = pRet[1]-1; - sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenRead(fnameStr); - sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenRead(fnameStr); } else { pWal->curStatus |= WAL_CUR_FILE_WRITABLE; - pWal->curFileLastVersion = -1; - sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenReadWrite(fnameStr); - sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenReadWrite(fnameStr); } - pWal->curFileFirstVersion = fname; - pWal->curLogTfd = logTfd; - pWal->curIdxTfd = idxTfd; + pWal->writeLogTfd = logTfd; + pWal->writeIdxTfd = idxTfd; return code; } int walSeekVer(SWal *pWal, int64_t ver) { - if((!(pWal->curStatus & WAL_CUR_FAILED)) - && ver == pWal->curVersion) { + int code; + if((!(pWal->curStatus & WAL_CUR_FAILED)) && ver == pWal->curVersion) { return 0; } if(ver > pWal->lastVersion) { @@ -102,11 +102,16 @@ int walSeekVer(SWal *pWal, int64_t ver) { if(ver < pWal->snapshotVersion) { //TODO: seek snapshotted log, invalid in some cases } - if(ver < pWal->curFileFirstVersion || - (pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) { - walChangeFile(pWal, ver); + if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { + code = walChangeFile(pWal, ver); + if(code != 0) { + return -1; + } } - walSeekFilePos(pWal, ver); - + code = walSeekFilePos(pWal, ver); + if(code != 0) { + return -1; + } + return 0; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c new file mode 100644 index 0000000000..65085bb96d --- /dev/null +++ b/source/libs/wal/src/walMeta.c @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tref.h" +#include "tfile.h" +#include "cJSON.h" +#include "walInt.h" + +#include +#include + +int walRollFileInfo(SWal* pWal) { + int64_t ts = taosGetTimestampSec(); + + SArray* pArray = pWal->fileInfoSet; + if(taosArrayGetSize(pArray) != 0) { + WalFileInfo *pInfo = taosArrayGetLast(pArray); + pInfo->lastVer = pWal->lastVersion; + pInfo->closeTs = ts; + } + + WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); + if(pNewInfo == NULL) { + return -1; + } + pNewInfo->firstVer = pWal->lastVersion + 1; + pNewInfo->lastVer = -1; + pNewInfo->createTs = ts; + pNewInfo->closeTs = -1; + pNewInfo->fileSize = 0; + taosArrayPush(pWal->fileInfoSet, pNewInfo); + return 0; +} + +char* walMetaSerialize(SWal* pWal) { + char buf[30]; + if(pWal == NULL || pWal->fileInfoSet == NULL) return 0; + int sz = pWal->fileInfoSet->size; + cJSON* pRoot = cJSON_CreateObject(); + cJSON* pMeta = cJSON_CreateObject(); + cJSON* pFiles = cJSON_CreateArray(); + cJSON* pField; + if(pRoot == NULL || pMeta == NULL || pFiles == NULL) { + //TODO + return NULL; + } + cJSON_AddItemToObject(pRoot, "meta", pMeta); + sprintf(buf, "%" PRId64, pWal->firstVersion); + cJSON_AddStringToObject(pMeta, "firstVer", buf); + sprintf(buf, "%" PRId64, pWal->snapshotVersion); + cJSON_AddStringToObject(pMeta, "snapshotVer", buf); + sprintf(buf, "%" PRId64, pWal->commitVersion); + cJSON_AddStringToObject(pMeta, "commitVer", buf); + sprintf(buf, "%" PRId64, pWal->lastVersion); + cJSON_AddStringToObject(pMeta, "lastVer", buf); + + cJSON_AddItemToObject(pRoot, "files", pFiles); + WalFileInfo* pData = pWal->fileInfoSet->pData; + for(int i = 0; i < sz; i++) { + WalFileInfo* pInfo = &pData[i]; + cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()); + if(pField == NULL) { + cJSON_Delete(pRoot); + return NULL; + } + //cjson only support int32_t or double + //string are used to prohibit the loss of precision + sprintf(buf, "%" PRId64, pInfo->firstVer); + cJSON_AddStringToObject(pField, "firstVer", buf); + sprintf(buf, "%" PRId64, pInfo->lastVer); + cJSON_AddStringToObject(pField, "lastVer", buf); + sprintf(buf, "%" PRId64, pInfo->createTs); + cJSON_AddStringToObject(pField, "createTs", buf); + sprintf(buf, "%" PRId64, pInfo->closeTs); + cJSON_AddStringToObject(pField, "closeTs", buf); + sprintf(buf, "%" PRId64, pInfo->fileSize); + cJSON_AddStringToObject(pField, "fileSize", buf); + } + return cJSON_Print(pRoot); +} + +int walMetaDeserialize(SWal* pWal, const char* bytes) { + ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); + cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; + pRoot = cJSON_Parse(bytes); + pMeta = cJSON_GetObjectItem(pRoot, "meta"); + pField = cJSON_GetObjectItem(pMeta, "firstVer"); + pWal->firstVersion = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); + pWal->snapshotVersion = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pMeta, "commitVer"); + pWal->commitVersion = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pMeta, "lastVer"); + pWal->lastVersion = atoll(cJSON_GetStringValue(pField)); + + pFiles = cJSON_GetObjectItem(pRoot, "files"); + int sz = cJSON_GetArraySize(pFiles); + //deserialize + SArray* pArray = taosArrayInit(sz, sizeof(WalFileInfo)); + WalFileInfo *pData = pArray->pData; + for(int i = 0; i < sz; i++) { + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + WalFileInfo* pInfo = &pData[i]; + pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); + pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); + pInfo->lastVer = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pInfoJson, "createTs"); + pInfo->createTs = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pInfoJson, "closeTs"); + pInfo->closeTs = atoll(cJSON_GetStringValue(pField)); + pField = cJSON_GetObjectItem(pInfoJson, "fileSize"); + pInfo->fileSize = atoll(cJSON_GetStringValue(pField)); + } + taosArraySetSize(pArray, sz); + pWal->fileInfoSet = pArray; + return 0; +} + +static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +} + +static int walFindCurMetaVer(SWal* pWal) { + const char * pattern = "^meta-ver[0-9]+$"; + regex_t walMetaRegexPattern; + regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED); + + DIR *dir = opendir(pWal->path); + if(dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent* ent; + + //find existing meta-ver[x].json + int metaVer = -1; + while((ent = readdir(dir)) != NULL) { + char *name = basename(ent->d_name); + int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); + if(code == 0) { + sscanf(name, "meta-ver%d", &metaVer); + break; + } + } + return metaVer; +} + +int walWriteMeta(SWal* pWal) { + int metaVer = walFindCurMetaVer(pWal); + char fnameStr[WAL_FILE_LEN]; + walBuildMetaName(pWal, metaVer+1, fnameStr); + int metaTfd = tfOpenCreateWrite(fnameStr); + if(metaTfd < 0) { + return -1; + } + char* serialized = walMetaSerialize(pWal); + int len = strlen(serialized); + if(len != tfWrite(metaTfd, serialized, len)) { + //TODO:clean file + return -1; + } + + tfClose(metaTfd); + //delete old file + if(metaVer > -1) { + walBuildMetaName(pWal, metaVer, fnameStr); + remove(fnameStr); + } + return 0; +} + +int walReadMeta(SWal* pWal) { + ASSERT(pWal->fileInfoSet->size == 0); + //find existing meta file + int metaVer = walFindCurMetaVer(pWal); + if(metaVer == -1) { + return 0; + } + char fnameStr[WAL_FILE_LEN]; + walBuildMetaName(pWal, metaVer, fnameStr); + //read metafile + struct stat statbuf; + stat(fnameStr, &statbuf); + int size = statbuf.st_size; + char* buf = malloc(size + 5); + if(buf == NULL) { + return -1; + } + int tfd = tfOpenRead(fnameStr); + if(tfRead(tfd, buf, size) != size) { + free(buf); + return -1; + } + //load into fileInfoSet + int code = walMetaDeserialize(pWal, buf); + if(code != 0) { + free(buf); + return -1; + } + free(buf); + return 0; +} diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index bc2e687069..6bf9008917 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -48,9 +48,15 @@ int32_t walInit() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); if(old == 1) return 0; + int code = tfInit(); + if(code != 0) { + wError("failed to init tfile since %s", tstrerror(code)); + atomic_store_8(&tsWal.inited, 0); + return code; + } tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); - int code = walCreateThread(); + code = walCreateThread(); if (code != 0) { wError("failed to init wal module since %s", tstrerror(code)); atomic_store_8(&tsWal.inited, 0); @@ -64,43 +70,31 @@ int32_t walInit() { void walCleanUp() { walStopThread(); taosCloseRef(tsWal.refSetId); + atomic_store_8(&tsWal.inited, 0); wInfo("wal module is cleaned up"); } -static int walLoadFileset(SWal *pWal) { - DIR *dir = opendir(pWal->path); - if (dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); - return -1; - } - - struct dirent* ent; - while ((ent = readdir(dir)) != NULL) { - char *name = ent->d_name; - name[WAL_NOSUFFIX_LEN] = 0; - //validate file name by regex matching - if(1 /* TODO:regex match */) { - int64_t fnameInt64 = atoll(name); - taosArrayPush(pWal->fileSet, &fnameInt64); - } - } - taosArraySort(pWal->fileSet, compareInt64Val); - return 0; -} - SWal *walOpen(const char *path, SWalCfg *pCfg) { SWal *pWal = malloc(sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } + pWal->writeLogTfd = -1; + pWal->writeIdxTfd = -1; + //set config pWal->vgId = pCfg->vgId; - pWal->curLogTfd = -1; - pWal->curIdxTfd = -1; - pWal->level = pCfg->walLevel; pWal->fsyncPeriod = pCfg->fsyncPeriod; + pWal->rollPeriod = pCfg->rollPeriod; + pWal->segSize = pCfg->segSize; + pWal->level = pCfg->walLevel; + //init status + pWal->lastVersion = -1; + pWal->lastRollSeq = -1; + + //init write buffer memset(&pWal->head, 0, sizeof(SWalHead)); pWal->head.sver = 0; @@ -120,7 +114,6 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walFreeObj(pWal); return NULL; } - walLoadFileset(pWal); wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); @@ -151,10 +144,10 @@ void walClose(SWal *pWal) { if (pWal == NULL) return; pthread_mutex_lock(&pWal->mutex); - tfClose(pWal->curLogTfd); - tfClose(pWal->curIdxTfd); - taosArrayDestroy(pWal->fileSet); - pWal->fileSet = NULL; + tfClose(pWal->writeLogTfd); + tfClose(pWal->writeIdxTfd); + /*taosArrayDestroy(pWal->fileInfoSet);*/ + /*pWal->fileInfoSet = NULL;*/ pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -164,8 +157,8 @@ static int32_t walInitObj(SWal *pWal) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - pWal->fileSet = taosArrayInit(0, sizeof(int64_t)); - if(pWal->fileSet == NULL) { + pWal->fileInfoSet = taosArrayInit(0, sizeof(WalFileInfo)); + if(pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -178,10 +171,10 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); - tfClose(pWal->curLogTfd); - tfClose(pWal->curIdxTfd); - taosArrayDestroy(pWal->fileSet); - pWal->fileSet = NULL; + tfClose(pWal->writeLogTfd); + tfClose(pWal->writeIdxTfd); + taosArrayDestroy(pWal->fileInfoSet); + pWal->fileInfoSet = NULL; pthread_mutex_destroy(&pWal->mutex); tfree(pWal); } @@ -208,9 +201,9 @@ static void walFsyncAll() { while (pWal) { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); - int32_t code = tfFsync(pWal->curLogTfd); + int32_t code = tfFsync(pWal->writeLogTfd); if (code != 0) { - wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(code)); + wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(code)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b475183b7b..e9f5bcbc5d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,16 +13,56 @@ * along with this program. If not, see . */ -#include "wal.h" +#include "walInt.h" +#include "tfile.h" #include "tchecksum.h" -static int walValidateChecksum(SWalHead *pHead, void* body, int64_t bodyLen) { - return taosCheckChecksum((uint8_t*)pHead, sizeof(SWalHead) - sizeof(uint32_t)*2, pHead->cksumHead) && - taosCheckChecksum(body, bodyLen, pHead->cksumBody); +static inline int walValidHeadCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)pHead, sizeof(SWalHead) - sizeof(uint32_t)*2, pHead->cksumHead); } +static inline int walValidBodyCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)pHead->cont, pHead->len, pHead->cksumBody); +} + +static int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { + return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); +} int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { + int code; + code = walSeekVer(pWal, ver); + if(code != 0) { + return code; + } + if(*ppHead == NULL) { + void* ptr = realloc(*ppHead, sizeof(SWalHead)); + if(ptr == NULL) { + return -1; + } + *ppHead = ptr; + } + if(tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + return -1; + } + //TODO: endian compatibility processing after read + if(walValidHeadCksum(*ppHead) != 0) { + return -1; + } + void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->len); + if(ptr == NULL) { + free(*ppHead); + *ppHead = NULL; + return -1; + } + if(tfRead(pWal->writeLogTfd, (*ppHead)->cont, (*ppHead)->len) != (*ppHead)->len) { + return -1; + } + //TODO: endian compatibility processing after read + if(walValidBodyCksum(*ppHead) != 0) { + return -1; + } + return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 69c83a9912..0c4989300f 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -21,29 +21,63 @@ #include "tfile.h" #include "walInt.h" +static void walFtruncate(SWal *pWal, int64_t ver); + int32_t walCommit(SWal *pWal, int64_t ver) { + ASSERT(pWal->snapshotVersion <= pWal->commitVersion); + ASSERT(pWal->commitVersion <= pWal->lastVersion); + ASSERT(ver >= pWal->commitVersion); + ASSERT(ver <= pWal->lastVersion); + pWal->commitVersion = ver; return 0; } int32_t walRollback(SWal *pWal, int64_t ver) { //TODO: ftruncate + ASSERT(ver > pWal->commitVersion); + ASSERT(ver <= pWal->lastVersion); + //seek position + walSeekVer(pWal, ver); + walFtruncate(pWal, ver); return 0; } int32_t walTakeSnapshot(SWal *pWal, int64_t ver) { pWal->snapshotVersion = ver; + int ts = taosGetTimestampSec(); + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; + WalFileInfo tmp; + tmp.firstVer = ver; //mark files safe to delete - int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE); - if(pRet != pWal->fileSet->pData) { - //delete files until less than retention size - - //find first file that exceeds retention time - + WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + //iterate files, until the searched result + for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + if(pWal->totSize > pWal->retentionSize || + iter->closeTs + pWal->retentionPeriod > ts) { + //delete according to file size or close time + deleteCnt++; + newTotSize -= iter->fileSize; + } + } + char fnameStr[WAL_FILE_LEN]; + //remove file + for(int i = 0; i < deleteCnt; i++) { + WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); } - //delete files living longer than retention limit - //remove file from fileset + //save snapshot ver, commit ver + + + //make new array, remove files + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); + pWal->totSize = newTotSize; + return 0; } @@ -138,105 +172,122 @@ void walRemoveAllOldFiles(void *handle) { } #endif -static int walRoll(SWal *pWal) { +int walRoll(SWal *pWal) { int code = 0; - code = tfClose(pWal->curIdxTfd); - if(code != 0) { - return code; + if(pWal->writeIdxTfd != -1) { + code = tfClose(pWal->writeIdxTfd); + if(code != 0) { + return -1; + } } - code = tfClose(pWal->curLogTfd); - if(code != 0) { - return code; + if(pWal->writeLogTfd != -1) { + code = tfClose(pWal->writeLogTfd); + if(code != 0) { + return -1; + } } int64_t idxTfd, logTfd; //create new file int64_t newFileFirstVersion = pWal->lastVersion + 1; char fnameStr[WAL_FILE_LEN]; - sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, newFileFirstVersion); + walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWrite(fnameStr); - sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, newFileFirstVersion); + if(idxTfd < 0) { + ASSERT(0); + return -1; + } + walBuildLogName(pWal, newFileFirstVersion, fnameStr); logTfd = tfOpenCreateWrite(fnameStr); + if(logTfd < 0) { + ASSERT(0); + return -1; + } + code = walRollFileInfo(pWal); + if(code != 0) { + ASSERT(0); + return -1; + } - taosArrayPush(pWal->fileSet, &newFileFirstVersion); - //switch file - pWal->curIdxTfd = idxTfd; - pWal->curLogTfd = logTfd; + pWal->writeIdxTfd = idxTfd; + pWal->writeLogTfd = logTfd; //change status - pWal->curFileLastVersion = -1; - pWal->curFileFirstVersion = newFileFirstVersion; - pWal->curVersion = newFileFirstVersion; - pWal->curLogOffset = 0; pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; - pWal->lastFileName = newFileFirstVersion; - pWal->lastFileWriteSize = 0; pWal->lastRollSeq = walGetSeq(); return 0; } int walChangeFileToLast(SWal *pWal) { int64_t idxTfd, logTfd; - int64_t* pRet = taosArrayGetLast(pWal->fileSet); + WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); - int64_t fname = *pRet; + int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; - sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenReadWrite(fnameStr); - sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + if(idxTfd < 0) { + return -1; + } + walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + return -1; + } //switch file - pWal->curIdxTfd = idxTfd; - pWal->curLogTfd = logTfd; + pWal->writeIdxTfd = idxTfd; + pWal->writeLogTfd = logTfd; //change status - pWal->curFileLastVersion = -1; - pWal->curFileFirstVersion = fname; - pWal->curVersion = fname; - pWal->curLogOffset = 0; + pWal->curVersion = fileFirstVer; pWal->curStatus = WAL_CUR_FILE_WRITABLE; return 0; } -int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { +static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int code = 0; //get index file - if(!tfValid(pWal->curIdxTfd)) { + if(!tfValid(pWal->writeIdxTfd)) { code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); + + wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + return code; } int64_t writeBuf[2] = { ver, offset }; - int size = tfWrite(pWal->curIdxTfd, writeBuf, sizeof(writeBuf)); + int size = tfWrite(pWal->writeIdxTfd, writeBuf, sizeof(writeBuf)); if(size != sizeof(writeBuf)) { - //TODO: + return -1; } return 0; } -int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t bodyLen) { +int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen) { if (pWal == NULL) return -1; + int code = 0; // no wal if (pWal->level == TAOS_WAL_NOLOG) return 0; if (index == pWal->lastVersion + 1) { - int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(passed > pWal->rollPeriod) { - walRoll(pWal); - } else if(pWal->lastFileWriteSize > pWal->segSize) { - walRoll(pWal); + if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + code = walRoll(pWal); + ASSERT(code == 0); } else { - walChangeFileToLast(pWal); + int64_t passed = walGetSeq() - pWal->lastRollSeq; + if(pWal->rollPeriod != -1 && passed > pWal->rollPeriod) { + walRoll(pWal); + } else if(pWal->segSize != -1 && walGetLastFileSize(pWal) > pWal->segSize) { + walRoll(pWal); + } } } else { //reject skip log or rewrite log //must truncate explicitly first return -1; } - if (!tfValid(pWal->curLogTfd)) return 0; + /*if (!tfValid(pWal->curLogTfd)) return 0;*/ pWal->head.version = index; - int32_t code = 0; pWal->head.signature = WAL_SIGNATURE; pWal->head.len = bodyLen; @@ -247,22 +298,27 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t pthread_mutex_lock(&pWal->mutex); - if (tfWrite(pWal->curLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->curLogTfd, &body, bodyLen) != bodyLen) { + if (tfWrite(pWal->writeLogTfd, &body, bodyLen) != bodyLen) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + } + code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); + if(code != 0) { + //TODO } - walWriteIndex(pWal, index, pWal->curLogOffset); - pWal->curLogOffset += sizeof(SWalHead) + bodyLen; //set status pWal->lastVersion = index; + pWal->totSize += sizeof(SWalHead) + bodyLen; + walGetCurFileInfo(pWal)->lastVer = index; + walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; pthread_mutex_unlock(&pWal->mutex); @@ -270,12 +326,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t } void walFsync(SWal *pWal, bool forceFsync) { - if (pWal == NULL || !tfValid(pWal->curLogTfd)) return; + if (pWal == NULL || !tfValid(pWal->writeLogTfd)) return; if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, pWal->curFileFirstVersion); - if (tfFsync(pWal->curLogTfd) < 0) { - wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); + wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal)); + if (tfFsync(pWal->writeLogTfd) < 0) { + wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } } } @@ -348,8 +404,36 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { } #endif -static void walFtruncate(SWal *pWal, int64_t tfd, int64_t offset) { - tfFtruncate(tfd, offset); +static int walValidateOffset(SWal* pWal, int64_t ver) { + int code = 0; + SWalHead *pHead = NULL; + code = (int)walRead(pWal, &pHead, ver); + if(pHead->version != ver) { + return -1; + } + return 0; +} + +static int64_t walGetOffset(SWal* pWal, int64_t ver) { + int code = walSeekVer(pWal, ver); + if(code != 0) { + return -1; + } + + code = walValidateOffset(pWal, ver); + if(code != 0) { + return -1; + } + + return 0; +} + +static void walFtruncate(SWal *pWal, int64_t ver) { + int64_t tfd = pWal->writeLogTfd; + tfFtruncate(tfd, ver); + tfFsync(tfd); + tfd = pWal->writeIdxTfd; + tfFtruncate(tfd, ver * WAL_IDX_ENTRY_SIZE); tfFsync(tfd); } diff --git a/source/libs/wal/test/CMakeLists.txt b/source/libs/wal/test/CMakeLists.txt new file mode 100644 index 0000000000..1c0a3a162a --- /dev/null +++ b/source/libs/wal/test/CMakeLists.txt @@ -0,0 +1,20 @@ +add_executable(walTest "") +target_sources(walTest + PRIVATE + "walMetaTest.cpp" +) +target_include_directories(walTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/wal" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(walTest + wal + gtest_main +) +enable_testing() +add_test( + NAME wal_test + COMMAND walTest +) diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp new file mode 100644 index 0000000000..96258662a1 --- /dev/null +++ b/source/libs/wal/test/walMetaTest.cpp @@ -0,0 +1,149 @@ +#include +#include +#include +#include + +#include "walInt.h" + +class WalCleanEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } + + static void TearDownTestCase() { + walCleanUp(); + } + + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->walLevel = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; +}; + +class WalKeepEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } + + static void TearDownTestCase() { + walCleanUp(); + } + + void SetUp() override { + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->walLevel = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; +}; + +TEST_F(WalCleanEnv, createNew) { + walRollFileInfo(pWal); + ASSERT(pWal->fileInfoSet != NULL); + ASSERT_EQ(pWal->fileInfoSet->size, 1); + WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + ASSERT_EQ(pInfo->firstVer, 0); + ASSERT_EQ(pInfo->lastVer, -1); + ASSERT_EQ(pInfo->closeTs, -1); + ASSERT_EQ(pInfo->fileSize, 0); +} + +TEST_F(WalCleanEnv, serialize) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + char*ss = walMetaSerialize(pWal); + printf("%s\n", ss); + code = walWriteMeta(pWal); + ASSERT(code == 0); +} + +TEST_F(WalCleanEnv, removeOldMeta) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + ASSERT(pWal->fileInfoSet != NULL); + code = walWriteMeta(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walWriteMeta(pWal); + ASSERT(code == 0); +} + +TEST_F(WalKeepEnv, readOldMeta) { + int code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walWriteMeta(pWal); + ASSERT(code == 0); + code = walRollFileInfo(pWal); + ASSERT(code == 0); + code = walWriteMeta(pWal); + ASSERT(code == 0); + char*oldss = walMetaSerialize(pWal); + + TearDown(); + SetUp(); + code = walReadMeta(pWal); + ASSERT(code == 0); + char* newss = walMetaSerialize(pWal); + + int len = strlen(oldss); + ASSERT_EQ(len, strlen(newss)); + for(int i = 0; i < len; i++) { + EXPECT_EQ(oldss[i], newss[i]); + } +} + +TEST_F(WalKeepEnv, write) { + const char* ranStr = "tvapq02tcp"; + const int len = strlen(ranStr); + int code; + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, len); + ASSERT_EQ(code, 0); + code = walWrite(pWal, i+2, i, (void*)ranStr, len); + ASSERT_EQ(code, -1); + } + code = walWriteMeta(pWal); + ASSERT_EQ(code, 0); +} diff --git a/source/libs/wal/test/walTests.cpp b/source/libs/wal/test/walTests.cpp deleted file mode 100644 index 505728fbe4..0000000000 --- a/source/libs/wal/test/walTests.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include "os.h" -#include "tutil.h" -#include "tglobal.h" -#include "tlog.h" -#include "twal.h" -#include "tfile.h" - -int64_t ver = 0; -void *pWal = NULL; - -int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { - // do nothing - SWalHead *pHead = data; - - if (pHead->version > ver) - ver = pHead->version; - - walWrite(pWal, pHead); - - return 0; -} - -int main(int argc, char *argv[]) { - char path[128] = "/tmp/wal"; - int level = 2; - int total = 5; - int rows = 10000; - int size = 128; - int keep = 0; - - for (int i=1; iversion = ++ver; - pHead->len = size; - walWrite(pWal, pHead); - } - - printf("renew a wal, i:%d\n", i); - walRenew(pWal); - } - - printf("%d wal files are written\n", total); - - int64_t index = 0; - char name[256]; - - while (1) { - int code = walGetWalFile(pWal, name, &index); - if (code == -1) { - printf("failed to get wal file, index:%" PRId64 "\n", index); - break; - } - - printf("index:%" PRId64 " wal:%s\n", index, name); - if (code == 0) break; - } - - getchar(); - - walClose(pWal); - walCleanUp(); - tfCleanup(); - - return 0; -} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 007ce06829..ff52477a6f 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -237,6 +237,16 @@ void taosArraySet(SArray* pArray, size_t index, void* pData) { memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize); } +void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { + assert(cnt <= pArray->size); + pArray->size = pArray->size - cnt; + if(pArray->size == 0) { + pArray->size = 0; + return; + } + memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size); +} + void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a36ad9b42..5110c8ba22 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -211,32 +211,41 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN, "Invalid func length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, "Cannot delete monitor database") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_SIZE, "Invalid database cache block size option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS, "Invalid database total blocks option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_DAYS, "Invalid database days option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP0, "Invalid database keep0 option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP1, "Invalid database keep1 option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP2, "Invalid database keep2 option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MIN_ROWS, "Invalid database min rows option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MAX_ROWS, "Invalid database max rows option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMMIT_TIME, "Invalid database commit time option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD, "Invalid database fsync periodoptions") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_WAL_LEVEL, "Invalid database wal level option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_PRECISION, "Invalid database precisin option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMP, "Invalid database compression option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_REPLICA, "Invalid database replication option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_QUORUM, "Invalid database quorum option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_UPDATE, "Invalid database update option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_LAST, "Invalid database cache last option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database not available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, "Database unsynced") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, "Invalid database option: days out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, "Invalid database option: keep2 >= keep1 >= keep0 >= days") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Invalid topic option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_PARTITONS, "Invalid topic partitons num, valid range: [1, 1000]") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 5d4789aae6..313f1d97af 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -22,20 +22,26 @@ static int32_t tsFileRsetId = -1; +static int8_t tfInited = 0; + static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } int32_t tfInit() { + int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); + if(old == 1) return 0; tsFileRsetId = taosOpenRef(2000, tfCloseFile); if (tsFileRsetId > 0) { return 0; } else { + atomic_store_8(&tfInited, 0); return -1; } } void tfCleanup() { + atomic_store_8(&tfInited, 0); if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); tsFileRsetId = -1; } diff --git a/source/util/src/tmd5.c b/source/util/src/tmd5.c index 9cc4b3b9d5..807f3c8122 100644 --- a/source/util/src/tmd5.c +++ b/source/util/src/tmd5.c @@ -84,8 +84,8 @@ static uint8_t PADDING[64] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x /* The routine MD5Init initializes the message-digest context mdContext. All fields are set to zero. */ -void MD5Init(MD5_CTX *mdContext) { - memset(mdContext, 0, sizeof(MD5_CTX)); +void tMD5Init(T_MD5_CTX *mdContext) { + memset(mdContext, 0, sizeof(T_MD5_CTX)); /* Load magic initialization constants. */ mdContext->buf[0] = (uint32_t)0x67452301; @@ -98,7 +98,7 @@ void MD5Init(MD5_CTX *mdContext) { account for the presence of each of the characters inBuf[0..inLen-1] in the message whose digest is being computed. */ -void MD5Update(MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { +void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { uint32_t in[16]; int mdi; unsigned int i, ii; @@ -129,7 +129,7 @@ void MD5Update(MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { /* The routine MD5Final terminates the message-digest computation and ends with the desired message digest in mdContext->digest[0...15]. */ -void MD5Final(MD5_CTX *mdContext) { +void tMD5Final(T_MD5_CTX *mdContext) { uint32_t in[16]; int mdi; unsigned int i, ii; @@ -144,7 +144,7 @@ void MD5Final(MD5_CTX *mdContext) { /* pad out to 56 mod 64 */ padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi); - MD5Update(mdContext, PADDING, padLen); + tMD5Update(mdContext, PADDING, padLen); /* append length in bits and transform */ for (i = 0, ii = 0; i < 14; i++, ii += 4) diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index b9d2da6939..3c3c8a1dfc 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -187,7 +187,7 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { } char* strtolower(char *dst, const char *src) { - int esc = 0; + int32_t esc = 0; char quote = 0, *p = dst, c; assert(dst != NULL); @@ -214,7 +214,7 @@ char* strtolower(char *dst, const char *src) { } char* strntolower(char *dst, const char *src, int32_t n) { - int esc = 0; + int32_t esc = 0; char quote = 0, *p = dst, c; assert(dst != NULL); @@ -347,7 +347,7 @@ char *strbetween(char *string, char *begin, char *end) { char *_begin = strstr(string, begin); if (_begin != NULL) { char *_end = strstr(_begin + strlen(begin), end); - int size = (int)(_end - _begin); + int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { result = (char *)calloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); @@ -402,7 +402,7 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { char *taosIpStr(uint32_t ipInt) { static char ipStrArray[3][30]; - static int ipStrIndex = 0; + static int32_t ipStrIndex = 0; char *ipStr = ipStrArray[(ipStrIndex++) % 3]; //sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); @@ -416,4 +416,17 @@ void taosIp2String(uint32_t ip, char *str) { void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); +} + +int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp + 1); + } + + return 0; } \ No newline at end of file