xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-98] merge develop branch

This commit is contained in:
hjxilinx 2020-04-19 23:12:21 +08:00
parent ec902f8b13 aad2d70eba
commit a842698415
42 changed files with 3061 additions and 1005 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.codecov.yml Normal file
View File

@ -0,0 +1,12 @@
coverage:
precision: 2
notify:
email:
default:
to:
- sdsang@taosdata.com
- &author
only_pulls: false
layout: reach, diff, flags, files
flags: null
paths: null

6
.travis.yml
View File

@ -146,8 +146,8 @@ matrix:
fi
cd ${TRAVIS_BUILD_DIR}
lcov -d . --capture -o coverage.info
lcov -l coverage.info || exit $?
lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info
lcov -l --rc lcov_branch_coverage=1 coverage.info || exit $?
gem install coveralls-lcov
@ -166,7 +166,7 @@ matrix:
exit $?
fi
bash <(curl -s https://codecov.io/bash) -f coverage.info
bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info
if [ "$?" -eq "0" ]; then
echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} "
else

4
src/common/inc/dataformat.h
View File

@ -111,7 +111,6 @@ typedef struct SDataCol {
int len;
int offset;
void * pData; // Original data
void * pCData; // Compressed data
} SDataCol;
typedef struct {
@ -133,9 +132,12 @@ typedef struct {
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
void tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols);
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);
int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows);
#ifdef __cplusplus
}

77
src/common/src/dataformat.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "dataformat.h"
#include "tutil.h"
static int tdFLenFromSchema(STSchema *pSchema);
@ -338,6 +339,28 @@ void tdFreeDataCols(SDataCols *pCols) {
}
}
SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints);
if (pRet == NULL) return NULL;
pRet->numOfCols = pDataCols->numOfCols;
pRet->sversion = pDataCols->sversion;
if (keepData) pRet->numOfPoints = pDataCols->numOfPoints;
for (int i = 0; i < pDataCols->numOfCols; i++) {
pRet->cols[i].type = pDataCols->cols[i].type;
pRet->cols[i].colId = pDataCols->cols[i].colId;
pRet->cols[i].bytes = pDataCols->cols[i].bytes;
pRet->cols[i].len = pDataCols->cols[i].len;
pRet->cols[i].offset = pDataCols->cols[i].offset;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].bytes * pDataCols->numOfPoints);
}
return pRet;
}
void tdResetDataCols(SDataCols *pCols) {
pCols->numOfPoints = 0;
for (int i = 0; i < pCols->maxCols; i++) {
@ -382,6 +405,58 @@ static int tdFLenFromSchema(STSchema *pSchema) {
return ret;
}
int tdMergeDataCols(SDataCols *target, SDataCols *source) {
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints);
SDataCols *pTarget = tdDupDataCols(target, true);
if (pTarget == NULL) goto _err;
// tdResetDataCols(target);
int iter1 = 0;
int iter2 = 0;
tdMergeTwoDataCols(target,pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
tdFreeDataCols(pTarget);
return 0;
_err:
tdFreeDataCols(pTarget);
return -1;
}
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) {
tdResetDataCols(target);
while (target->numOfPoints < tRows) {
if (*iter1 >= src1->numOfPoints && *iter2 >= src2->numOfPoints) break;
TSKEY key1 = (*iter1 >= src1->numOfPoints) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
TSKEY key2 = (*iter2 >= src2->numOfPoints) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
(void *)((char *)(src1->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * (*iter1)),
TYPE_BYTES[target->cols[i].type]);
target->cols[i].len += TYPE_BYTES[target->cols[i].type];
}
target->numOfPoints++;
(*iter1)++;
} else if (key1 > key2) {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
(void *)((char *)(src2->cols[i].pData) + TYPE_BYTES[src2->cols[i].type] * (*iter2)),
TYPE_BYTES[target->cols[i].type]);
target->cols[i].len += TYPE_BYTES[target->cols[i].type];
}
target->numOfPoints++;
(*iter2)++;
} else {
ASSERT(false);
}
}
}

2
src/dnode/CMakeLists.txt
View File

@ -27,7 +27,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ENDIF ()
IF (TD_SYNC)
TARGET_LINK_LIBRARIES(taosd replica sync)
TARGET_LINK_LIBRARIES(taosd balance sync)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")

7
src/dnode/src/dnodeMClient.c
View File

@ -23,12 +23,13 @@
#include "tsync.h"
#include "ttime.h"
#include "ttimer.h"
#include "treplica.h"
#include "tbalance.h"
#include "vnode.h"
#include "mnode.h"
#include "dnode.h"
#include "dnodeMClient.h"
#include "dnodeModule.h"
#include "dnodeMgmt.h"
#include "vnode.h"
#define MPEER_CONTENT_LEN 2000
@ -181,7 +182,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
tsMnodeInfos.nodeInfos[i].nodeName);
}
dnodeSaveMnodeIpList();
replicaNotify();
sdbUpdateSync();
}
taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);

1
src/inc/mnode.h
View File

@ -24,6 +24,7 @@ int32_t mgmtInitSystem();
int32_t mgmtStartSystem();
void mgmtCleanUpSystem();
void mgmtStopSystem();
void sdbUpdateSync();
#ifdef __cplusplus
}

17
src/inc/treplica.h → src/inc/tbalance.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_REPLICA_H
#define TDENGINE_REPLICA_H
#ifndef TDENGINE_BALANCE_H
#define TDENGINE_BALANCE_H
#ifdef __cplusplus
extern "C" {
@ -23,13 +23,12 @@ extern "C" {
struct SVgObj;
struct SDnodeObj;
int32_t replicaInit();
void replicaCleanUp();
void replicaNotify();
void replicaReset();
int32_t replicaAllocVnodes(struct SVgObj *pVgroup);
int32_t replicaForwardReqToPeer(void *pHead);
int32_t replicaDropDnode(struct SDnodeObj *pDnode);
int32_t balanceInit();
void balanceCleanUp();
void balanceNotify();
void balanceReset();
int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
int32_t balanceDropDnode(struct SDnodeObj *pDnode);
#ifdef __cplusplus
}

53
src/inc/tsdb.h
View File

@ -46,6 +46,7 @@ typedef struct {
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
typedef struct {
int8_t precision;
int8_t compression;
int32_t tsdbId;
int32_t maxTables; // maximum number of tables this repository can have
int32_t daysPerFile; // day per file sharding policy
@ -60,13 +61,13 @@ STsdbCfg *tsdbCreateDefaultCfg();
void tsdbFreeCfg(STsdbCfg *pCfg);
// --------- TSDB REPOSITORY DEFINITION
typedef void tsdb_repo_t; // use void to hide implementation details from outside
typedef void TsdbRepoT; // use void to hide implementation details from outside
int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
int32_t tsdbDropRepo(tsdb_repo_t *repo);
tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH);
int32_t tsdbCloseRepo(tsdb_repo_t *repo);
int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
int32_t tsdbDropRepo(TsdbRepoT *repo);
TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH);
int32_t tsdbCloseRepo(TsdbRepoT *repo);
int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg);
// --------- TSDB TABLE DEFINITION
typedef struct {
@ -96,11 +97,11 @@ int tsdbTableSetName(STableCfg *config, char *name, bool dup);
int tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
void tsdbClearTableCfg(STableCfg *config);
int32_t tsdbGetTableTagVal(tsdb_repo_t *repo, STableId id, int32_t col, int16_t* type, int16_t* bytes, char** val);
int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId id, int32_t col, int16_t *type, int16_t *bytes, char **val);
int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
// the TSDB repository info
typedef struct STsdbRepoInfo {
@ -110,7 +111,7 @@ typedef struct STsdbRepoInfo {
int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository
// TODO: Other informations to add
} STsdbRepoInfo;
STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo);
// the meter information report structure
typedef struct {
@ -119,7 +120,7 @@ typedef struct {
int64_t tableTotalDataSize; // In bytes
int64_t tableTotalDiskSize; // In bytes
} STableInfo;
STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
// -- FOR INSERT DATA
/**
@ -129,11 +130,11 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
*
* @return the number of points inserted, -1 for failure and the error number is set
*/
int32_t tsdbInsertData(tsdb_repo_t *pRepo, SSubmitMsg *pMsg);
int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg);
// -- FOR QUERY TIME SERIES DATA
typedef void *tsdb_query_handle_t; // Use void to hide implementation details
typedef void *TsdbQueryHandleT; // Use void to hide implementation details
typedef struct STableGroupList { // qualified table object list in group
SArray *pGroupList;
@ -181,7 +182,7 @@ typedef struct SQueryRowCond {
TSKEY ts;
} SQueryRowCond;
typedef void *tsdbpos_t;
typedef void *TsdbPosT;
/**
* Get the data block iterator, starting from position according to the query condition
@ -189,14 +190,14 @@ typedef void *tsdbpos_t;
* @param pTableList table sid list
* @return
*/
tsdb_query_handle_t *tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
/**
* move to next block
* @param pQueryHandle
* @return
*/
bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle);
bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
/**
* Get current data block information
@ -204,7 +205,7 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle);
* @param pQueryHandle
* @return
*/
SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle);
SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle);
/**
*
@ -216,7 +217,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle);
* @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
* @return
*/
int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis);
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis);
/**
* The query condition with primary timestamp is passed to iterator during its constructor function,
@ -226,7 +227,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SData
* @param pQueryHandle
* @return
*/
SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList);
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pIdList);
/**
* todo remove the parameter of position, and order type
@ -238,7 +239,7 @@ SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList
* @param order ascending order or descending order
* @return
*/
int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order);
int32_t tsdbResetQuery(TsdbQueryHandleT *pQueryHandle, STimeWindow *window, TsdbPosT position, int16_t order);
/**
* todo remove this function later
@ -246,7 +247,7 @@ int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, t
* @param pIdList
* @return
*/
SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
SArray *tsdbRetrieveDataRow(TsdbQueryHandleT *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
/**
* Get iterator for super tables, of which tags values satisfy the tag filter info
@ -259,7 +260,7 @@ SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList,
* @param pTagFilterStr tag filter info
* @return
*/
tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
TsdbQueryHandleT *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
/**
* Get the qualified tables for (super) table query.
@ -269,7 +270,7 @@ tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stable
* @param pQueryHandle
* @return table sid list. the invoker is responsible for the release of this the sid list.
*/
SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle);
SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
/**
* Get the qualified table id for a super table according to the tag query expression.
@ -277,16 +278,16 @@ SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle);
* @param pTagCond. tag query condition
*
*/
int32_t tsdbQueryByTagsCond(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupList,
int32_t tsdbQueryByTagsCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, STableGroupInfo *pGroupList,
SColIndex *pColIndex, int32_t numOfCols);
int32_t tsdbGetOneTableGroup(tsdb_repo_t* tsdb, int64_t uid, STableGroupInfo* pGroupInfo);
int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo);
/**
* clean up the query handle
* @param queryHandle
*/
void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle);
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
#ifdef __cplusplus
}

1
src/inc/vnode.h
View File

@ -47,7 +47,6 @@ void* vnodeGetVnode(int32_t vgId); // keep refcount unchanged
void* vnodeGetRqueue(void *);
void* vnodeGetWqueue(int32_t vgId);
void* vnodeGetWal(void *pVnode);
void* vnodeGetTsdb(void *pVnode);
int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
void vnodeBuildStatusMsg(void * param);

5
src/mnode/inc/mgmtMnode.h
View File

@ -39,10 +39,9 @@ int32_t mgmtGetMnodesNum();
void * mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
void mgmtReleaseMnode(struct SMnodeObj *pMnode);
bool mgmtIsMaster();
char * mgmtGetMnodeRoleStr();
void mgmtGetMnodeIpList(SRpcIpSet *ipSet, bool usePublicIp);
void mgmtGetMnodeList(void *mpeers);
void mgmtGetMnodeList(void *mnodes);
#ifdef __cplusplus
}

39
src/mnode/inc/mgmtSdb.h
View File

@ -36,20 +36,20 @@ typedef enum {
SDB_KEY_STRING,
SDB_KEY_INT,
SDB_KEY_AUTO
} ESdbKeyType;
} ESdbKey;
typedef enum {
SDB_OPER_GLOBAL,
SDB_OPER_LOCAL
} ESdbOperType;
} ESdbOper;
typedef struct {
ESdbOperType type;
ESdbOper type;
void * table;
void * pObj;
int32_t rowSize;
void * rowData;
} SSdbOperDesc;
} SSdbOper;
typedef struct {
char *tableName;
@ -57,33 +57,26 @@ typedef struct {
int32_t maxRowSize;
int32_t refCountPos;
ESdbTable tableId;
ESdbKeyType keyType;
int32_t (*insertFp)(SSdbOperDesc *pOper);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
int32_t (*updateFp)(SSdbOperDesc *pOper);
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*decodeFp)(SSdbOperDesc *pDesc);
int32_t (*destroyFp)(SSdbOperDesc *pDesc);
ESdbKey keyType;
int32_t (*insertFp)(SSdbOper *pOper);
int32_t (*deleteFp)(SSdbOper *pOper);
int32_t (*updateFp)(SSdbOper *pOper);
int32_t (*encodeFp)(SSdbOper *pOper);
int32_t (*decodeFp)(SSdbOper *pDesc);
int32_t (*destroyFp)(SSdbOper *pDesc);
int32_t (*restoredFp)();
} SSdbTableDesc;
typedef struct {
int64_t version;
void * wal;
pthread_mutex_t mutex;
} SSdbObject;
int32_t sdbInit();
void sdbCleanUp();
SSdbObject *sdbGetObj();
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
int sdbProcessWrite(void *param, void *data, int type);
bool sdbIsMaster();
void sdbUpdateMnodeRoles();
int32_t sdbInsertRow(SSdbOperDesc *pOper);
int32_t sdbDeleteRow(SSdbOperDesc *pOper);
int32_t sdbUpdateRow(SSdbOperDesc *pOper);
int32_t sdbInsertRow(SSdbOper *pOper);
int32_t sdbDeleteRow(SSdbOper *pOper);
int32_t sdbUpdateRow(SSdbOper *pOper);
void *sdbGetRow(void *handle, void *key);
void *sdbFetchRow(void *handle, void *pNode, void **ppRow);

16
src/mnode/src/mgmtAcct.c
View File

@ -30,28 +30,28 @@ void * tsAcctSdb = NULL;
int32_t tsAcctUpdateSize;
static void mgmtCreateRootAcct();
static int32_t mgmtActionAcctDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtActionAcctDestroy(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
pthread_mutex_destroy(&pAcct->mutex);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtAcctActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtAcctActionInsert(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
pthread_mutex_init(&pAcct->mutex, NULL);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtActionAcctDelete(SSdbOperDesc *pOper) {
static int32_t mgmtActionAcctDelete(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
mgmtDropAllUsers(pAcct);
mgmtDropAllDbs(pAcct);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtActionAcctUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtActionAcctUpdate(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
SAcctObj *pSaved = mgmtGetAcct(pAcct->user);
if (pAcct != pSaved) {
@ -61,14 +61,14 @@ static int32_t mgmtActionAcctUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtActionActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtActionActionEncode(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
memcpy(pOper->rowData, pAcct, tsAcctUpdateSize);
pOper->rowSize = tsAcctUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtActionAcctDecode(SSdbOperDesc *pOper) {
static int32_t mgmtActionAcctDecode(SSdbOper *pOper) {
SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj));
if (pAcct == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@ -110,7 +110,7 @@ int32_t mgmtInitAccts() {
return -1;
}
mTrace("account table is created");
mTrace("table:accounts table is created");
return acctInit();
}
@ -179,7 +179,7 @@ static void mgmtCreateRootAcct() {
pAcct->acctId = sdbGetId(tsAcctSdb);
pAcct->createdTime = taosGetTimestampMs();
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsAcctSdb,
.pObj = pAcct,

13
src/mnode/src/mgmtReplica.c → src/mnode/src/mgmtBalance.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "trpc.h"
#include "treplica.h"
#include "tbalance.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtMnode.h"
@ -25,13 +25,12 @@
#ifndef _SYNC
int32_t replicaInit() { return TSDB_CODE_SUCCESS; }
void replicaCleanUp() {}
void replicaNotify() {}
void replicaReset() {}
int32_t replicaForwardReqToPeer(void *pHead) { return TSDB_CODE_SUCCESS; }
int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
void balanceCleanUp() {}
void balanceNotify() {}
void balanceReset() {}
int32_t replicaAllocVnodes(SVgObj *pVgroup) {
int32_t balanceAllocVnodes(SVgObj *pVgroup) {
void * pNode = NULL;
SDnodeObj *pDnode = NULL;
SDnodeObj *pSelDnode = NULL;

2
src/mnode/src/mgmtDServer.c
View File

@ -21,7 +21,7 @@
#include "tsystem.h"
#include "tutil.h"
#include "tgrant.h"
#include "treplica.h"
#include "tbalance.h"
#include "tglobalcfg.h"
#include "dnode.h"
#include "mgmtDef.h"

22
src/mnode/src/mgmtDb.c
View File

@ -46,12 +46,12 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg);
static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg);
static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionInsert(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
@ -72,7 +72,7 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionDelete(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
@ -84,7 +84,7 @@ static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
SDbObj *pSaved = mgmtGetDb(pDb->name);
if (pDb != pSaved) {
@ -94,14 +94,14 @@ static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionEncode(SSdbOper *pOper) {
SDbObj *pDb = pOper->pObj;
memcpy(pOper->rowData, pDb, tsDbUpdateSize);
pOper->rowSize = tsDbUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtDbActionDecode(SSdbOper *pOper) {
SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj));
if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@ -146,7 +146,7 @@ int32_t mgmtInitDbs() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DB, mgmtGetDbMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mgmtRetrieveDbs);
mTrace("db data is initialized");
mTrace("table:dbs table is created");
return 0;
}
@ -318,7 +318,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
pDb->createdTime = taosGetTimestampMs();
pDb->cfg = *pCreate;
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
@ -671,7 +671,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
if (pDb->status) return TSDB_CODE_SUCCESS;
pDb->status = true;
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
@ -756,7 +756,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
@ -814,7 +814,7 @@ static void mgmtDropDb(SQueuedMsg *pMsg) {
SDbObj *pDb = pMsg->pDb;
mPrint("db:%s, drop db from sdb", pDb->name);
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb

26
src/mnode/src/mgmtDnode.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tgrant.h"
#include "treplica.h"
#include "tbalance.h"
#include "tglobalcfg.h"
#include "ttime.h"
#include "tutil.h"
@ -52,12 +52,12 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtDnodeActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtDnodeActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDnodeActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
@ -72,7 +72,7 @@ static int32_t mgmtDnodeActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
void * pNode = NULL;
void * pLastNode = NULL;
@ -85,7 +85,7 @@ static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
if (pVgroup == NULL) break;
if (pVgroup->vnodeGid[0].dnodeId == pDnode->dnodeId) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
@ -101,7 +101,7 @@ static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDnodeActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtDnodeActionUpdate(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
SDnodeObj *pSaved = mgmtGetDnode(pDnode->dnodeId);
if (pDnode != pSaved) {
@ -111,14 +111,14 @@ static int32_t mgmtDnodeActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDnodeActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtDnodeActionEncode(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
memcpy(pOper->rowData, pDnode, tsDnodeUpdateSize);
pOper->rowSize = tsDnodeUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDnodeActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtDnodeActionDecode(SSdbOper *pOper) {
SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
if (pDnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@ -180,7 +180,7 @@ int32_t mgmtInitDnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes);
mTrace("dnodes table is created");
mTrace("table:dnodes table is created");
return 0;
}
@ -221,7 +221,7 @@ void mgmtReleaseDnode(SDnodeObj *pDnode) {
}
void mgmtUpdateDnode(SDnodeObj *pDnode) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode,
@ -340,7 +340,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
replicaNotify();
balanceNotify();
}
mgmtReleaseDnode(pDnode);
@ -393,7 +393,7 @@ static int32_t mgmtCreateDnode(uint32_t ip) {
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
sprintf(pDnode->dnodeName, "n%d", sdbGetId(tsDnodeSdb) + 1);
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode,
@ -413,7 +413,7 @@ static int32_t mgmtCreateDnode(uint32_t ip) {
}
int32_t mgmtDropDnode(SDnodeObj *pDnode) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode

14
src/mnode/src/mgmtMain.c
View File

@ -17,7 +17,7 @@
#include "os.h"
#include "taosdef.h"
#include "tsched.h"
#include "treplica.h"
#include "tbalance.h"
#include "tgrant.h"
#include "ttimer.h"
#include "dnode.h"
@ -62,7 +62,7 @@ int32_t mgmtStartSystem() {
}
if (grantInit() < 0) {
mError("failed to init grants");
mError("failed to init grant");
return -1;
}
@ -92,7 +92,7 @@ int32_t mgmtStartSystem() {
}
if (mgmtInitMnodes() < 0) {
mError("failed to init mpeers");
mError("failed to init mnodes");
return -1;
}
@ -101,8 +101,8 @@ int32_t mgmtStartSystem() {
return -1;
}
if (replicaInit() < 0) {
mError("failed to init replica")
if (balanceInit() < 0) {
mError("failed to init balance")
}
if (mgmtInitDClient() < 0) {
@ -144,7 +144,7 @@ void mgmtCleanUpSystem() {
mPrint("starting to clean up mgmt");
grantCleanUp();
mgmtCleanupMnodes();
replicaCleanUp();
balanceCleanUp();
mgmtCleanUpShell();
mgmtCleanupDClient();
mgmtCleanupDServer();
@ -161,7 +161,7 @@ void mgmtCleanUpSystem() {
}
void mgmtStopSystem() {
if (mgmtIsMaster()) {
if (sdbIsMaster()) {
mTrace("it is a master mgmt node, it could not be stopped");
return;
}

28
src/mnode/src/mgmtMnode.c
View File

@ -18,7 +18,7 @@
#include "taoserror.h"
#include "trpc.h"
#include "tsync.h"
#include "treplica.h"
#include "tbalance.h"
#include "tutil.h"
#include "ttime.h"
#include "tsocket.h"
@ -30,18 +30,17 @@
#include "mgmtShell.h"
#include "mgmtUser.h"
int32_t tsMnodeIsMaster = true;
static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mgmtMnodeActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtMnodeActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtMnodeActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
@ -53,7 +52,7 @@ static int32_t mgmtMnodeActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtMnodeActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtMnodeActionDelete(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
@ -65,7 +64,7 @@ static int32_t mgmtMnodeActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtMnodeActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtMnodeActionUpdate(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
SMnodeObj *pSaved = mgmtGetMnode(pMnode->mnodeId);
if (pMnode != pSaved) {
@ -76,14 +75,14 @@ static int32_t mgmtMnodeActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtMnodeActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtMnodeActionEncode(SSdbOper *pOper) {
SMnodeObj *pMnode = pOper->pObj;
memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize);
pOper->rowSize = tsMnodeUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtMnodeActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) {
SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
if (pMnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@ -133,7 +132,7 @@ int32_t mgmtInitMnodes() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
mTrace("mnodes table is created");
mTrace("table:mnodes table is created");
return TSDB_CODE_SUCCESS;
}
@ -157,7 +156,7 @@ void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
return sdbFetchRow(tsMnodeSdb, pNode, (void **)pMnode);
}
static char *mgmtGetMnodeRoleStr(int32_t role) {
char *mgmtGetMnodeRoleStr(int32_t role) {
switch (role) {
case TAOS_SYNC_ROLE_OFFLINE:
return "offline";
@ -172,8 +171,6 @@ static char *mgmtGetMnodeRoleStr(int32_t role) {
}
}
bool mgmtIsMaster() { return tsMnodeIsMaster; }
void mgmtGetMnodeIpList(SRpcIpSet *ipSet, bool usePublicIp) {
void *pNode = NULL;
while (1) {
@ -213,10 +210,8 @@ void mgmtGetMnodeList(void *param) {
mnodes->nodeInfos[index].nodeIp = htonl(pMnode->pDnode->privateIp);
mnodes->nodeInfos[index].nodePort = htons(pMnode->pDnode->mnodeDnodePort);
strcpy(mnodes->nodeInfos[index].nodeName, pMnode->pDnode->dnodeName);
mPrint("node:%d role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
mnodes->inUse = index;
mPrint("node:%d inUse:%d", pMnode->mnodeId, mnodes->inUse);
}
index++;
@ -231,7 +226,7 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
pMnode->mnodeId = dnodeId;
pMnode->createdTime = taosGetTimestampMs();
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsMnodeSdb,
.pObj = pMnode,
@ -252,7 +247,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
return TSDB_CODE_DNODE_NOT_EXIST;
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsMnodeSdb,
.pObj = pMnode
@ -268,6 +263,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
}
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
sdbUpdateMnodeRoles();
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
if (pUser == NULL) return 0;

481
src/mnode/src/mgmtSdb.c
View File

@ -18,65 +18,93 @@
#include "taoserror.h"
#include "tlog.h"
#include "trpc.h"
#include "treplica.h"
#include "tutil.h"
#include "tbalance.h"
#include "tqueue.h"
#include "twal.h"
#include "tsync.h"
#include "hashint.h"
#include "hashstr.h"
#include "dnode.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtMnode.h"
#include "mgmtSdb.h"
typedef enum {
SDB_ACTION_INSERT,
SDB_ACTION_DELETE,
SDB_ACTION_UPDATE
} ESdbAction;
typedef enum {
SDB_STATUS_OFFLINE,
SDB_STATUS_SERVING,
SDB_ACTION_CLOSING
} ESdbStatus;
typedef struct _SSdbTable {
char tableName[TSDB_DB_NAME_LEN + 1];
ESdbTable tableId;
ESdbKeyType keyType;
ESdbKey keyType;
int32_t hashSessions;
int32_t maxRowSize;
int32_t refCountPos;
int32_t autoIndex;
int64_t numOfRows;
void * iHandle;
int32_t (*insertFp)(SSdbOperDesc *pDesc);
int32_t (*deleteFp)(SSdbOperDesc *pOper);
int32_t (*updateFp)(SSdbOperDesc *pOper);
int32_t (*decodeFp)(SSdbOperDesc *pOper);
int32_t (*encodeFp)(SSdbOperDesc *pOper);
int32_t (*destroyFp)(SSdbOperDesc *pOper);
int32_t (*insertFp)(SSdbOper *pDesc);
int32_t (*deleteFp)(SSdbOper *pOper);
int32_t (*updateFp)(SSdbOper *pOper);
int32_t (*decodeFp)(SSdbOper *pOper);
int32_t (*encodeFp)(SSdbOper *pOper);
int32_t (*destroyFp)(SSdbOper *pOper);
int32_t (*restoredFp)();
pthread_mutex_t mutex;
} SSdbTable;
typedef struct {
ESyncRole role;
ESdbStatus status;
int64_t version;
void * sync;
void * wal;
SSyncCfg cfg;
sem_t sem;
int32_t code;
int32_t numOfTables;
SSdbTable *tableList[SDB_TABLE_MAX];
pthread_mutex_t mutex;
} SSdbObject;
typedef struct {
int32_t rowSize;
void * row;
} SRowMeta;
typedef enum {
SDB_ACTION_INSERT,
SDB_ACTION_DELETE,
SDB_ACTION_UPDATE
} ESdbActionType;
static SSdbTable *tsSdbTableList[SDB_TABLE_MAX] = {0};
static int32_t tsSdbNumOfTables = 0;
static SSdbObject * tsSdbObj;
} SSdbRow;
static SSdbObject tsSdbObj = {0};
static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash};
static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash};
static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash};
static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData};
static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash};
static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData};
static int sdbWrite(void *param, void *data, int type);
int32_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->autoIndex; }
int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; }
int32_t sdbGetId(void *handle) {
return ((SSdbTable *)handle)->autoIndex;
}
int64_t sdbGetNumOfRows(void *handle) {
return ((SSdbTable *)handle)->numOfRows;
}
uint64_t sdbGetVersion() {
if (tsSdbObj)
return tsSdbObj->version;
else
return 0;
return tsSdbObj.version;
}
bool sdbIsMaster() {
return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER;
}
static char *sdbGetActionStr(int32_t action) {
@ -106,26 +134,26 @@ static char *sdbGetkeyStr(SSdbTable *pTable, void *row) {
}
static void *sdbGetTableFromId(int32_t tableId) {
return tsSdbTableList[tableId];
return tsSdbObj.tableList[tableId];
}
int32_t sdbInit() {
tsSdbObj = calloc(1, sizeof(SSdbObject));
pthread_mutex_init(&tsSdbObj->mutex, NULL);
static int32_t sdbInitWal() {
SWalCfg walCfg = {.commitLog = 2, .wals = 2, .keep = 1};
tsSdbObj->wal = walOpen(tsMnodeDir, &walCfg);
if (tsSdbObj->wal == NULL) {
sdbError("failed to open sdb in %s", tsMnodeDir);
tsSdbObj.wal = walOpen(tsMnodeDir, &walCfg);
if (tsSdbObj.wal == NULL) {
sdbError("failed to open sdb wal in %s", tsMnodeDir);
return -1;
}
sdbTrace("open sdb file for read");
walRestore(tsSdbObj->wal, tsSdbObj, sdbProcessWrite);
sdbTrace("open sdb wal for restore");
walRestore(tsSdbObj.wal, &tsSdbObj, sdbWrite);
return 0;
}
static void sdbRestoreTables() {
int32_t totalRows = 0;
int32_t numOfTables = 0;
for (int32_t tableId = SDB_TABLE_DNODE; tableId < SDB_TABLE_MAX; ++tableId) {
for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) {
SSdbTable *pTable = sdbGetTableFromId(tableId);
if (pTable == NULL) continue;
if (pTable->restoredFp) {
@ -134,23 +162,170 @@ int32_t sdbInit() {
totalRows += pTable->numOfRows;
numOfTables++;
sdbTrace("table:%s, is initialized, numOfRows:%d", pTable->tableName, pTable->numOfRows);
sdbTrace("table:%s, is restored, numOfRows:%d", pTable->tableName, pTable->numOfRows);
}
sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbObj->version, totalRows, numOfTables);
sdbTrace("sdb is restored, version:%d totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
}
replicaNotify();
void sdbUpdateMnodeRoles() {
if (tsSdbObj.sync == NULL) return;
SNodesRole roles = {0};
syncGetNodesRole(tsSdbObj.sync, &roles);
mPrint("update mnodes:%d sync roles", tsSdbObj.cfg.replica);
for (int32_t i = 0; i < tsSdbObj.cfg.replica; ++i) {
SMnodeObj *pMnode = mgmtGetMnode(roles.nodeId[i]);
if (pMnode != NULL) {
pMnode->role = roles.role[i];
mPrint("mnode:%d, role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
mgmtReleaseMnode(pMnode);
}
}
}
static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) {
sdbUpdateMnodeRoles();
return 0;
}
static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) {
strcpy(name, "wal0");
return 0;
}
static void sdbNotifyRole(void *ahandle, int8_t role) {
mPrint("mnode role changed from %s to %s", mgmtGetMnodeRoleStr(tsSdbObj.role), mgmtGetMnodeRoleStr(role));
if (role == TAOS_SYNC_ROLE_MASTER && tsSdbObj.role != TAOS_SYNC_ROLE_MASTER) {
balanceReset();
}
tsSdbObj.role = role;
sdbUpdateMnodeRoles();
}
static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
tsSdbObj.code = code;
sem_post(&tsSdbObj.sem);
mPrint("sdb forward request confirmed, result:%s", tstrerror(code));
}
static int32_t sdbForwardToPeer(void *pHead) {
if (tsSdbObj.sync == NULL) return TSDB_CODE_SUCCESS;
int32_t code = syncForwardToPeer(tsSdbObj.sync, pHead, NULL);
if (code > 0) {
sem_wait(&tsSdbObj.sem);
return tsSdbObj.code;
}
return code;
}
void sdbUpdateSync() {
SSyncCfg syncCfg = {0};
int32_t index = 0;
SDMNodeInfos *mnodes = dnodeGetMnodeList();
for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
SDMNodeInfo *node = &mnodes->nodeInfos[i];
syncCfg.nodeInfo[i].nodeId = node->nodeId;
syncCfg.nodeInfo[i].nodeIp = node->nodeIp;
strcpy(syncCfg.nodeInfo[i].name, node->nodeName);
index++;
}
if (index == 0) {
void *pNode = NULL;
while (1) {
SMnodeObj *pMnode = NULL;
pNode = mgmtGetNextMnode(pNode, &pMnode);
if (pMnode == NULL) break;
syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
syncCfg.nodeInfo[index].nodeIp = pMnode->pDnode->privateIp;
strcpy(syncCfg.nodeInfo[index].name, pMnode->pDnode->dnodeName);
index++;
mgmtReleaseMnode(pMnode);
}
}
syncCfg.replica = index;
syncCfg.arbitratorIp = syncCfg.nodeInfo[0].nodeIp;
if (syncCfg.replica == 1) {
syncCfg.quorum = 1;
} else {
syncCfg.quorum = 2;
}
bool hasThisDnode = false;
for (int32_t i = 0; i < syncCfg.replica; ++i) {
if (syncCfg.nodeInfo[i].nodeId == dnodeGetDnodeId()) {
hasThisDnode = true;
break;
}
}
if (!hasThisDnode) return;
if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
mPrint("work as mnode, replica:%d arbitratorIp:%s", syncCfg.replica, taosIpStr(syncCfg.arbitratorIp));
for (int32_t i = 0; i < syncCfg.replica; ++i) {
mPrint("mnode:%d, ip:%s name:%s", syncCfg.nodeInfo[i].nodeId, taosIpStr(syncCfg.nodeInfo[i].nodeIp),
syncCfg.nodeInfo[i].name);
}
SSyncInfo syncInfo;
syncInfo.vgId = 1;
syncInfo.version = sdbGetVersion();
syncInfo.syncCfg = syncCfg;
sprintf(syncInfo.path, "%s/", tsMnodeDir);
syncInfo.ahandle = NULL;
syncInfo.getWalInfo = sdbGetWalInfo;
syncInfo.getFileInfo = sdbGetFileInfo;
syncInfo.writeToCache = sdbWrite;
syncInfo.confirmForward = sdbConfirmForward;
syncInfo.notifyRole = sdbNotifyRole;
tsSdbObj.cfg = syncCfg;
if (tsSdbObj.sync) {
syncReconfig(tsSdbObj.sync, &syncCfg);
} else {
tsSdbObj.sync = syncStart(&syncInfo);
}
}
int32_t sdbInit() {
pthread_mutex_init(&tsSdbObj.mutex, NULL);
sem_init(&tsSdbObj.sem, 0, 0);
if (sdbInitWal() != 0) {
return -1;
}
sdbRestoreTables();
if (mgmtGetMnodesNum() == 1) {
tsSdbObj.role = TAOS_SYNC_ROLE_MASTER;
}
sdbUpdateSync();
tsSdbObj.status = SDB_STATUS_SERVING;
return TSDB_CODE_SUCCESS;
}
void sdbCleanUp() {
if (tsSdbObj) {
pthread_mutex_destroy(&tsSdbObj->mutex);
walClose(tsSdbObj->wal);
free(tsSdbObj);
tsSdbObj = NULL;
}
if (tsSdbObj.status != SDB_STATUS_SERVING) return;
syncStop(tsSdbObj.sync);
free(tsSdbObj.sync);
walClose(tsSdbObj.wal);
sem_destroy(&tsSdbObj.sem);
pthread_mutex_destroy(&tsSdbObj.mutex);
memset(&tsSdbObj, 0, sizeof(tsSdbObj));
}
void sdbIncRef(void *handle, void *pRow) {
@ -178,15 +353,15 @@ void sdbDecRef(void *handle, void *pRow) {
if (refCount <= 0 && *updateEnd) {
sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName,
sdbGetkeyStr(pTable, pRow), *pRefCount);
SSdbOperDesc oper = {.pObj = pRow};
SSdbOper oper = {.pObj = pRow};
(*pTable->destroyFp)(&oper);
}
}
}
static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
static SSdbRow *sdbGetRowMeta(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
SSdbRow * pMeta;
if (handle == NULL) return NULL;
@ -197,7 +372,7 @@ static SRowMeta *sdbGetRowMeta(void *handle, void *key) {
void *sdbGetRow(void *handle, void *key) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
SSdbRow * pMeta;
if (handle == NULL) return NULL;
@ -213,8 +388,8 @@ void *sdbGetRow(void *handle, void *key) {
return pMeta->row;
}
static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
SRowMeta rowMeta;
static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
SSdbRow rowMeta;
rowMeta.rowSize = pOper->rowSize;
rowMeta.row = pOper->pObj;
@ -229,20 +404,20 @@ static int32_t sdbInsertLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, insert record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
(*pTable->insertFp)(pOper);
return TSDB_CODE_SUCCESS;
}
static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
pthread_mutex_lock(&pTable->mutex);
(*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj);
pTable->numOfRows--;
pthread_mutex_unlock(&pTable->mutex);
sdbTrace("table:%s, delete record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
(*pTable->deleteFp)(pOper);
@ -253,112 +428,15 @@ static int32_t sdbDeleteLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t sdbUpdateLocal(SSdbTable *pTable, SSdbOperDesc *pOper) {
sdbTrace("table:%s, update record:%s, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
sdbTrace("table:%s, update record:%s in hash, numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj),
pTable->numOfRows);
(*pTable->updateFp)(pOper);
return TSDB_CODE_SUCCESS;
}
static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_t action) {
int32_t code = 0;
pthread_mutex_lock(&tsSdbObj->mutex);
tsSdbObj->version++;
pHead->version = tsSdbObj->version;
code = replicaForwardReqToPeer(pHead);
if (code != TSDB_CODE_SUCCESS) {
pthread_mutex_unlock(&tsSdbObj->mutex);
sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code));
return code;
}
code = walWrite(tsSdbObj->wal, pHead);
pthread_mutex_unlock(&tsSdbObj->mutex);
if (code < 0) {
sdbError("table:%s, failed to %s record:%s to file, version:%" PRId64 ", reason:%s", pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, tstrerror(code));
} else {
sdbTrace("table:%s, success to %s record:%s to file, version:%" PRId64, pTable->tableName, sdbGetActionStr(action),
sdbGetkeyStr(pTable, pHead->cont), pHead->version);
}
walFsync(tsSdbObj->wal);
taosFreeQitem(pHead);
return code;
}
static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_t action) {
pthread_mutex_lock(&tsSdbObj->mutex);
if (pHead->version <= tsSdbObj->version) {
pthread_mutex_unlock(&tsSdbObj->mutex);
return TSDB_CODE_SUCCESS;
} else if (pHead->version != tsSdbObj->version + 1) {
pthread_mutex_unlock(&tsSdbObj->mutex);
sdbError("table:%s, failed to restore %s record:%s from file, version:%" PRId64 " too large, sdb version:%" PRId64,
pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version,
tsSdbObj->version);
return TSDB_CODE_OTHERS;
}
tsSdbObj->version = pHead->version;
sdbTrace("table:%s, success to restore %s record:%s from file, version:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
int32_t code = -1;
if (action == SDB_ACTION_INSERT) {
SSdbOperDesc oper = {
.rowSize = pHead->len,
.rowData = pHead->cont,
.table = pTable,
};
code = (*pTable->decodeFp)(&oper);
if (code < 0) {
sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
pthread_mutex_unlock(&tsSdbObj->mutex);
return code;
}
code = sdbInsertLocal(pTable, &oper);
} else if (action == SDB_ACTION_DELETE) {
SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
SSdbOperDesc oper = {
.table = pTable,
.pObj = rowMeta->row,
};
code = sdbDeleteLocal(pTable, &oper);
} else if (action == SDB_ACTION_UPDATE) {
SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
SSdbOperDesc oper = {
.rowSize = pHead->len,
.rowData = pHead->cont,
.table = pTable,
};
code = (*pTable->decodeFp)(&oper);
if (code < 0) {
sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
pthread_mutex_unlock(&tsSdbObj->mutex);
return code;
}
code = sdbUpdateLocal(pTable, &oper);
}
pthread_mutex_unlock(&tsSdbObj->mutex);
return code;
}
int sdbProcessWrite(void *param, void *data, int type) {
static int sdbWrite(void *param, void *data, int type) {
SWalHead *pHead = data;
int32_t tableId = pHead->msgType / 10;
int32_t action = pHead->msgType % 10;
@ -366,14 +444,60 @@ int sdbProcessWrite(void *param, void *data, int type) {
SSdbTable *pTable = sdbGetTableFromId(tableId);
assert(pTable != NULL);
pthread_mutex_lock(&tsSdbObj.mutex);
if (pHead->version == 0) {
return sdbProcessWriteFromApp(pTable, pHead, action);
// assign version
tsSdbObj.version++;
pHead->version = tsSdbObj.version;
} else {
return sdbProcessWriteFromWal(pTable, pHead, action);
// for data from WAL or forward, version may be smaller
if (pHead->version <= tsSdbObj.version) {
pthread_mutex_unlock(&tsSdbObj.mutex);
return TSDB_CODE_SUCCESS;
} else if (pHead->version != tsSdbObj.version + 1) {
pthread_mutex_unlock(&tsSdbObj.mutex);
sdbError("table:%s, failed to restore %s record:%s from wal, version:%" PRId64 " too large, sdb version:%" PRId64,
pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version,
tsSdbObj.version);
return TSDB_CODE_OTHERS;
} else {
tsSdbObj.version = pHead->version;
}
}
int32_t sdbInsertRow(SSdbOperDesc *pOper) {
int32_t code = walWrite(tsSdbObj.wal, pHead);
if (code < 0) {
pthread_mutex_unlock(&tsSdbObj.mutex);
return code;
}
walFsync(tsSdbObj.wal);
sdbForwardToPeer(pHead);
pthread_mutex_unlock(&tsSdbObj.mutex);
// from app, oper is created
if (param == NULL) return code;
// from wal, should create oper
if (action == SDB_ACTION_INSERT) {
SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
code = (*pTable->decodeFp)(&oper);
return sdbInsertHash(pTable, &oper);
} else if (action == SDB_ACTION_DELETE) {
SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
SSdbOper oper = {.table = pTable, .pObj = rowMeta->row};
return sdbDeleteHash(pTable, &oper);
} else if (action == SDB_ACTION_UPDATE) {
SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
assert(rowMeta != NULL && rowMeta->row != NULL);
SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
code = (*pTable->decodeFp)(&oper);
return sdbUpdateHash(pTable, &oper);
} else { return TSDB_CODE_INVALID_MSG_TYPE; }
}
int32_t sdbInsertRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
@ -405,19 +529,19 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) {
(*pTable->encodeFp)(pOper);
pHead->len = pOper->rowSize;
int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
int32_t code = sdbWrite(NULL, pHead, pHead->msgType);
taosFreeQitem(pHead);
if (code < 0) return code;
}
return sdbInsertLocal(pTable, pOper);
return sdbInsertHash(pTable, pOper);
}
// row here can be object or null-terminated string
int32_t sdbDeleteRow(SSdbOperDesc *pOper) {
int32_t sdbDeleteRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
return -1;
@ -447,18 +571,19 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) {
pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
memcpy(pHead->cont, pOper->pObj, rowSize);
int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
int32_t code = sdbWrite(NULL, pHead, pHead->msgType);
taosFreeQitem(pHead);
if (code < 0) return code;
}
return sdbDeleteLocal(pTable, pOper);
return sdbDeleteHash(pTable, pOper);
}
int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
int32_t sdbUpdateRow(SSdbOper *pOper) {
SSdbTable *pTable = (SSdbTable *)pOper->table;
if (pTable == NULL) return -1;
SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj);
if (pMeta == NULL) {
sdbTrace("table:%s, record is not there, delete failed", pTable->tableName);
return -1;
@ -477,16 +602,17 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
(*pTable->encodeFp)(pOper);
pHead->len = pOper->rowSize;
int32_t code = sdbProcessWrite(tsSdbObj, pHead, pHead->msgType);
int32_t code = sdbWrite(NULL, pHead, pHead->msgType);
taosFreeQitem(pHead);
if (code < 0) return code;
}
return sdbUpdateLocal(pTable, pOper);
return sdbUpdateHash(pTable, pOper);
}
void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
SSdbTable *pTable = (SSdbTable *)handle;
SRowMeta * pMeta;
SSdbRow * pMeta;
*ppRow = NULL;
if (pTable == NULL) return NULL;
@ -520,13 +646,13 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
pTable->restoredFp = pDesc->restoredFp;
if (sdbInitIndexFp[pTable->keyType] != NULL) {
pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta));
pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SSdbRow));
}
pthread_mutex_init(&pTable->mutex, NULL);
tsSdbNumOfTables++;
tsSdbTableList[pTable->tableId] = pTable;
tsSdbObj.numOfTables++;
tsSdbObj.tableList[pTable->tableId] = pTable;
return pTable;
}
@ -534,16 +660,16 @@ void sdbCloseTable(void *handle) {
SSdbTable *pTable = (SSdbTable *)handle;
if (pTable == NULL) return;
tsSdbNumOfTables--;
tsSdbTableList[pTable->tableId] = NULL;
tsSdbObj.numOfTables--;
tsSdbObj.tableList[pTable->tableId] = NULL;
void *pNode = NULL;
while (1) {
SRowMeta *pMeta;
SSdbRow *pMeta;
pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta);
if (pMeta == NULL) break;
SSdbOperDesc oper = {
SSdbOper oper = {
.pObj = pMeta->row,
.table = pTable,
};
@ -557,6 +683,7 @@ void sdbCloseTable(void *handle) {
pthread_mutex_destroy(&pTable->mutex);
sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbNumOfTables);
sdbTrace("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables);
free(pTable);
}

2
src/mnode/src/mgmtShell.c
View File

@ -144,7 +144,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
return;
}
if (!mgmtIsMaster()) {
if (!sdbIsMaster()) {
// rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect());
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER);
rpcFreeCont(rpcMsg->pCont);

68
src/mnode/src/mgmtTable.c
View File

@ -83,12 +83,12 @@ static void mgmtDestroyChildTable(SChildTableObj *pTable) {
tfree(pTable);
}
static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtChildTableActionDestroy(SSdbOper *pOper) {
mgmtDestroyChildTable(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) {
SChildTableObj *pTable = pOper->pObj;
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
@ -128,7 +128,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtChildTableActionDelete(SSdbOper *pOper) {
SChildTableObj *pTable = pOper->pObj;
if (pTable->vgId == 0) {
return TSDB_CODE_INVALID_VGROUP_ID;
@ -169,7 +169,7 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
SChildTableObj *pNew = pOper->pObj;
SChildTableObj *pTable = mgmtGetChildTable(pNew->info.tableId);
if (pTable != pNew) {
@ -186,7 +186,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtChildTableActionEncode(SSdbOper *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SChildTableObj *pTable = pOper->pObj;
assert(pTable != NULL && pOper->rowData != NULL);
@ -208,7 +208,7 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj));
if (pTable == NULL) {
@ -252,7 +252,7 @@ static int32_t mgmtChildTableActionRestored() {
SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId);
if (pDb == NULL) {
mError("ctable:%s, failed to get db, discard it", pTable->info.tableId);
SSdbOperDesc desc = {0};
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -266,7 +266,7 @@ static int32_t mgmtChildTableActionRestored() {
if (pVgroup == NULL) {
mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -280,7 +280,7 @@ static int32_t mgmtChildTableActionRestored() {
mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -292,7 +292,7 @@ static int32_t mgmtChildTableActionRestored() {
if (pVgroup->tableList == NULL) {
mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -306,7 +306,7 @@ static int32_t mgmtChildTableActionRestored() {
if (pSuperTable == NULL) {
mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId);
pTable->vgId = 0;
SSdbOperDesc desc = {0};
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -347,7 +347,7 @@ static int32_t mgmtInitChildTables() {
return -1;
}
mTrace("child table is initialized");
mTrace("table:ctables is created");
return 0;
}
@ -392,12 +392,12 @@ static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
tfree(pStable);
}
static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtSuperTableActionDestroy(SSdbOper *pOper) {
mgmtDestroySuperTable(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtSuperTableActionInsert(SSdbOper *pOper) {
SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
@ -408,7 +408,7 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtSuperTableActionDelete(SSdbOper *pOper) {
SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) {
@ -420,7 +420,7 @@ static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
SSuperTableObj *pNew = pOper->pObj;
SSuperTableObj *pTable = mgmtGetSuperTable(pNew->info.tableId);
if (pTable != pNew) {
@ -435,7 +435,7 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtSuperTableActionEncode(SSdbOper *pOper) {
const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS;
SSuperTableObj *pStable = pOper->pObj;
@ -454,7 +454,7 @@ static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtSuperTableActionDecode(SSdbOper *pOper) {
assert(pOper->rowData != NULL);
SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj));
@ -505,7 +505,7 @@ static int32_t mgmtInitSuperTables() {
return -1;
}
mTrace("stables is initialized");
mTrace("table:stables is created");
return 0;
}
@ -731,7 +731,7 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
tschema[col].bytes = htons(tschema[col].bytes);
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@ -755,7 +755,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable
@ -806,7 +806,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
pStable->numOfColumns += ntags;
pStable->sversion++;
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@ -837,7 +837,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
pStable->schema = realloc(pStable->schema, schemaSize);
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@ -872,7 +872,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
SSchema *schema = (SSchema *) (pStable->schema + (pStable->numOfColumns + col) * sizeof(SSchema));
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@ -931,7 +931,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
mgmtDecAcctRef(pAcct);
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@ -968,7 +968,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
mgmtDecAcctRef(pAcct);
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
@ -1116,7 +1116,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
}
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsSuperTableSdb,
.pObj = pTable,
@ -1354,7 +1354,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj
}
}
SSdbOperDesc desc = {0};
SSdbOper desc = {0};
desc.type = SDB_OPER_GLOBAL;
desc.pObj = pTable;
desc.table = tsChildTableSdb;
@ -1508,7 +1508,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
mgmtDecAcctRef(pAcct);
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
@ -1542,7 +1542,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
mgmtDecAcctRef(pAcct);
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
@ -1687,7 +1687,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
}
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
@ -1716,7 +1716,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
}
if (pTable->superTable == pStable) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsChildTableSdb,
.pObj = pTable,
@ -1805,7 +1805,7 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
return;
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable
@ -1848,7 +1848,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId,
queueMsg->thandle, tstrerror(rpcMsg->code));
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable

22
src/mnode/src/mgmtUser.c
View File

@ -36,12 +36,12 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg);
static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg);
static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionDestroy(SSdbOper *pOper) {
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionInsert(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
@ -56,7 +56,7 @@ static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionDelete(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
@ -67,7 +67,7 @@ static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionUpdate(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
SUserObj *pSaved = mgmtGetUser(pUser->user);
if (pUser != pSaved) {
@ -77,14 +77,14 @@ static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionEncode(SSdbOper *pOper) {
SUserObj *pUser = pOper->pObj;
memcpy(pOper->rowData, pUser, tsUserUpdateSize);
pOper->rowSize = tsUserUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtUserActionDecode(SSdbOper *pOper) {
SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj));
if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@ -137,7 +137,7 @@ int32_t mgmtInitUsers() {
mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_USER, mgmtGetUserMeta);
mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mgmtRetrieveUsers);
mTrace("user data is initialized");
mTrace("table:users table is created");
return 0;
}
@ -154,7 +154,7 @@ void mgmtReleaseUser(SUserObj *pUser) {
}
static int32_t mgmtUpdateUser(SUserObj *pUser) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
@ -202,7 +202,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
pUser->superAuth = 1;
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
@ -219,7 +219,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
}
static int32_t mgmtDropUser(SUserObj *pUser) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser
@ -493,7 +493,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) {
if (pUser == NULL) break;
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsUserSdb,
.pObj = pUser,

30
src/mnode/src/mgmtVgroup.c
View File

@ -22,7 +22,7 @@
#include "tidpool.h"
#include "tsync.h"
#include "ttime.h"
#include "treplica.h"
#include "tbalance.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtDb.h"
@ -48,7 +48,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ;
static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle);
static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionDestroy(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->idPool) {
taosIdPoolCleanUp(pVgroup->idPool);
@ -62,7 +62,7 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
@ -104,7 +104,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionDelete(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->pDb != NULL) {
@ -124,7 +124,7 @@ static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
SVgObj *pNew = pOper->pObj;
SVgObj *pVgroup = mgmtGetVgroup(pNew->vgId);
if (pVgroup != pNew) {
@ -147,14 +147,14 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionEncode(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
pOper->rowSize = tsVgUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) {
static int32_t mgmtVgroupActionDecode(SSdbOper *pOper) {
SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj));
if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
@ -199,7 +199,7 @@ int32_t mgmtInitVgroups() {
mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp);
mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg);
mTrace("vgroup is initialized");
mTrace("table:vgroups is created");
return 0;
}
@ -213,7 +213,7 @@ SVgObj *mgmtGetVgroup(int32_t vgId) {
}
void mgmtUpdateVgroup(SVgObj *pVgroup) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
@ -249,7 +249,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
strcpy(pVgroup->dbName, pDb->name);
pVgroup->numOfVnodes = pDb->cfg.replications;
pVgroup->createdTime = taosGetTimestampMs();
if (replicaAllocVnodes(pVgroup) != 0) {
if (balanceAllocVnodes(pVgroup) != 0) {
mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
free(pVgroup);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES);
@ -257,7 +257,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
return;
}
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
@ -289,7 +289,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
} else {
mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mgmtSendDropVgroupMsg(pVgroup, NULL);
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
@ -596,7 +596,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(queueMsg);
mgmtAddToShellQueue(newMsg);
} else {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
@ -659,7 +659,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
if (queueMsg->received != queueMsg->expected) return;
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup
@ -716,7 +716,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
if (pVgroup == NULL) break;
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
SSdbOperDesc oper = {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,

4
src/query/src/queryExecutor.c
View File

@ -2479,7 +2479,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d",
GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order);
tsdb_query_handle_t pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle;
TsdbQueryHandleT pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle;
while (tsdbNextDataBlock(pQueryHandle)) {
if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
@ -4221,7 +4221,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) {
int64_t st = taosGetTimestampMs();
tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle;
TsdbQueryHandleT *pQueryHandle = pRuntimeEnv->pQueryHandle;
while (tsdbNextDataBlock(pQueryHandle)) {
if (isQueryKilled(pQInfo)) {
break;

121
src/tsdb/inc/tsdbMain.h
View File

@ -120,7 +120,7 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable);
#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
#define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
STsdbMeta *tsdbGetMeta(tsdb_repo_t *pRepo);
STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo);
int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg);
int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
@ -160,10 +160,10 @@ typedef struct {
STsdbCacheBlock *curBlock;
SCacheMem * mem;
SCacheMem * imem;
tsdb_repo_t * pRepo;
TsdbRepoT * pRepo;
} STsdbCache;
STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo);
STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo);
void tsdbFreeCache(STsdbCache *pCache);
void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);
@ -220,11 +220,12 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles);
void tsdbCloseFileH(STsdbFileH *pFileH);
int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader,
int toClose);
int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
int tsdbOpenFile(SFile *pFile, int oflag);
int tsdbCloseFile(SFile *pFile);
SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid);
int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid);
int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
@ -270,6 +271,8 @@ typedef struct {
TSKEY keyLast;
} SCompBlock;
// Maximum number of sub-blocks a super-block can have
#define TSDB_MAX_SUBBLOCKS 8
#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1)
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
@ -307,19 +310,11 @@ typedef struct {
SCompCol cols[];
} SCompData;
STsdbFileH *tsdbGetFile(tsdb_repo_t *pRepo);
STsdbFileH *tsdbGetFile(TsdbRepoT *pRepo);
int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast,
SDataCols *pCols);
int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables);
int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf);
int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf);
int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf);
int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData);
SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid);
void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey);
// TSDB repository definition
@ -375,9 +370,103 @@ typedef struct {
int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
int32_t tsdbTriggerCommit(tsdb_repo_t *repo);
int32_t tsdbLockRepo(tsdb_repo_t *repo);
int32_t tsdbUnLockRepo(tsdb_repo_t *repo);
int32_t tsdbTriggerCommit(TsdbRepoT *repo);
int32_t tsdbLockRepo(TsdbRepoT *repo);
int32_t tsdbUnLockRepo(TsdbRepoT *repo);
typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
typedef struct {
tsdb_rw_helper_t type; // helper type
int maxTables;
int maxRowSize;
int maxRows;
int maxCols;
int minRowsPerFileBlock;
int maxRowsPerFileBlock;
int8_t compress;
} SHelperCfg;
typedef struct {
int fid;
TSKEY minKey;
TSKEY maxKey;
// For read/write purpose
SFile headF;
SFile dataF;
SFile lastF;
// For write purpose only
SFile nHeadF;
SFile nLastF;
} SHelperFile;
typedef struct {
int64_t uid;
int32_t tid;
int32_t sversion;
} SHelperTable;
typedef struct {
// Global configuration
SHelperCfg config;
int8_t state;
// For file set usage
SHelperFile files;
SCompIdx * pCompIdx;
// For table set usage
SHelperTable tableInfo;
SCompInfo * pCompInfo;
bool hasOldLastBlock;
// For block set usage
SCompData *pCompData;
SDataCols *pDataCols[2];
} SRWHelper;
// --------- Helper state
#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state
#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set
#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded
#define TSDB_HELPER_TABLE_SET 0x4 // Table is set
#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded
#define TSDB_HELPER_TYPE(h) ((h)->config.type)
#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
// int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg);
void tsdbDestroyHelper(SRWHelper *pHelper);
void tsdbResetHelper(SRWHelper *pHelper);
// --------- For set operations
int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
// void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema);
void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
// --------- For read operations
int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target);
// --------- For write operations
int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);
int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper);
int tsdbWriteCompInfo(SRWHelper *pHelper);
int tsdbWriteCompIdx(SRWHelper *pHelper);
#ifdef __cplusplus
}

2
src/tsdb/src/tsdbCache.c
View File

@ -21,7 +21,7 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache);
static void tsdbFreeBlockList(SList *list);
static void tsdbFreeCacheMem(SCacheMem *mem);
STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo) {
STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) {
STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache));
if (pCache == NULL) return NULL;

131
src/tsdb/src/tsdbFile.c
View File

@ -21,10 +21,12 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <libgen.h>
#include "talgo.h"
#include "tchecksum.h"
#include "tsdbMain.h"
#include "tutil.h"
#include "talgo.h"
const char *tsdbFileSuffix[] = {
".head", // TSDB_FILE_TYPE_HEAD
@ -34,7 +36,6 @@ const char *tsdbFileSuffix[] = {
static int compFGroupKey(const void *key, const void *fgroup);
static int compFGroup(const void *arg1, const void *arg2);
static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
static int tsdbWriteFileHead(SFile *pFile);
static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables);
static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid);
@ -93,24 +94,36 @@ static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) {
return 0;
}
int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) {
if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1;
/**
* Create the file group if the file group not exists.
*
* @return A pointer to
*/
SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) {
if (pFileH->numOfFGroups >= pFileH->maxFGroups) return NULL;
SFileGroup fGroup;
SFileGroup *pFGroup = &fGroup;
if (tsdbSearchFGroup(pFileH, fid) == NULL) { // if not exists, create one
SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid);
if (pGroup == NULL) { // if not exists, create one
pFGroup->fileId = fid;
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) {
// TODO: deal with the ERROR here, remove those creaed file
return -1;
}
if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]),
type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0)
goto _err;
}
pFileH->fGroup[pFileH->numOfFGroups++] = fGroup;
qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup);
return tsdbSearchFGroup(pFileH, fid);
}
return 0;
return pGroup;
_err:
// TODO: deal with the err here
return NULL;
}
int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) {
@ -183,27 +196,27 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
return ret;
}
int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) {
SCompBlock *pBlock = pStartBlock;
for (int i = 0; i < numOfBlocks; i++) {
if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
pCols->numOfPoints += (pCompData->cols[0].len / 8);
for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
SCompCol *pCompCol = &(pCompData->cols[iCol]);
// pCols->numOfPoints += pBlock->numOfPoints;
int k = 0;
for (; k < pCols->numOfCols; k++) {
if (pCompCol->colId == pCols->cols[k].colId) break;
}
// int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) {
// SCompBlock *pBlock = pStartBlock;
// for (int i = 0; i < numOfBlocks; i++) {
// if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
// pCols->numOfPoints += (pCompData->cols[0].len / 8);
// for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
// SCompCol *pCompCol = &(pCompData->cols[iCol]);
// // pCols->numOfPoints += pBlock->numOfPoints;
// int k = 0;
// for (; k < pCols->numOfCols; k++) {
// if (pCompCol->colId == pCols->cols[k].colId) break;
// }
if (tsdbLoadColData(pFile, pCompCol, pBlock->offset,
(void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0)
return -1;
}
pStartBlock++;
}
return 0;
}
// if (tsdbLoadColData(pFile, pCompCol, pBlock->offset,
// (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0)
// return -1;
// }
// pStartBlock++;
// }
// return 0;
// }
int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) {
SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx);
@ -239,42 +252,42 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf
return 0;
}
int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) {
SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
// int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) {
// SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
// if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1;
// TODO: need to check the correctness
return 0;
}
// if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1;
// // TODO: need to check the correctness
// return 0;
// }
int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) {
SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
// int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) {
// SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1;
// if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1;
if (read(pFile->fd, buf, pIdx->len) < 0) return -1;
// if (read(pFile->fd, buf, pIdx->len) < 0) return -1;
// TODO: need to check the correctness
// // TODO: need to check the correctness
return 0;
}
// return 0;
// }
int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) {
// assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
// int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) {
// // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1;
size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols;
if (read(pFile->fd, buf, size) < 0) return -1;
// if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1;
// size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols;
// if (read(pFile->fd, buf, size) < 0) return -1;
return 0;
}
// return 0;
// }
int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) {
if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1;
if (read(pFile->fd, buf, pCol->len) < 0) return -1;
return 0;
}
// int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) {
// if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1;
// if (read(pFile->fd, buf, pCol->len) < 0) return -1;
// return 0;
// }
static int compFGroupKey(const void *key, const void *fgroup) {
int fid = *(int *)key;
@ -299,7 +312,7 @@ static int tsdbWriteFileHead(SFile *pFile) {
}
static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
int size = sizeof(SCompIdx) * maxTables;
int size = sizeof(SCompIdx) * maxTables + sizeof(TSCKSUM);
void *buf = calloc(1, size);
if (buf == NULL) return -1;
@ -308,6 +321,8 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
return -1;
}
taosCalcChecksumAppend(0, (uint8_t *)buf, size);
if (write(pFile->fd, buf, size) < 0) {
free(buf);
return -1;
@ -319,7 +334,7 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
return 0;
}
static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
if (dataDir == NULL || fname == NULL) return -1;
sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix);

465
src/tsdb/src/tsdbMain.c
View File

@ -12,15 +12,16 @@
#include <tlog.h>
#include <unistd.h>
// #include "taosdef.h"
// #include "disk.h"
#include "os.h"
#include "talgo.h"
#include "tsdb.h"
#include "tsdbMain.h"
#include "tscompression.h"
#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI // default precision
#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
#define TSDB_MIN_ID 0
#define TSDB_MAX_ID INT_MAX
#define TSDB_MIN_TABLES 10
@ -53,15 +54,16 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
// static int tsdbOpenMetaFile(char *tsdbDir);
static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock);
static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock);
static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg);
static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname);
static void * tsdbCommitData(void *arg);
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols);
static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey);
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper,
SDataCols *pDataCols);
static TSKEY tsdbNextIterKey(SSkipListIterator *pIter);
static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len,
int64_t uid);
// static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len,
// int64_t uid);
#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
@ -82,6 +84,7 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) {
pCfg->maxRowsPerFileBlock = -1;
pCfg->keep = -1;
pCfg->maxCacheSize = -1;
pCfg->compression = TWO_STAGE_COMP;
}
/**
@ -153,7 +156,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */)
*
* @return 0 for success, -1 for failure and the error number is set
*/
int32_t tsdbDropRepo(tsdb_repo_t *repo) {
int32_t tsdbDropRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
pRepo->state = TSDB_REPO_STATE_CLOSED;
@ -179,7 +182,7 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) {
*
* @return a TSDB repository handle on success, NULL for failure and the error number is set
*/
tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
char dataDir[128] = "\0";
if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) {
return NULL;
@ -202,7 +205,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
return NULL;
}
pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (tsdb_repo_t *)pRepo);
pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo);
if (pRepo->tsdbCache == NULL) {
tsdbFreeMeta(pRepo->tsdbMeta);
free(pRepo->rootDir);
@ -222,7 +225,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
pRepo->state = TSDB_REPO_STATE_ACTIVE;
return (tsdb_repo_t *)pRepo;
return (TsdbRepoT *)pRepo;
}
// static int32_t tsdbFlushCache(STsdbRepo *pRepo) {
@ -237,7 +240,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
*
* @return 0 for success, -1 for failure and the error number is set
*/
int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
int32_t tsdbCloseRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
if (pRepo == NULL) return 0;
@ -282,7 +285,7 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
*
* @return 0 for success, -1 for failure and the error number is set
*/
int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
pRepo->config = *pCfg;
@ -290,7 +293,7 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
return 0;
}
int32_t tsdbTriggerCommit(tsdb_repo_t *repo) {
int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
tsdbLockRepo(repo);
@ -322,12 +325,12 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) {
return 0;
}
int32_t tsdbLockRepo(tsdb_repo_t *repo) {
int32_t tsdbLockRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
return pthread_mutex_lock(&(pRepo->mutex));
}
int32_t tsdbUnLockRepo(tsdb_repo_t *repo) {
int32_t tsdbUnLockRepo(TsdbRepoT *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
return pthread_mutex_unlock(&(pRepo->mutex));
}
@ -340,35 +343,35 @@ int32_t tsdbUnLockRepo(tsdb_repo_t *repo) {
* @return a info struct handle on success, NULL for failure and the error number is set. The upper
* layers should free the info handle themselves or memory leak will occur
*/
STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) {
// TODO
return NULL;
}
int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg);
}
int tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
// TODO
return 0;
}
int tsdbDropTable(tsdb_repo_t *repo, STableId tableId) {
int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
if (repo == NULL) return -1;
STsdbRepo *pRepo = (STsdbRepo *)repo;
return tsdbDropTableImpl(pRepo->tsdbMeta, tableId);
}
STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tableId) {
STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) {
// TODO
return NULL;
}
// TODO: need to return the number of data inserted
int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg) {
SSubmitMsgIter msgIter;
tsdbInitSubmitMsgIter(pMsg, &msgIter);
@ -397,6 +400,7 @@ int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t ti
config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
config->tableId.uid = uid;
config->tableId.tid = tid;
config->name = strdup("test1");
return 0;
}
@ -552,12 +556,12 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
return pBlock;
}
STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo) {
STsdbMeta* tsdbGetMeta(TsdbRepoT* pRepo) {
STsdbRepo *tsdb = (STsdbRepo *)pRepo;
return tsdb->tsdbMeta;
}
STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo) {
STsdbFileH* tsdbGetFile(TsdbRepoT* pRepo) {
STsdbRepo* tsdb = (STsdbRepo*) pRepo;
return tsdb->tsdbFileH;
}
@ -571,6 +575,13 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
if (!IS_VALID_PRECISION(pCfg->precision)) return -1;
}
// Check compression
if (pCfg->compression == -1) {
pCfg->compression = TSDB_DEFAULT_COMPRESSION;
} else {
if (!IS_VALID_COMPRESSION(pCfg->compression)) return -1;
}
// Check tsdbId
if (pCfg->tsdbId < 0) return -1;
@ -761,7 +772,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
return 0;
}
static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid};
@ -785,6 +796,9 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
}
static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) {
ASSERT(maxRowsToRead > 0);
if (pIter == NULL) return 0;
int numOfRows = 0;
do {
@ -823,19 +837,16 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
if (pTable == NULL || pTable->imem == NULL) continue;
iters[tid] = tSkipListCreateIter(pTable->imem->pData);
if (iters[tid] == NULL) {
tsdbDestroyTableIters(iters, maxTables);
return NULL;
}
if (iters[tid] == NULL) goto _err;
if (!tSkipListIterNext(iters[tid])) {
// No data in this iterator
tSkipListDestroyIter(iters[tid]);
iters[tid] = NULL;
}
if (!tSkipListIterNext(iters[tid])) goto _err;
}
return iters;
_err:
tsdbDestroyTableIters(iters, maxTables);
return NULL;
}
static void tsdbFreeMemTable(SMemTable *pMemTable) {
@ -847,14 +858,17 @@ static void tsdbFreeMemTable(SMemTable *pMemTable) {
// Commit to file
static void *tsdbCommitData(void *arg) {
// TODO
printf("Starting to commit....\n");
STsdbRepo * pRepo = (STsdbRepo *)arg;
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbCache *pCache = pRepo->tsdbCache;
STsdbCfg * pCfg = &(pRepo->config);
SDataCols * pDataCols = NULL;
SRWHelper whelper = {0};
if (pCache->imem == NULL) return NULL;
pRepo->appH.walCallBack(pRepo->appH.appH);
// Create the iterator to read from cache
SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables);
if (iters == NULL) {
@ -862,23 +876,21 @@ static void *tsdbCommitData(void *arg) {
return NULL;
}
// Create a data column buffer for commit
SDataCols *pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock);
if (pDataCols == NULL) {
// TODO: deal with the error
return NULL;
}
if (tsdbInitWriteHelper(&whelper, pRepo) < 0) goto _exit;
if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit;
int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision);
int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision);
// Loop to commit to each file
for (int fid = sfid; fid <= efid; fid++) {
if (tsdbCommitToFile(pRepo, fid, iters, pDataCols) < 0) {
// TODO: deal with the error here
// assert(0);
if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
ASSERT(false);
goto _exit;
}
}
_exit:
tdFreeDataCols(pDataCols);
tsdbDestroyTableIters(iters, pCfg->maxTables);
@ -888,7 +900,6 @@ static void *tsdbCommitData(void *arg) {
free(pCache->imem);
pCache->imem = NULL;
pRepo->commit = 0;
// TODO: free the skiplist
for (int i = 0; i < pCfg->maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->imem) {
@ -901,19 +912,12 @@ static void *tsdbCommitData(void *arg) {
return NULL;
}
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) {
int isNewLastFile = 0;
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) {
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
STsdbCfg * pCfg = &pRepo->config;
SFile hFile, lFile;
SFileGroup *pGroup = NULL;
SCompIdx * pIndices = NULL;
SCompInfo * pCompInfo = NULL;
// size_t compInfoSize = 0;
// SCompBlock compBlock;
// SCompBlock *pBlock = &compBlock;
TSKEY minKey = 0, maxKey = 0;
tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
@ -922,334 +926,93 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey);
if (!hasDataToCommit) return 0; // No data to commit, just return
// TODO: make it more flexible
pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + sizeof(SCompBlock) * 1000);
// Create and open files for commit
tsdbGetDataDirName(pRepo, dataDir);
if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */
}
pGroup = tsdbOpenFilesForCommit(pFileH, fid);
if (pGroup == NULL) { /* TODO */
}
tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 1);
tsdbOpenFile(&hFile, O_RDWR);
if (0 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) {
// TODO: make it not to write the last file every time
tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0);
isNewLastFile = 1;
}
if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) goto _err;
// Load the SCompIdx
pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables);
if (pIndices == NULL) { /* TODO*/
}
if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */
}
lseek(hFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET);
// Open files for write/read
if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err;
// Loop to commit data in each table
for (int tid = 0; tid < pCfg->maxTables; tid++) {
STable * pTable = pMeta->tables[tid];
if (pTable == NULL) continue;
SSkipListIterator *pIter = iters[tid];
SCompIdx * pIdx = &pIndices[tid];
int nNewBlocks = 0;
if (pTable == NULL || pIter == NULL) continue;
/* If no new data to write for this table, just write the old data to new file
* if there are.
*/
if (!tsdbHasDataInRange(pIter, minKey, maxKey)) {
// has old data
if (pIdx->len > 0) {
goto _table_over;
// if (isNewLastFile && pIdx->hasLast) {
if (0) {
// need to move the last block to new file
if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */
}
if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */
}
tdInitDataCols(pCols, tsdbGetTableSchema(pMeta, pTable));
SCompBlock *pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks);
int nBlocks = 0;
TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks);
SCompData tBlock;
int64_t toffset;
int32_t tlen;
tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock);
tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid);
pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks);
pTBlock->offset = toffset;
pTBlock->len = tlen;
pTBlock->numOfPoints = pCols->numOfPoints;
pTBlock->numOfSubBlocks = 1;
pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR);
if (nBlocks > 1) {
pIdx->len -= (sizeof(SCompBlock) * nBlocks);
}
write(hFile.fd, (void *)pCompInfo, pIdx->len);
} else {
pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR);
sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len);
hFile.info.size += pIdx->len;
}
}
continue;
}
pCompInfo->delimiter = TSDB_FILE_DELIMITER;
pCompInfo->checksum = 0;
pCompInfo->uid = pTable->tableId.uid;
// Load SCompBlock part if neccessary
// int isCompBlockLoaded = 0;
if (0) {
// if (pIdx->offset > 0) {
if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) {
// has last block || cache key overlap with commit key
pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100);
if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */
}
// if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1;
} else {
// TODO: No need to load the SCompBlock part, just sendfile the SCompBlock part
// and write those new blocks to it
}
}
tdInitDataCols(pCols, tsdbGetTableSchema(pMeta, pTable));
// Set the helper and the buffer dataCols object to help to write this table
tsdbSetHelperTable(pHelper, pTable, pRepo);
tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable));
// Loop to write the data in the cache to files. If no data to write, just break the loop
int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5;
while (1) {
tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols);
if (pCols->numOfPoints == 0) break;
int nLoop = 0;
while (true) {
int rowsRead = tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pDataCols);
assert(rowsRead >= 0);
if (pDataCols->numOfPoints == 0) break;
nLoop++;
int pointsWritten = pCols->numOfPoints;
// TODO: all write to the end of .data file
int64_t toffset = 0;
int32_t tlen = 0;
tsdbWriteBlockToFileImpl(&pGroup->files[TSDB_FILE_TYPE_DATA], pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid);
ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey);
ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey);
// Make the compBlock
SCompBlock *pTBlock = pCompInfo->blocks + nNewBlocks++;
pTBlock->offset = toffset;
pTBlock->len = tlen;
pTBlock->keyFirst = dataColsKeyFirst(pCols);
pTBlock->keyLast = dataColsKeyLast(pCols);
pTBlock->last = 0;
pTBlock->algorithm = 0;
pTBlock->numOfPoints = pCols->numOfPoints;
pTBlock->sversion = pTable->sversion;
pTBlock->numOfSubBlocks = 1;
pTBlock->numOfCols = pCols->numOfCols;
int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
ASSERT(rowsWritten != 0);
if (rowsWritten < 0) goto _err;
ASSERT(rowsWritten <= pDataCols->numOfPoints);
if (dataColsKeyLast(pCols) > pIdx->maxKey) pIdx->maxKey = dataColsKeyLast(pCols);
tdPopDataColsPoints(pCols, pointsWritten);
maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints;
tdPopDataColsPoints(pDataCols, rowsWritten);
maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfPoints;
}
ASSERT(pDataCols->numOfPoints == 0);
// Move the last block to the new .l file if neccessary
if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) goto _err;
_table_over:
// Write the SCompBlock part
pIdx->offset = lseek(hFile.fd, 0, SEEK_END);
if (pIdx->len > 0) {
int bytes = tsendfile(hFile.fd, pGroup->files[TSDB_FILE_TYPE_HEAD].fd, NULL, pIdx->len);
if (bytes < pIdx->len) {
printf("Failed to send file, reason: %s\n", strerror(errno));
}
if (nNewBlocks > 0) {
write(hFile.fd, (void *)(pCompInfo->blocks), sizeof(SCompBlock) * nNewBlocks);
pIdx->len += (sizeof(SCompBlock) * nNewBlocks);
}
} else {
if (nNewBlocks > 0) {
write(hFile.fd, (void *)pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks);
pIdx->len += sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks;
}
if (tsdbWriteCompInfo(pHelper) < 0) goto _err;
}
pIdx->checksum = 0;
pIdx->numOfSuperBlocks += nNewBlocks;
pIdx->hasLast = 0;
}
if (tsdbWriteCompIdx(pHelper) < 0) goto _err;
// Write the SCompIdx part
if (lseek(hFile.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {/* TODO */}
if (write(hFile.fd, (void *)pIndices, sizeof(SCompIdx) * pCfg->maxTables) < 0) {/* TODO */}
tsdbCloseHelperFile(pHelper, 0);
// TODO: make it atomic with some methods
pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF;
pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF;
pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF;
// close the files
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
tsdbCloseFile(&pGroup->files[type]);
}
tsdbCloseFile(&hFile);
if (isNewLastFile) tsdbCloseFile(&lFile);
// TODO: replace the .head and .last file
rename(hFile.fname, pGroup->files[TSDB_FILE_TYPE_HEAD].fname);
pGroup->files[TSDB_FILE_TYPE_HEAD].info = hFile.info;
if (isNewLastFile) {
rename(lFile.fname, pGroup->files[TSDB_FILE_TYPE_LAST].fname);
pGroup->files[TSDB_FILE_TYPE_LAST].info = lFile.info;
}
if (pIndices) free(pIndices);
if (pCompInfo) free(pCompInfo);
return 0;
}
static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey) {
if (pIter == NULL) return 0;
SSkipListNode *node = tSkipListIterGet(pIter);
if (node == NULL) return 0;
SDataRow row = SL_GET_NODE_DATA(node);
if (dataRowKey(row) >= minKey && dataRowKey(row) <= maxKey) return 1;
return 0;
}
static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) {
SSkipListIterator *pIter = iters[i];
if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1;
}
return 0;
}
static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) {
size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols;
SCompData *pCompData = (SCompData *)malloc(size);
if (pCompData == NULL) return -1;
pCompData->delimiter = TSDB_FILE_DELIMITER;
pCompData->uid = uid;
pCompData->numOfCols = pCols->numOfCols;
*offset = lseek(pFile->fd, 0, SEEK_END);
*len = size;
int toffset = size;
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SCompCol *pCompCol = pCompData->cols + iCol;
SDataCol *pDataCol = pCols->cols + iCol;
pCompCol->colId = pDataCol->colId;
pCompCol->type = pDataCol->type;
pCompCol->offset = toffset;
// TODO: add compression
pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite;
toffset += pCompCol->len;
}
// Write the block
if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err;
*len += size;
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SDataCol *pDataCol = pCols->cols + iCol;
SCompCol *pCompCol = pCompData->cols + iCol;
if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err;
*len += pCompCol->len;
}
tfree(pCompData);
return 0;
_err:
tfree(pCompData);
ASSERT(false);
tsdbCloseHelperFile(pHelper, 1);
return -1;
}
static int compareKeyBlock(const void *arg1, const void *arg2) {
TSKEY key = *(TSKEY *)arg1;
SCompBlock *pBlock = (SCompBlock *)arg2;
/**
* Return the next iterator key.
*
* @return the next key if iter has
* -1 if iter not
*/
static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
if (pIter == NULL) return -1;
if (key < pBlock->keyFirst) {
return -1;
} else if (key > pBlock->keyLast) {
return 1;
SSkipListNode *node = tSkipListIterGet(pIter);
if (node == NULL) return -1;
SDataRow row = SL_GET_NODE_DATA(node);
return dataRowKey(row);
}
static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) {
TSKEY nextKey;
for (int i = 0; i < nIters; i++) {
SSkipListIterator *pIter = iters[i];
nextKey = tsdbNextIterKey(pIter);
if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
return 0;
}
int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) {
STsdbCfg * pCfg = &(pRepo->config);
SFile * pFile = NULL;
int numOfPointsToWrite = 0;
int64_t offset = 0;
int32_t len = 0;
memset((void *)pCompBlock, 0, sizeof(SCompBlock));
if (pCompInfo == NULL) {
// Just append the data block to .data or .l or .last file
numOfPointsToWrite = pCols->numOfPoints;
if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file
pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]);
} else { // Write to .last or .l file
pCompBlock->last = 1;
if (lFile) {
pFile = lFile;
} else {
pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]);
}
}
tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid);
pCompBlock->offset = offset;
pCompBlock->len = len;
pCompBlock->algorithm = 2; // TODO : add to configuration
pCompBlock->sversion = pCols->sversion;
pCompBlock->numOfPoints = pCols->numOfPoints;
pCompBlock->numOfSubBlocks = 1;
pCompBlock->numOfCols = pCols->numOfCols;
pCompBlock->keyFirst = dataColsKeyFirst(pCols);
pCompBlock->keyLast = dataColsKeyLast(pCols);
} else {
// Need to merge the block to either the last block or the other block
TSKEY keyFirst = dataColsKeyFirst(pCols);
SCompBlock *pMergeBlock = NULL;
// Search the block to merge in
void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks,
compareKeyBlock, TD_GE);
if (ptr == NULL) {
// No block greater or equal than the key, but there are data in the .last file, need to merge the last file block
// and merge the data
pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1);
} else {
pMergeBlock = (SCompBlock *)ptr;
}
if (pMergeBlock->last) {
if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) {
// Need to load the data from .last and combine data in pCols to write to .data file
} else { // Just append the block to .last or .l file
if (lFile) {
// read the block from .last file and merge with pCols, write to .l file
} else {
// tsdbWriteBlockToFileImpl();
}
}
} else { // The block need to merge in .data file
}
}
return numOfPointsToWrite;
}

2
src/tsdb/src/tsdbMeta.c
View File

@ -225,7 +225,7 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) {
}
}
int32_t tsdbGetTableTagVal(tsdb_repo_t* repo, STableId id, int32_t colId, int16_t* type, int16_t* bytes, char** val) {
int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId id, int32_t colId, int16_t* type, int16_t* bytes, char** val) {
STsdbMeta* pMeta = tsdbGetMeta(repo);
STable* pTable = tsdbGetTableByUid(pMeta, id.uid);

1128
src/tsdb/src/tsdbRWHelper.c Normal file
View File

File diff suppressed because it is too large Load Diff

67
src/tsdb/src/tsdbRead.c
View File

@ -120,6 +120,7 @@ typedef struct STsdbQueryHandle {
SFileGroup* pFileGroup;
SFileGroupIter fileIter;
SCompIdx* compIndex;
SRWHelper rhelper;
} STsdbQueryHandle;
static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
@ -134,7 +135,7 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
pCompBlockLoadInfo->fileListIndex = -1;
}
tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
// todo 1. filter not exist table
// todo 2. add the reference count for each table that is involved in query
@ -142,7 +143,8 @@ tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, S
pQueryHandle->order = pCond->order;
pQueryHandle->window = pCond->twindow;
pQueryHandle->pTsdb = tsdb;
pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)),
pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx));
tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
pQueryHandle->cur.fid = -1;
@ -196,7 +198,7 @@ tsdb_query_handle_t* tsdbQueryTables(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, S
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
return (tsdb_query_handle_t)pQueryHandle;
return (TsdbQueryHandleT)pQueryHandle;
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
@ -288,14 +290,10 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
SFileGroup* fileGroup = pQueryHandle->pFileGroup;
assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0);
if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) {
fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY);
} else {
assert(FD_VALID(fileGroup->files[TSDB_FILE_TYPE_HEAD].fd));
}
tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup);
// load all the comp offset value for all tables in this file
tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables
// tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables
*numOfBlocks = 0;
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
@ -303,7 +301,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
for (int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
SCompIdx* compIndex = &pQueryHandle->compIndex[pCheckInfo->tableId.tid];
SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid];
if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file
continue;//no data blocks in the file belongs to pCheckInfo->pTable
} else {
@ -317,8 +315,13 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
pCheckInfo->compSize = compIndex->len;
}
tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo);
// tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo);
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->tableId.uid);
assert(pTable != NULL);
tsdbSetHelperTable(&pQueryHandle->rhelper, pTable, pQueryHandle->pTsdb);
tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
SCompInfo* pCompInfo = pCheckInfo->pCompInfo;
TSKEY s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey);
@ -409,12 +412,12 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj));
SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA];
if (pFile->fd == FD_INITIALIZER) {
pFile->fd = open(pFile->fname, O_RDONLY);
}
// SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA];
// if (pFile->fd == FD_INITIALIZER) {
// pFile->fd = open(pFile->fname, O_RDONLY);
// }
if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) {
if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) {
SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo;
pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup;
@ -427,9 +430,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
taosArrayDestroy(sa);
tfree(data);
TSKEY* d = (TSKEY*)pCheckInfo->pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData;
assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast);
return blockLoaded;
}
@ -596,8 +596,9 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j);
if (pCol->info.colId == colId) {
SDataCol* pDataCol = &pCols->cols[i];
memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start,
// SDataCol* pDataCol = &pCols->cols[i];
// pCol->pData = pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start;
memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start,
pQueryHandle->realNumOfRows * pCol->info.bytes);
break;
}
@ -909,7 +910,7 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
}
// handle data in cache situation
bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) {
bool tsdbNextDataBlock(TsdbQueryHandleT* pqHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle;
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
@ -937,7 +938,6 @@ bool tsdbNextDataBlock(tsdb_query_handle_t* pqHandle) {
return getDataBlocksInFiles(pQueryHandle);
}
}
static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey,
@ -1009,7 +1009,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max
}
// copy data from cache into data block
SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) {
SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
STable* pTable = NULL;
@ -1067,12 +1067,12 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) {
}
// return null for data block in cache
int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t* pQueryHandle, SDataStatis** pBlockStatis) {
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
*pBlockStatis = NULL;
return TSDB_CODE_SUCCESS;
}
SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList) {
SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
/**
* In the following two cases, the data has been loaded to SColumnInfoData.
* 1. data is from cache, 2. data block is not completed qualified to query time range
@ -1109,17 +1109,17 @@ SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList
}
}
int32_t tsdbResetQuery(tsdb_query_handle_t* pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) {
int32_t tsdbResetQuery(TsdbQueryHandleT* pQueryHandle, STimeWindow* window, TsdbPosT position, int16_t order) {
return 0;
}
SArray* tsdbRetrieveDataRow(tsdb_query_handle_t* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
tsdb_query_handle_t* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
return NULL;
}
SArray* tsdbGetTableList(tsdb_query_handle_t* pQueryHandle) { return NULL; }
SArray* tsdbGetTableList(TsdbQueryHandleT* pQueryHandle) { return NULL; }
static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@ -1432,7 +1432,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
return TSDB_CODE_SUCCESS;
}
int32_t tsdbQueryByTagsCond(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo,
int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols) {
STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@ -1474,7 +1474,7 @@ int32_t tsdbQueryByTagsCond(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond
return ret;
}
int32_t tsdbGetOneTableGroup(tsdb_repo_t* tsdb, int64_t uid, STableGroupInfo* pGroupInfo) {
int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGroupInfo) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
if (pTable == NULL) {
return TSDB_CODE_INVALID_TABLE_ID;
@ -1491,7 +1491,7 @@ int32_t tsdbGetOneTableGroup(tsdb_repo_t* tsdb, int64_t uid, STableGroupInfo* pG
return TSDB_CODE_SUCCESS;
}
void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) {
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle;
if (pQueryHandle == NULL) {
return;
@ -1523,5 +1523,6 @@ void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) {
taosArrayDestroy(pQueryHandle->pColumns);
tfree(pQueryHandle->pDataBlockInfo);
tsdbDestroyHelper(&pQueryHandle->rhelper);
tfree(pQueryHandle);
}

287
src/tsdb/tests/tsdbTests.cpp
View File

@ -5,12 +5,84 @@
#include "dataformat.h"
#include "tsdbMain.h"
double getCurTime() {
static double getCurTime() {
struct timeval tv;
gettimeofday(&tv, NULL);
return tv.tv_sec + tv.tv_usec * 1E-6;
}
typedef struct {
TsdbRepoT *pRepo;
int tid;
int64_t uid;
int sversion;
TSKEY startTime;
TSKEY interval;
int totalRows;
int rowsPerSubmit;
STSchema * pSchema;
} SInsertInfo;
static int insertData(SInsertInfo *pInfo) {
SSubmitMsg *pMsg =
(SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
if (pMsg == NULL) return -1;
TSKEY start_time = pInfo->startTime;
// Loop to write data
double stime = getCurTime();
for (int k = 0; k < pInfo->totalRows/pInfo->rowsPerSubmit; k++) {
memset((void *)pMsg, 0, sizeof(SSubmitMsg));
SSubmitBlk *pBlock = pMsg->blocks;
pBlock->uid = pInfo->uid;
pBlock->tid = pInfo->tid;
pBlock->sversion = pInfo->sversion;
pBlock->len = 0;
for (int i = 0; i < pInfo->rowsPerSubmit; i++) {
// start_time += 1000;
start_time += pInfo->interval;
SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
tdInitDataRow(row, pInfo->pSchema);
for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
if (j == 0) { // Just for timestamp
tdAppendColVal(row, (void *)(&start_time), schemaColAt(pInfo->pSchema, j));
} else { // For int
int val = 10;
tdAppendColVal(row, (void *)(&val), schemaColAt(pInfo->pSchema, j));
}
}
pBlock->len += dataRowLen(row);
}
pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
pMsg->numOfBlocks = 1;
pBlock->len = htonl(pBlock->len);
pBlock->numOfRows = htonl(pBlock->numOfRows);
pBlock->uid = htobe64(pBlock->uid);
pBlock->tid = htonl(pBlock->tid);
pBlock->sversion = htonl(pBlock->sversion);
pBlock->padding = htonl(pBlock->padding);
pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
pMsg->compressed = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg) < 0) {
tfree(pMsg);
return -1;
}
}
double etime = getCurTime();
printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows);
tfree(pMsg);
return 0;
}
TEST(TsdbTest, DISABLED_tableEncodeDecode) {
// TEST(TsdbTest, tableEncodeDecode) {
STable *pTable = (STable *)malloc(sizeof(STable));
@ -48,135 +120,132 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0);
}
TEST(TsdbTest, DISABLED_createRepo) {
// TEST(TsdbTest, createRepo) {
// STsdbCfg config;
// TEST(TsdbTest, DISABLED_createRepo) {
TEST(TsdbTest, createRepo) {
STsdbCfg config;
STsdbRepo *repo;
// // 1. Create a tsdb repository
// tsdbSetDefaultCfg(&config);
// tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
// ASSERT_NE(pRepo, nullptr);
// 1. Create a tsdb repository
tsdbSetDefaultCfg(&config);
ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0);
// // 2. Create a normal table
// STableCfg tCfg;
// ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
// ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
TsdbRepoT *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
ASSERT_NE(pRepo, nullptr);
// int nCols = 5;
// STSchema *schema = tdNewSchema(nCols);
// 2. Create a normal table
STableCfg tCfg;
ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
// for (int i = 0; i < nCols; i++) {
// if (i == 0) {
// tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
// } else {
// tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
// }
// }
// tsdbTableSetSchema(&tCfg, schema, true);
// tsdbCreateTable(pRepo, &tCfg);
// // // 3. Loop to write some simple data
// int nRows = 1;
// int rowsPerSubmit = 1;
// int64_t start_time = 1584081000000;
// SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit);
// double stime = getCurTime();
// for (int k = 0; k < nRows/rowsPerSubmit; k++) {
// memset((void *)pMsg, 0, sizeof(SSubmitMsg));
// SSubmitBlk *pBlock = pMsg->blocks;
// pBlock->uid = 987607499877672L;
// pBlock->tid = 0;
// pBlock->sversion = 0;
// pBlock->len = 0;
// for (int i = 0; i < rowsPerSubmit; i++) {
// // start_time += 1000;
// start_time += 1000;
// SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
// tdInitDataRow(row, schema);
// for (int j = 0; j < schemaNCols(schema); j++) {
// if (j == 0) { // Just for timestamp
// tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j));
// } else { // For int
// int val = 10;
// tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
// }
// }
// pBlock->len += dataRowLen(row);
// }
// pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
// pMsg->numOfBlocks = 1;
// pBlock->len = htonl(pBlock->len);
// pBlock->numOfRows = htonl(pBlock->numOfRows);
// pBlock->uid = htobe64(pBlock->uid);
// pBlock->tid = htonl(pBlock->tid);
// pBlock->sversion = htonl(pBlock->sversion);
// pBlock->padding = htonl(pBlock->padding);
// pMsg->length = htonl(pMsg->length);
// pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
// pMsg->compressed = htonl(pMsg->numOfBlocks);
// tsdbInsertData(pRepo, pMsg);
// }
// double etime = getCurTime();
// void *ptr = malloc(150000);
// free(ptr);
// printf("Spent %f seconds to write %d records\n", etime - stime, nRows);
// tsdbCloseRepo(pRepo);
int nCols = 5;
STSchema *schema = tdNewSchema(nCols);
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
}
// TEST(TsdbTest, DISABLED_openRepo) {
TEST(TsdbTest, openRepo) {
tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
ASSERT_NE(repo, nullptr);
tsdbTableSetSchema(&tCfg, schema, true);
STsdbRepo *pRepo = (STsdbRepo *)repo;
tsdbCreateTable(pRepo, &tCfg);
SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655);
// Insert Some Data
SInsertInfo iInfo = {
.pRepo = pRepo,
.tid = tCfg.tableId.tid,
.uid = tCfg.tableId.uid,
.sversion = tCfg.sversion,
.startTime = 1584081000000,
.interval = 1000,
.totalRows = 50,
.rowsPerSubmit = 1,
.pSchema = schema
};
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
tsdbOpenFile(&pGroup->files[type], O_RDONLY);
}
ASSERT_EQ(insertData(&iInfo), 0);
SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
// Close the repository
tsdbCloseRepo(pRepo);
SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
// Open the repository again
pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
repo = (STsdbRepo *)pRepo;
ASSERT_NE(pRepo, nullptr);
tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo);
// Insert more data
iInfo.startTime = iInfo.startTime + iInfo.interval * iInfo.totalRows;
iInfo.totalRows = 10;
iInfo.pRepo = pRepo;
ASSERT_EQ(insertData(&iInfo), 0);
int blockIdx = 0;
SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
// Close the repository
tsdbCloseRepo(pRepo);
SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
// Open the repository again
pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
repo = (STsdbRepo *)pRepo;
ASSERT_NE(pRepo, nullptr);
tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
// Read from file
SRWHelper rhelper;
tsdbInitReadHelper(&rhelper, repo);
STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833);
ASSERT_NE(pFGroup, nullptr);
ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0);
tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
tdResetDataCols(pDataCols);
tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
STable *pTable = tsdbGetTableByUid(repo->tsdbMeta, tCfg.tableId.uid);
ASSERT_NE(pTable, nullptr);
tsdbSetHelperTable(&rhelper, pTable, repo);
ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0);
ASSERT_EQ(tsdbLoadBlockData(&rhelper, blockAtIdx(&rhelper, 0), NULL), 0);
int k = 0;
}
TEST(TsdbTest, DISABLED_openRepo) {
// TEST(TsdbTest, openRepo) {
// tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
// ASSERT_NE(repo, nullptr);
// STsdbRepo *pRepo = (STsdbRepo *)repo;
// SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655);
// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
// tsdbOpenFile(&pGroup->files[type], O_RDONLY);
// }
// SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
// tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
// SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
// tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo);
// int blockIdx = 0;
// SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
// tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
// STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
// SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
// tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
// tdResetDataCols(pDataCols);
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
// int k = 0;
}

7
src/util/inc/tutil.h
View File

@ -176,6 +176,13 @@ uint32_t ip2uint(const char *const ip_addr);
void taosSetAllocMode(int mode, const char* path, bool autoDump);
void taosDumpMemoryLeak();
void * tmalloc(size_t size);
void * tcalloc(size_t nmemb, size_t size);
size_t tsizeof(void *ptr);
void tmemset(void *ptr, int c);
void * trealloc(void *ptr, size_t size);
void tzfree(void *ptr);
#ifdef TAOS_MEM_CHECK
void * taos_malloc(size_t size, const char *file, uint32_t line);

45
src/util/src/tutil.c
View File

@ -618,3 +618,48 @@ char *taosCharsetReplace(char *charsetstr) {
return strdup(charsetstr);
}
void *tmalloc(size_t size) {
if (size <= 0) return NULL;
void *ret = malloc(size + sizeof(size_t));
if (ret == NULL) return NULL;
*(size_t *)ret = size;
return (void *)((char *)ret + sizeof(size_t));
}
void *tcalloc(size_t nmemb, size_t size) {
size_t tsize = nmemb * size;
void * ret = tmalloc(tsize);
if (ret == NULL) return NULL;
tmemset(ret, 0);
return ret;
}
size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
void * trealloc(void *ptr, size_t size) {
if (ptr == NULL) return tmalloc(size);
if (size <= tsizeof(ptr)) return ptr;
void * tptr = (void *)((char *)ptr - sizeof(size_t));
size_t tsize = size + sizeof(size_t);
tptr = realloc(tptr, tsize);
if (tptr == NULL) return NULL;
*(size_t *)tptr = size;
return (void *)((char *)tptr + sizeof(size_t));
}
void tzfree(void *ptr) {
if (ptr) {
free((void *)((char *)ptr - sizeof(size_t)));
}
}

414
src/util/tests/taosbsearchTest.cpp Normal file
View File

@ -0,0 +1,414 @@
#include <gtest/gtest.h>
#include "talgo.h"
static int compareFunc(const void *arg1, const void *arg2) { return (*(int *)arg1) - (*(int *)arg2); }
TEST(testCase, taosbsearch_equal) {
// For equal test
int key = 3;
void *pRet = NULL;
pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
// 1 element
int array1[1] = {5};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 5;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_EQ);
ASSERT_NE(pRet, nullptr);
ASSERT_EQ(*(int *)pRet, key);
// 2 element
int array2[2] = {3, 6};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(*(int *)pRet, key);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
// 3 element
int array3[3] = {3, 6, 8};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_EQ);
ASSERT_EQ(pRet, nullptr);
}
TEST(testCase, taosbsearch_greater_or_equal) {
// For equal test
int key = 3;
void *pRet = NULL;
pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(pRet, nullptr);
// 1 element
int array1[1] = {5};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 5);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(pRet, nullptr);
key = 5;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GE);
ASSERT_NE(pRet, nullptr);
ASSERT_EQ(*(int *)pRet, 5);
// 2 element
int array2[2] = {3, 6};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(pRet, nullptr);
// 3 element
int array3[3] = {3, 6, 8};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 8);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(pRet, nullptr);
// 4 element
int array4[4] = {3, 6, 8, 11};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 8);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 11);
key = 11;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 11);
key = 13;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(pRet, nullptr);
// 5 element
int array5[5] = {3, 6, 8, 11, 15};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 8);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 11);
key = 11;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 11);
key = 13;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 15);
key = 15;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(*(int *)pRet, 15);
key = 17;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GE);
ASSERT_EQ(pRet, nullptr);
}
TEST(testCase, taosbsearch_less_or_equal) {
// For equal test
int key = 3;
void *pRet = NULL;
pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(pRet, nullptr);
// 1 element
int array1[1] = {5};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(pRet, nullptr);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 5);
key = 5;
pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LE);
ASSERT_NE(pRet, nullptr);
ASSERT_EQ(*(int *)pRet, 5);
// 2 element
int array2[2] = {3, 6};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(pRet, nullptr);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
// 3 element
int array3[3] = {3, 6, 8};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(pRet, nullptr);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 8);
// 4 element
int array4[4] = {3, 6, 8, 11};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(pRet, nullptr);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 8);
key = 11;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 11);
key = 13;
pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 11);
// 5 element
int array5[5] = {3, 6, 8, 11, 15};
key = 1;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(pRet, nullptr);
key = 3;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 4;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 3);
key = 6;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 7;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 6);
key = 8;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 8);
key = 9;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 8);
key = 11;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 11);
key = 13;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 11);
key = 15;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 15);
key = 17;
pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE);
ASSERT_EQ(*(int *)pRet, 15);
}

8
src/vnode/src/vnodeMain.c
View File

@ -45,6 +45,9 @@ static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT;
#ifndef _SYNC
tsync_h syncStart(const SSyncInfo *info) { return NULL; }
int syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle) { return 0; }
void syncStop(tsync_h shandle) {}
int syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
#endif
static void vnodeInit() {
@ -88,6 +91,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
STsdbCfg tsdbCfg = {0};
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = -1;
tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId;
tsdbCfg.maxTables = pVnodeCfg->cfg.maxSessions;
tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile;
@ -270,10 +274,6 @@ void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
}
void *vnodeGetTsdb(void *pVnode) {
return ((SVnodeObj *)pVnode)->tsdb;
}
void vnodeBuildStatusMsg(void *param) {
SDMStatusMsg *pStatus = param;
taosVisitIntHashWithFp(tsDnodeVnodesHash, vnodeBuildVloadMsg, pStatus);

3
src/vnode/src/vnodeRead.c
View File

@ -56,8 +56,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont
qinfo_t pQInfo = NULL;
if (contLen != 0) {
void* tsdb = vnodeGetTsdb(pVnode);
pRet->code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo);
pRet->code = qCreateQueryInfo(pVnode->tsdb, pQueryTableMsg, &pQInfo);
SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->qhandle = htobe64((uint64_t) (pQInfo));

6
src/vnode/src/vnodeWrite.c
View File

@ -147,8 +147,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
tsdbTableSetTagValue(&tCfg, dataRow, false);
}
void *pTsdb = vnodeGetTsdb(pVnode);
code = tsdbCreateTable(pTsdb, &tCfg);
code = tsdbCreateTable(pVnode->tsdb, &tCfg);
tfree(pDestSchema);
@ -211,8 +210,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
tsdbTableSetTagValue(&tCfg, dataRow, false);
}
void *pTsdb = vnodeGetTsdb(pVnode);
code = tsdbAlterTable(pTsdb, &tCfg);
code = tsdbAlterTable(pVnode->tsdb, &tCfg);
tfree(pDestSchema);

4
src/wal/src/walMain.c
View File

@ -208,6 +208,8 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
}
}
closedir(dir);
if (count == 0) {
if (pWal->keep) code = walRenew(pWal);
return code;
@ -248,8 +250,6 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
}
}
closedir(dir);
return code;
}

3
tests/test/c/CMakeLists.txt
View File

@ -14,4 +14,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
add_executable(importOneRow importOneRow.c)
target_link_libraries(importOneRow taos_static pthread)
add_executable(importPerTabe importPerTabe.c)
target_link_libraries(importPerTabe taos_static pthread)
ENDIF()

321
tests/test/c/importPerTabe.c Normal file
View File

@ -0,0 +1,321 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taos.h"
#include "tlog.h"
#include "ttimer.h"
#include "tutil.h"
#define MAX_RANDOM_POINTS 20000
#define GREEN "\033[1;32m"
#define NC "\033[0m"
typedef struct {
int64_t rowsPerTable;
int64_t pointsPerTable;
int64_t tableBeginIndex;
int64_t tableEndIndex;
int threadIndex;
char dbName[32];
char stableName[64];
pthread_t thread;
} SInfo;
void *syncTest(void *param);
void generateRandomPoints();
void shellParseArgument(int argc, char *argv[]);
void createDbAndTable();
void insertData();
int32_t randomData[MAX_RANDOM_POINTS];
int64_t rowsPerTable = 10000;
int64_t pointsPerTable = 1;
int64_t numOfThreads = 1;
int64_t numOfTablesPerThread = 1;
char dbName[32] = "db";
char stableName[64] = "st";
int32_t cache = 16384;
int32_t tables = 1000;
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
generateRandomPoints();
taos_init();
createDbAndTable();
insertData();
}
void createDbAndTable() {
dPrint("start to create table");
TAOS * con;
struct timeval systemTime;
int64_t st, et;
char qstr[64000];
con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
if (con == NULL) {
dError("failed to connect to DB, reason:%s", taos_errstr(con));
exit(1);
}
sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
if (taos_query(con, qstr)) {
dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
exit(0);
}
sprintf(qstr, "use %s", dbName);
if (taos_query(con, qstr)) {
dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
int64_t totalTables = numOfTablesPerThread * numOfThreads;
if (strcmp(stableName, "no") != 0) {
int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName);
for (int64_t f = 0; f < pointsPerTable; ++f) {
len += sprintf(qstr + len, ", f%ld double", f);
}
sprintf(qstr + len, ") tags(t int)");
if (taos_query(con, qstr)) {
dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
for (int64_t t = 0; t < totalTables; ++t) {
sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
if (taos_query(con, qstr)) {
dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
}
} else {
for (int64_t t = 0; t < totalTables; ++t) {
int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t);
for (int64_t f = 0; f < pointsPerTable; ++f) {
len += sprintf(qstr + len, ", f%ld double", f);
}
sprintf(qstr + len, ")");
if (taos_query(con, qstr)) {
dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
}
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables);
}
void insertData() {
struct timeval systemTime;
int64_t st, et;
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
dPrint("%d threads are spawned to import data", numOfThreads);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads);
// Start threads to write
for (int i = 0; i < numOfThreads; ++i) {
pInfo[i].rowsPerTable = rowsPerTable;
pInfo[i].pointsPerTable = pointsPerTable;
pInfo[i].tableBeginIndex = i * numOfTablesPerThread;
pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread;
pInfo[i].threadIndex = i;
strcpy(pInfo[i].dbName, dbName);
strcpy(pInfo[i].stableName, stableName);
pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i));
}
taosMsleep(300);
for (int i = 0; i < numOfThreads; i++) {
pthread_join(pInfo[i].thread, NULL);
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
double seconds = (et - st) / 1000.0 / 1000.0;
int64_t totalTables = numOfTablesPerThread * numOfThreads;
int64_t totalRows = totalTables * rowsPerTable;
int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable;
double speedOfRows = totalRows / seconds;
double speedOfPoints = totalPoints / seconds;
dPrint(
"%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf "
"PointsPerSecond:%.1lf%s",
GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC);
dPrint("threads exit");
pthread_attr_destroy(&thattr);
free(pInfo);
}
void *syncTest(void *param) {
TAOS * con;
SInfo * pInfo = (SInfo *)param;
struct timeval systemTime;
int64_t st, et;
char qstr[65000];
int maxBytes = 60000;
dPrint("thread:%d, start to run", pInfo->threadIndex);
con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
if (con == NULL) {
dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
exit(1);
}
sprintf(qstr, "use %s", pInfo->dbName);
taos_query(con, qstr);
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
int64_t start = 1587225600000;
int64_t interval = 1000; // 1000 ms
char *sql = qstr;
char inserStr[] = "import into";
int len = sprintf(sql, "%s", inserStr);
for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) {
len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
for (int64_t row = 0; row < pInfo->rowsPerTable; row++) {
len += sprintf(sql + len, "(%ld", start - row * interval);
for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) {
len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]);
// len += sprintf(sql + len, ",%ld", row);
}
len += sprintf(sql + len, ")");
if (len > maxBytes) {
if (taos_query(con, qstr)) {
dError("thread:%d, failed to import table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
table, row, taos_errstr(con));
}
// "insert into"
len = sprintf(sql, "%s", inserStr);
// "insert into st1 values"
if (row != pInfo->rowsPerTable - 1) {
len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
}
}
}
}
if (len != strlen(inserStr)) {
taos_query(con, qstr);
}
gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex;
int64_t totalRows = totalTables * pInfo->rowsPerTable;
int64_t totalPoints = totalRows * pInfo->pointsPerTable;
dPrint("thread:%d, import finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex,
(et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints);
return NULL;
}
void generateRandomPoints() {
for (int r = 0; r < MAX_RANDOM_POINTS; ++r) {
randomData[r] = rand() % 1000;
}
}
void printHelp() {
char indent[10] = " ";
printf("Used to test the performance of TDengine\n After writing all the data in one table, start the next table\n");
printf("%s%s\n", indent, "-d");
printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
printf("%s%s\n", indent, "-s");
printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table");
printf("%s%s\n", indent, "-c");
printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
printf("%s%s\n", indent, "-r");
printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable);
printf("%s%s\n", indent, "-p");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable);
printf("%s%s\n", indent, "-t");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads);
printf("%s%s\n", indent, "-n");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread);
printf("%s%s\n", indent, "-tables");
printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables);
printf("%s%s\n", indent, "-cache");
printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache);
exit(EXIT_SUCCESS);
}
void shellParseArgument(int argc, char *argv[]) {
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
printHelp();
exit(0);
} else if (strcmp(argv[i], "-d") == 0) {
strcpy(dbName, argv[++i]);
} else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]);
} else if (strcmp(argv[i], "-s") == 0) {
strcpy(stableName, argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) {
rowsPerTable = atoi(argv[++i]);
} else if (strcmp(argv[i], "-p") == 0) {
pointsPerTable = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t") == 0) {
numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-n") == 0) {
numOfTablesPerThread = atoi(argv[++i]);
} else if (strcmp(argv[i], "-tables") == 0) {
tables = atoi(argv[++i]);
} else if (strcmp(argv[i], "-cache") == 0) {
cache = atoi(argv[++i]);
} else {
}
}
dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC);
dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
dPrint("%sdbName:%s%s", GREEN, dbName, NC);
dPrint("%stableName:%s%s", GREEN, stableName, NC);
dPrint("%sstart to run%s", GREEN, NC);
}