xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-186] fix bugs for last_row.

This commit is contained in:
hjxilinx 2020-04-29 11:17:15 +08:00
parent 6bb6ea469b 5a3f5224f0
commit 41d414eaa7
199 changed files with 8830 additions and 2323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -33,4 +33,5 @@ Target/
*.failed
*.sql
sim/
*DS_Store

7
.travis.yml
View File

@ -46,10 +46,10 @@ matrix:
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
cd ${TRAVIS_BUILD_DIR}/tests
./test-all.sh || travis_terminate $?
./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $?
cd ${TRAVIS_BUILD_DIR}/tests/pytest
./simpletest.sh -g 2>&1 | tee mem-error-out.txt
./smoketest.sh -g 2>&1 | tee mem-error-out.txt
sleep 1
# Color setting
@ -86,13 +86,12 @@ matrix:
addons:
coverity_scan:
# GitHub project metadata
# ** specific to your project **
project:
name: TDengine
version: 2.x
description: taosdata/TDengine
description: TDengine
# Where email notification of build analysis results will be sent
notification_email: sdsang@taosdata.com

2
src/client/CMakeLists.txt
View File

@ -48,7 +48,7 @@ ELSEIF (TD_WINDOWS_64)
IF (NOT TD_GODLL)
SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def)
ENDIF ()
TARGET_LINK_LIBRARIES(taos trpc)
TARGET_LINK_LIBRARIES(taos trpc tutil)
ELSEIF (TD_DARWIN_64)
SET(CMAKE_MACOSX_RPATH 1)

3
src/client/inc/tscUtil.h
View File

@ -25,6 +25,7 @@ extern "C" {
*/
#include "os.h"
#include "tbuffer.h"
#include "exception.h"
#include "qextbuffer.h"
#include "taosdef.h"
#include "tscSecondaryMerge.h"
@ -177,7 +178,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId);
// get starter position of metric query condition (query on tags) in SSqlCmd.payload
SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid);
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf);
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw);
void tscTagCondCopy(STagCond* dest, const STagCond* src);
void tscTagCondRelease(STagCond* pCond);

2
src/client/inc/tsclient.h
View File

@ -45,8 +45,6 @@ enum {
DATA_FROM_DATA_FILE = 2,
};
typedef SCMSTableVgroupRspMsg SVgroupsInfo;
typedef struct STableComInfo {
uint8_t numOfTags;
uint8_t precision;

2
src/client/src/tscAsync.c
View File

@ -47,7 +47,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
pSql->signature = pSql;
pSql->param = param;
pSql->pTscObj = pObj;
pSql->maxRetry = TSDB_REPLICA_MAX_NUM;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM;
pSql->fp = fp;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {

97
src/client/src/tscSQLParser.c
View File

@ -385,7 +385,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
pMsg += sizeof(SMgmtHead);
SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg;
strncpy(pCfg->ip, pDCL->a[0].z, pDCL->a[0].n);
strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n);
@ -1185,10 +1185,18 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
return invalidSqlErrMsg(pQueryInfo->msg, "invalid arithmetic expression in select clause");
}
SBuffer buf = exprTreeToBinary(pNode);
SBufferWriter bw = tbufInitWriter(NULL, false);
TRY(0) {
exprTreeToBinary(&bw, pNode);
} CATCH(code) {
tbufCloseWriter(&bw);
UNUSED(code);
// TODO: other error handling
} END_TRY
size_t len = tbufTell(&buf);
char* c = tbufGetData(&buf, true);
size_t len = tbufTell(&bw);
char* c = tbufGetData(&bw, true);
// set the serialized binary string as the parameter of arithmetic expression
addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex);
@ -3294,7 +3302,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
// set join query condition
if (pRight->nSQLOptr == TK_ID) { // no need to keep the timestamp join condition
pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY;
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY);
pCondExpr->tsJoin = true;
/*
@ -3714,14 +3722,14 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p
getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index);
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
// int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index);
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
// columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
}
}
@ -3739,7 +3747,15 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
SArray* colList = taosArrayInit(10, sizeof(SColIndex));
ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, colList);
SBuffer buf = exprTreeToBinary(p);
SBufferWriter bw = tbufInitWriter(NULL, false);
TRY(0) {
exprTreeToBinary(&bw, p);
} CATCH(code) {
tbufCloseWriter(&bw);
UNUSED(code);
// TODO: more error handling
} END_TRY
// add to source column list
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@ -3753,7 +3769,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
tscColumnListInsert(pTableMetaInfo->tagColList, &index);
}
tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &buf);
tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw);
doCompactQueryExpr(pExpr);
tSQLExprDestroy(p1);
@ -4654,17 +4670,17 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL*
return TSDB_CODE_SUCCESS;
}
// todo refactor
if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) {
bool queryOnTags = tscQueryTags(pQueryInfo);
if (queryOnTags != true) { // local handle the super table tag query
if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query
if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) {
return invalidSqlErrMsg(pQueryInfo->msg, msg3);
}
// for projection query on super table, all queries are subqueries
if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) {
pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY;
}
}
@ -4785,16 +4801,15 @@ static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCrea
}
static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->blocksPerTable = htons(pCreateDb->numOfBlocksPerTable);
pMsg->compression = pCreateDb->compressionLevel;
pMsg->commitLog = (char)pCreateDb->commitLog;
pMsg->commitTime = htonl(pCreateDb->commitTime);
pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode);
pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks;
pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
pMsg->rowsInFileBlock = htonl(pCreateDb->rowPerFileBlock);
pMsg->cacheBlockSize = htonl(-1);
pMsg->totalBlocks = htonl(-1);
pMsg->daysPerFile = htonl(pCreateDb->daysPerFile);
pMsg->commitTime = htonl(pCreateDb->commitTime);
pMsg->minRowsPerFileBlock = htonl(-1);
pMsg->maxRowsPerFileBlock = htonl(-1);
pMsg->compression = pCreateDb->compressionLevel;
pMsg->commitLog = (char)pCreateDb->commitLog;
pMsg->replications = pCreateDb->replica;
pMsg->ignoreExist = pCreateDb->ignoreExists;
}
@ -5327,29 +5342,22 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
char msg[512] = {0};
if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 2)) {
if (pCreate->commitLog != -1 && (pCreate->commitLog < TSDB_MIN_CLOG_LEVEL || pCreate->commitLog > TSDB_MAX_CLOG_LEVEL)) {
snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
if (pCreate->replications != -1 &&
(pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) {
(pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) {
snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications,
TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM);
TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
int32_t val = htonl(pCreate->daysPerFile);
if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) {
if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) {
snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val,
TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
val = htonl(pCreate->rowsInFileBlock);
if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) {
snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val,
TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK);
TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
@ -5361,9 +5369,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
}
val = htonl(pCreate->maxSessions);
if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) {
if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) {
snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val,
TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE);
TSDB_MIN_TABLES, TSDB_MAX_TABLES);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
@ -5373,24 +5381,17 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS ||
pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) {
snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]",
pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
val = htonl(pCreate->commitTime);
if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) {
if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) {
snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val,
TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL);
TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
if (pCreate->compression != -1 &&
(pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) {
(pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) {
snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression,
TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL);
TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}

232
src/client/src/tscServer.c
View File

@ -49,11 +49,11 @@ static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) {
SRpcIpSet* pIpList = &pSql->ipList;
pIpList->numOfIps = pTableMeta->vgroupInfo.numOfIps;
pIpList->port = tsDnodeShellPort;
pIpList->inUse = 0;
for(int32_t i = 0; i < pTableMeta->vgroupInfo.numOfIps; ++i) {
pIpList->ip[i] = pTableMeta->vgroupInfo.ipAddr[i].ip;
strcpy(pIpList->fqdn[i], pTableMeta->vgroupInfo.ipAddr[i].fqdn);
pIpList->port[i] = pTableMeta->vgroupInfo.ipAddr[i].port;
}
}
@ -62,7 +62,7 @@ void tscPrintMgmtIp() {
tscError("invalid mgmt IP list:%d", tscMgmtIpSet.numOfIps);
} else {
for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
tscTrace("mgmt index:%d ip:%d", i, tscMgmtIpSet.ip[i]);
tscTrace("mgmt index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
}
}
}
@ -70,9 +70,8 @@ void tscPrintMgmtIp() {
void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) {
tscMgmtIpSet.numOfIps = pIpList->numOfIps;
tscMgmtIpSet.inUse = pIpList->inUse;
tscMgmtIpSet.port = htons(pIpList->port);
for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
tscMgmtIpSet.ip[i] = htonl(pIpList->ip[i]);
tscMgmtIpSet.port[i] = htons(pIpList->port[i]);
}
}
@ -80,8 +79,7 @@ void tscSetMgmtIpListFromEdge() {
if (tscMgmtIpSet.numOfIps != 1) {
tscMgmtIpSet.numOfIps = 1;
tscMgmtIpSet.inUse = 0;
tscMgmtIpSet.port = tsMnodeShellPort;
tscMgmtIpSet.ip[0] = inet_addr(tsMasterIp);
taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
tscTrace("edge mgmt IP list:");
tscPrintMgmtIp();
}
@ -213,9 +211,6 @@ int tscSendMsgToServer(SSqlObj *pSql) {
rpcSendRequest(pVnodeConn, &pSql->ipList, &rpcMsg);
} else {
pSql->ipList = tscMgmtIpSet;
pSql->ipList.port = tsMnodeShellPort;
tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port);
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = {
.msgType = pSql->cmd.msgType,
@ -224,6 +219,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.handle = pSql,
.code = 0
};
tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg);
}
@ -294,11 +290,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
}
}
}
if (pRes->code == TSDB_CODE_SUCCESS) {
tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry);
pSql->retry = 0;
}
pRes->rspLen = 0;
@ -308,6 +299,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code));
}
if (pRes->code == TSDB_CODE_SUCCESS) {
tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry);
pSql->retry = 0;
}
if (pRes->code != TSDB_CODE_QUERY_CANCELLED) {
assert(rpcMsg->msgType == pCmd->msgType + 1);
pRes->code = rpcMsg->code;
@ -565,7 +561,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
tscSetDnodeIpList(pSql, pTableMeta);
tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes));
tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d numberOfIP:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes), pSql->ipList.numOfIps);
return TSDB_CODE_SUCCESS;
}
@ -660,11 +656,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index];
pSql->ipList.numOfIps = pVgroupInfo->numOfIps; // todo fix me
pSql->ipList.port = tsDnodeShellPort;
pSql->ipList.inUse = 0;
for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) {
pSql->ipList.ip[i] = pVgroupInfo->ipAddr[i].ip;
strcpy(pSql->ipList.fqdn[i], pVgroupInfo->ipAddr[i].fqdn);
pSql->ipList.port[i] = pVgroupInfo->ipAddr[i].port;
}
tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index);
@ -931,7 +927,8 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload;
strncpy(pCreate->ip, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE;
return TSDB_CODE_SUCCESS;
@ -1074,7 +1071,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pDrop->ip, pTableMetaInfo->name);
strcpy(pDrop->ep, pTableMetaInfo->name);
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
return TSDB_CODE_SUCCESS;
@ -1319,11 +1316,11 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name);
pAlterTableMsg->type = htons(pAlterInfo->type);
pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo));
pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo);
memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN);
SSchema *pSchema = pAlterTableMsg->schema;
for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) {
for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
pSchema->type = pField->type;
@ -1348,11 +1345,6 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pCmd->payloadLen = sizeof(SCMAlterDbMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB;
if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) {
tscError("%p failed to malloc for query msg", pSql);
return TSDB_CODE_CLI_OUT_OF_MEMORY;
}
SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pAlterDbMsg->db, pTableMetaInfo->name);
@ -1585,32 +1577,32 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return pCmd->payloadLen;
}
static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) {
const int32_t defaultSize =
minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
int32_t n = 0;
size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond);
for (int32_t i = 0; i < size; ++i) {
assert(0);
// n += strlen(pQueryInfo->tagCond.cond[i].cond);
}
int32_t tagLen = n * TSDB_NCHAR_SIZE;
if (pQueryInfo->tagCond.tbnameCond.cond != NULL) {
tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE;
}
int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2;
int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables;
int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex);
int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize;
return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE);
}
//static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) {
//// const int32_t defaultSize =
//// minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS;
//// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
////
//// int32_t n = 0;
//// size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond);
//// for (int32_t i = 0; i < size; ++i) {
//// assert(0);
////// n += strlen(pQueryInfo->tagCond.cond[i].cond);
//// }
////
//// int32_t tagLen = n * TSDB_NCHAR_SIZE;
//// if (pQueryInfo->tagCond.tbnameCond.cond != NULL) {
//// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE;
//// }
////
//// int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2;
//// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables;
////
//// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex);
////
//// int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize;
////
//// return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE);
//}
int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
@ -1758,14 +1750,22 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
#endif
SSqlCmd *pCmd = &pSql->cmd;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload;
strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId));
char* pMsg = pCmd->payload;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg;
pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables);
pMsg += sizeof(SCMSTableVgroupMsg);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
strncpy(pMsg, pTableMetaInfo->name, TSDB_TABLE_ID_LEN);
pMsg += TSDB_TABLE_ID_LEN;
}
pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP;
pCmd->payloadLen = sizeof(SCMSTableVgroupMsg);
pCmd->payloadLen = (pMsg - pCmd->payload);
return TSDB_CODE_SUCCESS;
}
@ -1856,10 +1856,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
}
for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) {
pMetaMsg->vgroup.ipAddr[i].ip = htonl(pMetaMsg->vgroup.ipAddr[i].ip);
pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port);
assert(pMetaMsg->vgroup.ipAddr[i].ip != 0);
}
SSchema* pSchema = pMetaMsg->schema;
@ -2123,28 +2120,37 @@ _error_clean:
#endif
SSqlRes* pRes = &pSql->res;
// NOTE: the order of several table must be preserved.
SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp;
pStableVgroup->numOfVgroups = htonl(pStableVgroup->numOfVgroups);
pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables);
char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg);
// master sqlObj locates in param
SSqlObj* parent = pSql->param;
assert(parent != NULL);
SSqlCmd* pCmd = &parent->cmd;
STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
pInfo->vgroupList = malloc(pRes->rspLen);
memcpy(pInfo->vgroupList, pStableVgroup, pRes->rspLen);
for(int32_t i = 0; i < pInfo->vgroupList->numOfVgroups; ++i) {
SCMVgroupInfo* pVgroups = &pInfo->vgroupList->vgroups[i];
pVgroups->vgId = htonl(pVgroups->vgId);
assert(pVgroups->numOfIps >= 1);
for(int32_t j = 0; j < pVgroups->numOfIps; ++j) {
pVgroups->ipAddr[j].ip = htonl(pVgroups->ipAddr[j].ip);
pVgroups->ipAddr[j].port = htons(pVgroups->ipAddr[j].port);
for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) {
STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
SVgroupsInfo * pVgroupInfo = (SVgroupsInfo *)pMsg;
pVgroupInfo->numOfVgroups = htonl(pVgroupInfo->numOfVgroups);
size_t size = sizeof(SCMVgroupInfo) * pVgroupInfo->numOfVgroups + sizeof(SVgroupsInfo);
pInfo->vgroupList = calloc(1, size);
assert(pInfo->vgroupList != NULL);
memcpy(pInfo->vgroupList, pVgroupInfo, size);
for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
pVgroups->vgId = htonl(pVgroups->vgId);
assert(pVgroups->numOfIps >= 1);
for (int32_t k = 0; k < pVgroups->numOfIps; ++k) {
pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port);
}
pMsg += size;
}
}
@ -2277,7 +2283,6 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
if (pTableMetaInfo->pTableMeta) {
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true);
}
return 0;
@ -2503,42 +2508,27 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) {
return code;
}
static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
if (pTableMetaInfo->vgroupList == NULL) {
return false;
}
}
// all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore
return true;
}
int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
int code = TSDB_CODE_NETWORK_UNAVAIL;
SSqlCmd *pCmd = &pSql->cmd;
//the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache.
// bool required = false;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
if (pQueryInfo->pTableMetaInfo[0]->vgroupList != NULL) {
if (allVgroupInfoRetrieved(pCmd, clauseIndex)) {
return TSDB_CODE_SUCCESS;
}
#if 0
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0};
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid);
// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false);
SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr);
if (ppMeta == NULL) {
required = true;
break;
} else {
// pTableMetaInfo->pMetricMeta = ppMeta;
}
}
// all metricmeta for one clause are retrieved from cache, no need to retrieve metricmeta from management node
if (!required) {
return TSDB_CODE_SUCCESS;
}
#endif
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
@ -2550,11 +2540,11 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
return code;
}
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i);
STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name);
tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->tagColList);
STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i);
STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta);
tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList);
}
if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
@ -2562,25 +2552,9 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
return code;
}
tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond);
pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr;
pNewQueryInfo->numOfTables = pQueryInfo->numOfTables;
tscTrace("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables);
pNewQueryInfo->slimit = pQueryInfo->slimit;
pNewQueryInfo->order = pQueryInfo->order;
STagCond* pTagCond = &pNewQueryInfo->tagCond;
tscTrace("%p new sqlobj:%p info, numOfTables:%d, slimit:%" PRId64 ", soffset:%" PRId64 ", order:%d, tbname cond:%s",
pSql, pNew, pNewQueryInfo->numOfTables, pNewQueryInfo->slimit.limit, pNewQueryInfo->slimit.offset,
pNewQueryInfo->order.order, pTagCond->tbnameCond.cond)
// if (pSql->fp != NULL && pSql->pStream == NULL) {
// pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
// tscFreeQueryInfo(pCmd);
// }
tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew);
pNew->fp = tscTableMetaCallBack;
pNew->param = pSql;
code = tscProcessSql(pNew);

30
src/client/src/tscSql.c
View File

@ -72,24 +72,24 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
return NULL;
}
tscMgmtIpSet.numOfIps = 0;
if (ip && ip[0]) {
tscMgmtIpSet.inUse = 0;
tscMgmtIpSet.port = tsMnodeShellPort;
tscMgmtIpSet.numOfIps = 1;
tscMgmtIpSet.ip[0] = inet_addr(ip);
if (tsMasterIp[0] && strcmp(ip, tsMasterIp) != 0) {
tscMgmtIpSet.numOfIps = 2;
tscMgmtIpSet.ip[1] = inet_addr(tsMasterIp);
strcpy(tscMgmtIpSet.fqdn[0], ip);
tscMgmtIpSet.port[0] = port? port: tsMnodeShellPort;
} else {
if (tsMaster[0] != 0) {
taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
tscMgmtIpSet.numOfIps++;
}
if (tsSecondIp[0] && strcmp(tsSecondIp, tsMasterIp) != 0) {
tscMgmtIpSet.numOfIps = 3;
tscMgmtIpSet.ip[2] = inet_addr(tsSecondIp);
if (tsSecond[0] != 0) {
taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
tscMgmtIpSet.numOfIps++;
}
}
tscMgmtIpSet.port = port ? port : tsMnodeShellPort;
STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
if (NULL == pObj) {
@ -131,7 +131,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
pSql->pTscObj = pObj;
pSql->signature = pSql;
pSql->maxRetry = TSDB_REPLICA_MAX_NUM;
pSql->maxRetry = TSDB_MAX_REPLICA_NUM;
tsem_init(&pSql->rspSem, 0, 0);
@ -167,10 +167,6 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) {
}
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
if (ip == NULL || (ip != NULL && (strcmp("127.0.0.1", ip) == 0 || strcasecmp("localhost", ip) == 0))) {
ip = tsMasterIp;
}
tscTrace("try to create a connection to %s", ip);
STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL);
@ -716,7 +712,7 @@ char *taos_errstr(TAOS *taos) {
void taos_config(int debug, char *log_path) {
uDebugFlag = debug;
strcpy(logDir, log_path);
strcpy(tsLogDir, log_path);
}
char *taos_get_server_info(TAOS *taos) {

8
src/client/src/tscSub.c
View File

@ -220,7 +220,7 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
static int tscLoadSubscriptionProgress(SSub* pSub) {
char buf[TSDB_MAX_SQL_LEN];
sprintf(buf, "%s/subscribe/%s", dataDir, pSub->topic);
sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(buf, "r");
if (fp == NULL) {
@ -281,12 +281,12 @@ void tscSaveSubscriptionProgress(void* sub) {
SSub* pSub = (SSub*)sub;
char path[256];
sprintf(path, "%s/subscribe", dataDir);
sprintf(path, "%s/subscribe", tsDataDir);
if (access(path, 0) != 0) {
mkdir(path, 0777);
}
sprintf(path, "%s/subscribe/%s", dataDir, pSub->topic);
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(path, "w+");
if (fp == NULL) {
tscError("failed to create progress file for subscription: %s", pSub->topic);
@ -416,7 +416,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
tscSaveSubscriptionProgress(pSub);
} else {
char path[256];
sprintf(path, "%s/subscribe/%s", dataDir, pSub->topic);
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
remove(path);
}

8
src/client/src/tscSubquery.c
View File

@ -928,7 +928,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pQueryInfo->numOfTables;
tscTrace("%p start launched subquery, total:%d", pSql, pQueryInfo->numOfTables);
tscTrace("%p start launch subquery, total:%d", pSql, pQueryInfo->numOfTables);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
SJoinSubquerySupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i);
@ -1231,7 +1231,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
// data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].ip, pTableMetaInfo->vgroupList->vgroups[0].vgId,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
numOfRowsFromSubquery, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
@ -1459,12 +1459,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
if (pState->code != TSDB_CODE_SUCCESS) { // at least one peer subquery failed, abort current query
tscTrace("%p sub:%p query failed,ip:%u,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql,
pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex, pState->code);
pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex, pState->code);
tscHandleSubqueryError(param, tres, pState->code);
} else { // success, proceed to retrieve data from dnode
tscTrace("%p sub:%p query complete, ip:%u, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSqlObj, pSql,
pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex);
pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex);
if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode
tscRetrieveFromDnodeCallBack(param, pSql, 0);

23
src/client/src/tscSystem.c
View File

@ -55,7 +55,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) {
if (pVnodeConn == NULL) {
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsLocalIp;
rpcInit.localPort = 0;
rpcInit.label = "TSC-vnode";
rpcInit.numOfThreads = tscNumOfThreads;
@ -76,7 +75,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) {
if (*pMgmtConn == NULL) {
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsLocalIp;
rpcInit.localPort = 0;
rpcInit.label = "TSC-mgmt";
rpcInit.numOfThreads = 1;
@ -109,23 +107,17 @@ void taos_init_imp() {
deltaToUtcInitOnce();
if (tscEmbedded == 0) {
/*
* set localIp = 0
* means unset tsLocalIp in client
* except read from config file
*/
strcpy(tsLocalIp, "0.0.0.0");
// Read global configuration.
taosInitGlobalCfg();
taosReadGlobalLogCfg();
// For log directory
if (stat(logDir, &dirstat) < 0) mkdir(logDir, 0755);
if (stat(tsLogDir, &dirstat) < 0) mkdir(tsLogDir, 0755);
sprintf(temp, "%s/taoslog", logDir);
sprintf(temp, "%s/taoslog", tsLogDir);
if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) {
printf("failed to open log file in directory:%s\n", logDir);
printf("failed to open log file in directory:%s\n", tsLogDir);
}
taosReadGlobalCfg();
@ -133,7 +125,7 @@ void taos_init_imp() {
taosPrintGlobalCfg();
tscTrace("starting to initialize TAOS client ...");
tscTrace("Local IP address is:%s", tsLocalIp);
tscTrace("Local End Point is:%s", tsLocalEp);
}
taosSetCoreDump();
@ -143,13 +135,12 @@ void taos_init_imp() {
}
tscMgmtIpSet.inUse = 0;
tscMgmtIpSet.port = tsMnodeShellPort;
tscMgmtIpSet.numOfIps = 1;
tscMgmtIpSet.ip[0] = inet_addr(tsMasterIp);
taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
if (tsSecondIp[0] && strcmp(tsSecondIp, tsMasterIp) != 0) {
if (tsSecond[0] && strcmp(tsSecond, tsMaster) != 0) {
tscMgmtIpSet.numOfIps = 2;
tscMgmtIpSet.ip[1] = inet_addr(tsSecondIp);
taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]);
}
tscInitMsgsFp();

8
src/client/src/tscUtil.c
View File

@ -47,18 +47,18 @@ SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
return NULL;
}
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) {
if (tbufTell(pBuf) == 0) {
void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
if (tbufTell(bw) == 0) {
return;
}
SCond cond = {
.uid = uid,
.len = tbufTell(pBuf),
.len = tbufTell(bw),
.cond = NULL,
};
cond.cond = tbufGetData(pBuf, true);
cond.cond = tbufGetData(bw, true);
if (pTagCond->pCond == NULL) {
pTagCond->pCond = taosArrayInit(3, sizeof(SCond));

25
src/common/inc/tglobal.h
View File

@ -24,10 +24,10 @@ extern char configDir[];
extern char tsVnodeDir[];
extern char tsDnodeDir[];
extern char tsMnodeDir[];
extern char dataDir[];
extern char logDir[];
extern char scriptDir[];
extern char osName[];
extern char tsDataDir[];
extern char tsLogDir[];
extern char tsScriptDir[];
extern char tsOsName[];
// system info
extern int64_t tsPageSize;
@ -51,8 +51,10 @@ extern int32_t tsVersion;
extern int32_t tscEmbedded;
extern int64_t tsMsPerDay[2];
extern char tsMasterIp[];
extern char tsSecondIp[];
extern char tsMaster[];
extern char tsSecond[];
extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern uint16_t tsMnodeDnodePort;
extern uint16_t tsMnodeShellPort;
extern uint16_t tsDnodeShellPort;
@ -74,11 +76,13 @@ extern int16_t tsNumOfVnodesPerCore;
extern int16_t tsNumOfTotalVnodes;
extern uint32_t tsPublicIpInt;
extern int32_t tsMaxCacheSize;
extern int32_t tsSessionsPerVnode;
extern int32_t tsCacheBlockSize;
extern int32_t tsTotalBlocks;
extern int32_t tsTablesPerVnode;
extern int16_t tsDaysPerFile;
extern int32_t tsDaysToKeep;
extern int32_t tsRowsInFileBlock;
extern int32_t tsMinRowsInFileBlock;
extern int32_t tsMaxRowsInFileBlock;
extern int16_t tsCommitTime; // seconds
extern int32_t tsTimePrecision;
extern int16_t tsCompression;
@ -176,7 +180,8 @@ void taosInitGlobalCfg();
bool taosCheckGlobalCfg();
void taosSetAllDebugFlag();
bool taosCfgDynamicOptions(char *msg);
int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port);
#ifdef __cplusplus
}
#endif

353
src/common/src/tglobal.c
View File

@ -30,9 +30,9 @@ char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
char dataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
char scriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
char osName[10] = "Linux";
char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsOsName[10] = "Linux";
// system info, not configurable
int64_t tsPageSize;
@ -60,8 +60,10 @@ int32_t tscEmbedded = 0;
*/
int64_t tsMsPerDay[] = {86400000L, 86400000000L};
char tsMasterIp[TSDB_IPv4ADDR_LEN] = {0};
char tsSecondIp[TSDB_IPv4ADDR_LEN] = {0};
char tsMaster[TSDB_FQDN_LEN] = {0};
char tsSecond[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_FQDN_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030;
uint16_t tsMnodeShellPort = 6030; // udp[6030-6034] tcp[6030]
uint16_t tsDnodeShellPort = 6035; // udp[6035-6039] tcp[6035]
uint16_t tsMnodeDnodePort = 6040; // udp/tcp
@ -70,8 +72,6 @@ uint16_t tsSyncPort = 6050;
int32_t tsStatusInterval = 1; // second
int32_t tsShellActivityTimer = 3; // second
int32_t tsVnodePeerHBTimer = 1; // second
int32_t tsMgmtPeerHBTimer = 1; // second
int32_t tsMeterMetaKeepTimer = 7200; // second
int32_t tsMetricMetaKeepTimer = 600; // second
int32_t tsRpcTimer = 300;
@ -79,26 +79,26 @@ int32_t tsRpcMaxTime = 600; // seconds;
float tsNumOfThreadsPerCore = 1.0;
float tsRatioOfQueryThreads = 0.5;
char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0};
char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0};
int16_t tsNumOfVnodesPerCore = 8;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
#ifdef _TD_ARM_32_
int32_t tsSessionsPerVnode = 100;
int32_t tsTablesPerVnode = 100;
#else
int32_t tsSessionsPerVnode = 1000;
int32_t tsTablesPerVnode = TSDB_DEFAULT_TABLES;
#endif
int32_t tsMaxCacheSize = 64; //64M
int16_t tsDaysPerFile = 10;
int32_t tsDaysToKeep = 3650;
int32_t tsRowsInFileBlock = 4096;
int16_t tsCommitTime = 3600; // seconds
int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI;
int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL;
int16_t tsCommitLog = 1;
int32_t tsReplications = TSDB_REPLICA_MIN_NUM;
int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
int32_t tsTotalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP;
int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds
int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int16_t tsCommitLog = TSDB_DEFAULT_CLOG_LEVEL;
int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM;
/**
* Change the meaning of affected rows:
@ -110,7 +110,6 @@ int32_t tsNumOfMPeers = 3;
int32_t tsMaxShellConns = 2000;
int32_t tsMaxTables = 100000;
char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0};
char tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
char tsDefaultUser[64] = "root";
char tsDefaultPass[64] = "taosdata";
@ -181,7 +180,6 @@ float tsStreamComputDelayRatio = 0.1;
int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once
int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
char tsHttpIp[TSDB_IPv4ADDR_LEN] = "0.0.0.0";
uint16_t tsHttpPort = 6020; // only tcp, range tcp[6020]
// uint16_t tsNginxPort = 6060; //only tcp, range tcp[6060]
int32_t tsHttpCacheSessions = 100;
@ -192,7 +190,6 @@ int32_t tsHttpEnableRecordSql = 0;
int32_t tsTelegrafUseFieldNum = 0;
int32_t tsTscEnableRecordSql = 0;
int32_t tsAnyIp = 1;
uint32_t tsPublicIpInt = 0;
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
@ -272,69 +269,29 @@ static void doInitGlobalConfig() {
SGlobalCfg cfg = {0};
// ip address
cfg.option = "masterIp";
cfg.ptr = tsMasterIp;
cfg.option = "master";
cfg.ptr = tsMaster;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "secondIp";
cfg.ptr = tsSecondIp;
cfg.option = "second";
cfg.ptr = tsSecond;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "publicIp";
cfg.ptr = tsPublicIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "privateIp";
cfg.ptr = tsPrivateIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "localIp";
cfg.ptr = tsLocalIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "httpIp";
cfg.ptr = tsHttpIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// port
cfg.option = "mnodeShellPort";
cfg.ptr = &tsMnodeShellPort;
cfg.option = "serverPort";
cfg.ptr = &tsServerPort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
@ -343,56 +300,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "dnodeShellPort";
cfg.ptr = &tsDnodeShellPort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mnodeDnodePort";
cfg.ptr = &tsMnodeDnodePort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "dnodeMnodePort";
cfg.ptr = &tsDnodeMnodePort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// cfg.option = "syncPort";
// cfg.ptr = &syncPort;
// cfg.valType = TAOS_CFG_VTYPE_INT16;
// cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
// cfg.minValue = 1;
// cfg.maxValue = 65535;
// cfg.ptrLength = 0;
// cfg.unitType = TAOS_CFG_UTYPE_NONE;
// taosInitConfigOption(cfg);
cfg.option = "httpPort";
cfg.ptr = &tsHttpPort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// directory
cfg.option = "configDir";
cfg.ptr = configDir;
@ -405,7 +312,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
cfg.option = "logDir";
cfg.ptr = logDir;
cfg.ptr = tsLogDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
@ -415,7 +322,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
cfg.option = "scriptDir";
cfg.ptr = scriptDir;
cfg.ptr = tsScriptDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
@ -425,7 +332,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
cfg.option = "dataDir";
cfg.ptr = dataDir;
cfg.ptr = tsDataDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
@ -567,16 +474,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosInitConfigOption(cfg);
cfg.option = "ctime";
cfg.ptr = &tsCommitTime;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 30;
cfg.maxValue = 40960;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosInitConfigOption(cfg);
cfg.option = "statusInterval";
cfg.ptr = &tsStatusInterval;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -678,32 +575,42 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
// database configs
cfg.option = "clog";
cfg.ptr = &tsCommitLog;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.option = "tables";
cfg.ptr = &tsTablesPerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 2;
cfg.minValue = TSDB_MIN_TABLES;
cfg.maxValue = TSDB_MAX_TABLES;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "comp";
cfg.ptr = &tsCompression;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.option = "cache";
cfg.ptr = &tsCacheBlockSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 2;
cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE;
cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosInitConfigOption(cfg);
cfg.option = "blocks";
cfg.ptr = &tsTotalBlocks;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TOTAL_BLOCKS;
cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosInitConfigOption(cfg);
cfg.option = "days";
cfg.ptr = &tsDaysPerFile;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 365;
cfg.minValue = TSDB_MIN_DAYS_PER_FILE;
cfg.maxValue = TSDB_MAX_DAYS_PER_FILE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -712,8 +619,58 @@ static void doInitGlobalConfig() {
cfg.ptr = &tsDaysToKeep;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 365000;
cfg.minValue = TSDB_MIN_KEEP;
cfg.maxValue = TSDB_MAX_KEEP;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "minRows";
cfg.ptr = &tsMinRowsInFileBlock;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK;
cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "maxRows";
cfg.ptr = &tsMaxRowsInFileBlock;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK;
cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "ctime";
cfg.ptr = &tsCommitTime;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_COMMIT_TIME;
cfg.maxValue = TSDB_MAX_COMMIT_TIME;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosInitConfigOption(cfg);
cfg.option = "comp";
cfg.ptr = &tsCompression;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_COMP_LEVEL;
cfg.maxValue = TSDB_MAX_COMP_LEVEL;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "clog";
cfg.ptr = &tsCommitLog;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_CLOG_LEVEL;
cfg.maxValue = TSDB_MAX_CLOG_LEVEL;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -722,38 +679,8 @@ static void doInitGlobalConfig() {
cfg.ptr = &tsReplications;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 3;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "tables";
cfg.ptr = &tsSessionsPerVnode;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_TABLES_PER_VNODE;
cfg.maxValue = TSDB_MAX_TABLES_PER_VNODE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "cache";
cfg.ptr = &tsMaxCacheSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 100000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosInitConfigOption(cfg);
cfg.option = "rows";
cfg.ptr = &tsRowsInFileBlock;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 200;
cfg.maxValue = 1048576;
cfg.minValue = TSDB_MIN_REPLICA_NUM;
cfg.maxValue = TSDB_MAX_REPLICA_NUM;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
@ -1205,16 +1132,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "anyIp";
cfg.ptr = &tsAnyIp;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// version info
cfg.option = "gitinfo";
cfg.ptr = gitinfo;
@ -1262,25 +1179,15 @@ void taosInitGlobalCfg() {
}
bool taosCheckGlobalCfg() {
if (tsPrivateIp[0] == 0) {
taosGetPrivateIp(tsPrivateIp);
taosGetFqdn(tsLocalEp);
sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
if (tsMaster[0] == 0) {
strcpy(tsMaster, tsLocalEp);
}
if (tsPublicIp[0] == 0) {
strcpy(tsPublicIp, tsPrivateIp);
}
tsPublicIpInt = inet_addr(tsPublicIp);
if (tsLocalIp[0] == 0) {
strcpy(tsLocalIp, tsPrivateIp);
}
if (tsMasterIp[0] == 0) {
strcpy(tsMasterIp, tsPrivateIp);
}
if (tsSecondIp[0] == 0) {
strcpy(tsSecondIp, tsMasterIp);
if (tsSecond[0] == 0) {
strcpy(tsSecond, tsLocalEp);
}
taosGetSystemInfo();
@ -1302,15 +1209,6 @@ bool taosCheckGlobalCfg() {
tsNumOfTotalVnodes = tsNumOfTotalVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : tsNumOfTotalVnodes;
}
if (strlen(tsPrivateIp) == 0) {
uError("privateIp is null");
return false;
}
if (tscEmbedded) {
strcpy(tsLocalIp, tsPrivateIp);
}
// todo refactor
tsVersion = 0;
for (int i = 0; i < 10; i++) {
@ -1323,5 +1221,26 @@ bool taosCheckGlobalCfg() {
tsVersion = 10 * tsVersion;
tsMnodeShellPort = tsServerPort + TSDB_PORT_MNODESHELL; // udp[6030-6034] tcp[6030]
tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035]
tsMnodeDnodePort = tsServerPort + TSDB_PORT_MNODEDNODE; // udp/tcp
tsDnodeMnodePort = tsServerPort + TSDB_PORT_DNODEMNODE; // udp/tcp
tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
return true;
}
int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) {
*port = 0;
strcpy(fqdn, ep);
char *temp = strchr(fqdn, ':');
if (temp) {
*temp = 0;
*port = atoi(temp+1);
}
if (*port == 0) *port = tsServerPort;
return 0;
}

79
src/dnode/src/dnodeMClient.c
View File

@ -59,16 +59,13 @@ void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
tsMnodeIpSet = *pIpSet;
}
void dnodeGetMnodeIpSet(void *ipSetRaw, bool usePublicIp) {
void dnodeGetMnodeDnodeIpSet(void *ipSetRaw) {
SRpcIpSet *ipSet = ipSetRaw;
ipSet->numOfIps = tsMnodeInfos.nodeNum;
ipSet->inUse = tsMnodeInfos.inUse;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) {
if (usePublicIp) {
ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
} else {
ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
}
taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, ipSet->fqdn[i], &ipSet->port[i]);
ipSet->port[i] += TSDB_PORT_MNODEDNODE;
}
}
@ -85,26 +82,23 @@ int32_t dnodeInitMClient() {
if (!dnodeReadMnodeInfos()) {
memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
tsMnodeIpSet.port = tsMnodeDnodePort;
tsMnodeIpSet.numOfIps = 1;
tsMnodeIpSet.ip[0] = inet_addr(tsMasterIp);
if (strcmp(tsSecondIp, tsMasterIp) != 0) {
taosGetFqdnPortFromEp(tsMaster, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
tsMnodeIpSet.port[0] += TSDB_PORT_MNODEDNODE;
if (strcmp(tsSecond, tsMaster) != 0) {
tsMnodeIpSet.numOfIps = 2;
tsMnodeIpSet.ip[1] = inet_addr(tsSecondIp);
taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
}
} else {
tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
tsMnodeIpSet.port = tsMnodeInfos.nodeInfos[0].nodePort;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
tsMnodeIpSet.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
}
}
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = 0;
rpcInit.label = "DND-MC";
rpcInit.numOfThreads = 1;
rpcInit.cfp = dnodeProcessRspFromMnode;
@ -182,9 +176,6 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId);
pMnodeInfo->nodeIp = htonl(pMnodeInfo->nodeIp);
pMnodeInfo->nodePort = htons(pMnodeInfo->nodePort);
pMnodeInfo->syncPort = htons(pMnodeInfo->syncPort);
}
SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess;
@ -207,15 +198,14 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
tsMnodeIpSet.port = tsMnodeInfos.nodeInfos[0].nodePort;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
tsMnodeIpSet.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
}
dPrint("mnodes is changed, nodeNum:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp),
tsMnodeInfos.nodeInfos[i].nodePort, tsMnodeInfos.nodeInfos[i].nodeName);
dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
}
dnodeSaveMnodeInfos();
@ -291,42 +281,19 @@ static bool dnodeReadMnodeInfos() {
}
tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp");
if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeIp not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring);
cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort");
if (!nodePort || nodePort->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodePort not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint;
cJSON *syncPort = cJSON_GetObjectItem(nodeInfo, "syncPort");
if (!syncPort || syncPort->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, syncPort not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].syncPort = (uint16_t)syncPort->valueint;
cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName");
if (!nodeName || nodeName->type != cJSON_String || nodeName->valuestring == NULL) {
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeName not found");
goto PARSE_OVER;
}
strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN);
}
strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN);
}
ret = true;
dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId,
taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), tsMnodeInfos.nodeInfos[i].nodePort,
tsMnodeInfos.nodeInfos[i].nodeName);
dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
}
PARSE_OVER:
@ -352,10 +319,7 @@ static void dnodeSaveMnodeInfos() {
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp));
len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort);
len += snprintf(content + len, maxLen - len, " \"syncPort\": %u,\n", tsMnodeInfos.nodeInfos[i].syncPort);
len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName);
len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeEp);
if (i < tsMnodeInfos.nodeNum -1) {
len += snprintf(content + len, maxLen - len, " },{\n");
} else {
@ -371,8 +335,8 @@ static void dnodeSaveMnodeInfos() {
dPrint("save mnode iplist successed");
}
uint32_t dnodeGetMnodeMasteIp() {
return tsMnodeIpSet.ip[tsMnodeIpSet.inUse];
char *dnodeGetMnodeMasterEp() {
return tsMnodeInfos.nodeInfos[tsMnodeIpSet.inUse].nodeEp;
}
void* dnodeGetMnodeInfos() {
@ -402,8 +366,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
//strcpy(pStatus->dnodeName, tsDnodeName);
pStatus->version = htonl(tsVersion);
pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId);
pStatus->privateIp = htonl(inet_addr(tsPrivateIp));
pStatus->publicIp = htonl(inet_addr(tsPublicIp));
strcpy(pStatus->dnodeEp, tsLocalEp);
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
@ -500,4 +463,4 @@ void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
int32_t dnodeGetDnodeId() {
return tsDnodeCfg.dnodeId;
}
}

23
src/dnode/src/dnodeMain.c
View File

@ -144,12 +144,12 @@ static int32_t dnodeInitSystem() {
signal(SIGPIPE, SIG_IGN);
struct stat dirstat;
if (stat(logDir, &dirstat) < 0) {
mkdir(logDir, 0755);
if (stat(tsLogDir, &dirstat) < 0) {
mkdir(tsLogDir, 0755);
}
char temp[TSDB_FILENAME_LEN];
sprintf(temp, "%s/taosdlog", logDir);
sprintf(temp, "%s/taosdlog", tsLogDir);
if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) {
printf("failed to init log file\n");
}
@ -161,8 +161,7 @@ static int32_t dnodeInitSystem() {
}
taosPrintGlobalCfg();
dPrint("Server IP address is:%s", tsPrivateIp);
dPrint("starting to initialize TDengine ...");
dPrint("start to initialize TDengine on %s", tsLocalEp);
if (dnodeInitStorage() != 0) return -1;
if (dnodeInitRead() != 0) return -1;
@ -218,13 +217,13 @@ static void dnodeCheckDataDirOpenned(char *dir) {
static int32_t dnodeInitStorage() {
struct stat dirstat;
if (stat(dataDir, &dirstat) < 0) {
mkdir(dataDir, 0755);
if (stat(tsDataDir, &dirstat) < 0) {
mkdir(tsDataDir, 0755);
}
sprintf(tsMnodeDir, "%s/mnode", dataDir);
sprintf(tsVnodeDir, "%s/vnode", dataDir);
sprintf(tsDnodeDir, "%s/dnode", dataDir);
sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
mkdir(tsVnodeDir, 0755);
mkdir(tsDnodeDir, 0755);
@ -237,5 +236,5 @@ static int32_t dnodeInitStorage() {
static void dnodeCleanupStorage() {}
bool dnodeIsFirstDeploy() {
return strcmp(tsMasterIp, tsPrivateIp) == 0;
}
return strcmp(tsMaster, tsLocalEp) == 0;
}

10
src/dnode/src/dnodeMgmt.c
View File

@ -128,20 +128,20 @@ static void dnodeCloseVnodes() {
static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
SMDCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->cfg.vgId = htonl(pCreate->cfg.vgId);
pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion);
pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables);
pCreate->cfg.maxCacheSize = htobe64(pCreate->cfg.maxCacheSize);
pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize);
pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks);
pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile);
pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1);
pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2);
pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep);
pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime);
pCreate->cfg.arbitratorIp = htonl(pCreate->cfg.arbitratorIp);
for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
pCreate->nodes[j].nodeIp = htonl(pCreate->nodes[j].nodeIp);
}
void *pVnode = vnodeAccquireVnode(pCreate->cfg.vgId);

1
src/dnode/src/dnodeMnode.c
View File

@ -38,7 +38,6 @@ int32_t dnodeInitMnode() {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = tsDnodeMnodePort;
rpcInit.label = "DND-MS";
rpcInit.numOfThreads = 1;

1
src/dnode/src/dnodeShell.c
View File

@ -47,7 +47,6 @@ int32_t dnodeInitShell() {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = tsDnodeShellPort;
rpcInit.label = "DND-shell";
rpcInit.numOfThreads = numOfThreads;

4
src/inc/dnode.h
View File

@ -42,8 +42,8 @@ void dnodeFreeRqueue(void *rqueue);
void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code);
bool dnodeIsFirstDeploy();
uint32_t dnodeGetMnodeMasteIp();
void dnodeGetMnodeIpSet(void *ipSet, bool usePublicIp);
char *dnodeGetMnodeMasterEp();
void dnodeGetMnodeDnodeIpSet(void *ipSet);
void * dnodeGetMnodeInfos();
int32_t dnodeGetDnodeId();

107
src/inc/taosdef.h
View File

@ -187,11 +187,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_CITY_LEN 20
#define TSDB_STATE_LEN 20
#define TSDB_COUNTRY_LEN 20
#define TSDB_VNODES_SUPPORT 6
#define TSDB_MGMT_SUPPORT 4
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 64
#define TSDB_FQDN_LEN 64
#define TSDB_IPv4ADDR_LEN 16
#define TSDB_FILENAME_LEN 128
#define TSDB_METER_VNODE_BITS 20
@ -218,47 +217,58 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DNODE_ROLE_MGMT 1
#define TSDB_DNODE_ROLE_VNODE 2
#define TSDB_MAX_MPEERS 5
#define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1)
#define TSDB_REPLICA_MIN_NUM 1
#define TSDB_REPLICA_MAX_NUM 3
#define TSDB_MAX_REPLICA 5
#define TSDB_TBNAME_COLUMN_INDEX (-1)
#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta
//default value == 10
#define TSDB_FILE_MIN_PARTITION_RANGE 1 //minimum partition range of vnode file in days
#define TSDB_FILE_MAX_PARTITION_RANGE 3650 //max partition range of vnode file in days
#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved.
#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years
#define TSDB_MIN_COMPRESSION_LEVEL 0
#define TSDB_MAX_COMPRESSION_LEVEL 2
#define TSDB_MIN_COMMIT_TIME_INTERVAL 30
#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960
#define TSDB_MIN_ROWS_IN_FILEBLOCK 200
#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000
#define TSDB_MIN_CACHE_BLOCK_SIZE 1
#define TSDB_MAX_CACHE_BLOCK_SIZE 1000000
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
#define TSDB_MIN_CACHE_BLOCKS 100
#define TSDB_MAX_CACHE_BLOCKS 409600
#define TSDB_MIN_TOTAL_BLOCKS 2
#define TSDB_MAX_TOTAL_BLOCKS 10000
#define TSDB_DEFAULT_TOTAL_BLOCKS 2
#define TSDB_MIN_AVG_BLOCKS 2
#define TSDB_MAX_AVG_BLOCKS 2048
#define TSDB_DEFAULT_AVG_BLOCKS 4
#define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 200000
#define TSDB_DEFAULT_TABLES 1000
/*
* There is a bug in function taosAllocateId.
* When "create database tables 1" is executed, the wrong sid is assigned, so the minimum value is set to 2.
*/
#define TSDB_MIN_TABLES_PER_VNODE 2
#define TSDB_MAX_TABLES_PER_VNODE 220000
#define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650
#define TSDB_DEFAULT_DAYS_PER_FILE 10
#define TSDB_MIN_KEEP 1 // data in db to be reserved.
#define TSDB_MAX_KEEP 365000 // data in db to be reserved.
#define TSDB_DEFAULT_KEEP 3650 // ten years
#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100
#define TSDB_MIN_MIN_ROW_FBLOCK 10
#define TSDB_MAX_MIN_ROW_FBLOCK 1000
#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096
#define TSDB_MIN_MAX_ROW_FBLOCK 200
#define TSDB_MAX_MAX_ROW_FBLOCK 10000
#define TSDB_MIN_COMMIT_TIME 30
#define TSDB_MAX_COMMIT_TIME 40960
#define TSDB_DEFAULT_COMMIT_TIME 3600
#define TSDB_MIN_PRECISION TSDB_PRECISION_MILLI
#define TSDB_MAX_PRECISION TSDB_PRECISION_NANO
#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI
#define TSDB_MIN_COMP_LEVEL 0
#define TSDB_MAX_COMP_LEVEL 2
#define TSDB_DEFAULT_COMP_LEVEL 2
#define TSDB_MIN_CLOG_LEVEL 0
#define TSDB_MAX_CLOG_LEVEL 2
#define TSDB_DEFAULT_CLOG_LEVEL 2
#define TSDB_MIN_REPLICA_NUM 1
#define TSDB_MAX_REPLICA_NUM 3
#define TSDB_DEFAULT_REPLICA_NUM 1
#define TSDB_MAX_JOIN_TABLE_NUM 5
#define TSDB_MAX_UNION_CLAUSE 5
@ -269,24 +279,25 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_RPC_THREADS 5
#define TSDB_QUERY_TYPE_NON_TYPE 0x00U // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01U // free qhandle at vnode
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
/*
* 1. ordinary sub query for select * from super_table
* 2. all sqlobj generated by createSubqueryObj with this flag
*/
#define TSDB_QUERY_TYPE_SUBQUERY 0x02U
#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04U // two-stage subquery for super table
#define TSDB_QUERY_TYPE_SUBQUERY 0x02u
#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table
#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08U // query ordinary table; below only apply to client side
#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10U // query on super table
#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20U // join query
#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40U // select *,columns... query
#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80U // join sub query at the second stage
#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side
#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table
#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query
#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query
#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage
#define TSDB_QUERY_TYPE_INSERT 0x100U // insert type
#define TSDB_QUERY_TYPE_IMPORT 0x200U // import data
#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type
#define TSDB_QUERY_TYPE_IMPORT 0x200u // import data
#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0)
#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type))
@ -308,6 +319,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_NORMAL_TABLES 1000
#define TSDB_MAX_CHILD_TABLES 100000
#define TSDB_PORT_MNODESHELL 0
#define TSDB_PORT_DNODESHELL 5
#define TSDB_PORT_DNODEMNODE 10
#define TSDB_PORT_MNODEDNODE 15
#define TSDB_PORT_SYNC 20
typedef enum {
TSDB_PRECISION_MILLI,
TSDB_PRECISION_MICRO,

7
src/inc/taoserror.h
View File

@ -97,6 +97,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 203, "invalid table n
TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 204, "no super table") // operation only available for super table
TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active table")
TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch")
TAOS_DEFINE_ERROR(TSDB_CODE_TAG_ALREAY_EXIST, 0, 207, "tag already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_TAG_NOT_EXIST, 0, 208, "tag not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_ALREAY_EXIST, 0, 209, "field already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_NOT_EXIST, 0, 210, "field not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_COL_NAME_TOO_LONG, 0, 211, "column name too long")
TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TAGS, 0, 211, "too many tags")
// dnode & mnode
TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 250, "no enough dnodes")

175
src/inc/taosmsg.h
View File

@ -187,7 +187,7 @@ extern char *taosMsg[];
#pragma pack(push, 1)
typedef struct {
uint32_t ip;
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
} SIpAddr;
@ -238,7 +238,7 @@ typedef struct {
typedef struct SSchema {
uint8_t type;
char name[TSDB_COL_NAME_LEN];
char name[TSDB_COL_NAME_LEN + 1];
int16_t colId;
int16_t bytes;
} SSchema;
@ -256,15 +256,16 @@ typedef struct {
uint64_t uid;
uint64_t superTableUid;
uint64_t createdTime;
char tableId[TSDB_TABLE_ID_LEN];
char superTableId[TSDB_TABLE_ID_LEN];
char tableId[TSDB_TABLE_ID_LEN + 1];
char superTableId[TSDB_TABLE_ID_LEN + 1];
char data[];
} SMDCreateTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN];
char db[TSDB_DB_NAME_LEN];
char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
int8_t igExists;
int8_t getMeta;
int16_t numOfTags;
int16_t numOfColumns;
int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string
@ -274,13 +275,13 @@ typedef struct {
} SCMCreateTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN];
char tableId[TSDB_TABLE_ID_LEN + 1];
int8_t igNotExists;
} SCMDropTableMsg;
typedef struct {
char tableId[TSDB_TABLE_ID_LEN];
char db[TSDB_DB_NAME_LEN];
char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
int16_t type; /* operation type */
char tagVal[TSDB_MAX_BYTES_PER_ROW];
int8_t numOfCols; /* number of schema */
@ -345,7 +346,6 @@ typedef struct {
} SMDDropTableMsg;
typedef struct {
int32_t contLen;
int32_t vgId;
int64_t uid;
char tableId[TSDB_TABLE_ID_LEN + 1];
@ -492,6 +492,7 @@ typedef struct SRetrieveTableRsp {
typedef struct {
int32_t vgId;
int32_t cfgVersion;
int64_t totalStorage;
int64_t compStorage;
int64_t pointsWritten;
@ -502,29 +503,23 @@ typedef struct {
} SVnodeLoad;
typedef struct {
char acct[TSDB_USER_LEN];
char db[TSDB_DB_NAME_LEN];
uint32_t vgId;
char acct[TSDB_USER_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
int32_t maxSessions;
int32_t cacheBlockSize;
union {
int32_t totalBlocks;
float fraction;
} cacheNumOfBlocks;
int32_t daysPerFile;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t daysToKeep;
int32_t commitTime;
int32_t rowsInFileBlock;
int16_t blocksPerTable;
int8_t compression;
int8_t commitLog;
int8_t replications;
int8_t repStrategy;
int8_t loadLatest; // load into mem or not
uint8_t precision; // time resolution
int8_t ignoreExist;
int32_t cacheBlockSize; //MB
int32_t totalBlocks;
int32_t daysPerFile;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t daysToKeep;
int32_t commitTime;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int8_t compression;
int8_t commitLog;
int8_t replications;
uint8_t precision; // time resolution
int8_t ignoreExist;
} SCMCreateDbMsg, SCMAlterDbMsg;
typedef struct {
@ -555,24 +550,19 @@ typedef struct {
typedef struct {
int32_t nodeId;
uint32_t nodeIp;
uint16_t nodePort;
uint16_t syncPort;
char nodeName[TSDB_NODE_NAME_LEN + 1];
char nodeEp[TSDB_FQDN_LEN];
} SDMMnodeInfo;
typedef struct {
int8_t inUse;
int8_t nodeNum;
SDMMnodeInfo nodeInfos[TSDB_MAX_MPEERS];
SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA];
} SDMMnodeInfos;
typedef struct {
uint32_t version;
int32_t dnodeId;
char dnodeName[TSDB_NODE_NAME_LEN + 1];
uint32_t privateIp;
uint32_t publicIp;
char dnodeEp[TSDB_FQDN_LEN];
uint32_t moduleStatus;
uint32_t lastReboot; // time stamp for last reboot
uint16_t numOfTotalVnodes; // from config file
@ -592,34 +582,34 @@ typedef struct {
typedef struct {
uint32_t vgId;
int32_t cfgVersion;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t maxTables;
int64_t maxCacheSize;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t daysPerFile;
int32_t daysToKeep;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t commitTime;
uint8_t precision; // time resolution
int8_t precision;
int8_t compression;
int8_t wals;
int8_t commitLog;
int8_t replications;
int8_t wals;
int8_t quorum;
uint32_t arbitratorIp;
int8_t reserved[16];
} SMDVnodeCfg;
typedef struct {
int32_t nodeId;
uint32_t nodeIp;
char nodeName[TSDB_NODE_NAME_LEN + 1];
char nodeEp[TSDB_FQDN_LEN];
} SMDVnodeDesc;
typedef struct {
SMDVnodeCfg cfg;
SMDVnodeDesc nodes[TSDB_MAX_MPEERS];
SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
} SMDCreateVnodeMsg;
typedef struct {
@ -634,48 +624,54 @@ typedef struct {
} SCMMultiTableInfoMsg;
typedef struct SCMSTableVgroupMsg {
char tableId[TSDB_TABLE_ID_LEN];
} SCMSTableVgroupMsg;
int32_t numOfTables;
} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg;
typedef struct {
int32_t vgId;
int8_t numOfIps;
SIpAddr ipAddr[TSDB_REPLICA_MAX_NUM];
SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM];
} SCMVgroupInfo;
typedef struct {
int32_t numOfVgroups;
int32_t numOfVgroups;
SCMVgroupInfo vgroups[];
} SCMSTableVgroupRspMsg;
} SVgroupsInfo;
typedef struct {
int16_t elemLen;
//typedef struct {
// int32_t numOfTables;
//// int32_t numOfVgroups;
//// SCMVgroupInfo vgroups[];
//} SCMSTableVgroupRspMsg;
char tableId[TSDB_TABLE_ID_LEN + 1];
int16_t orderIndex;
int16_t orderType; // used in group by xx order by xxx
int16_t rel; // denotes the relation between condition and table list
int32_t tableCond; // offset value of table name condition
int32_t tableCondLen;
int32_t cond; // offset of column query condition
int32_t condLen;
int16_t tagCols[TSDB_MAX_TAGS + 1]; // required tag columns, plus one is for table name
int16_t numOfTags; // required number of tags
int16_t numOfGroupCols; // num of group by columns
int32_t groupbyTagColumnList;
} SSuperTableMetaElemMsg;
typedef struct {
int32_t numOfTables;
int32_t join;
int32_t joinCondLen; // for join condition
int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM];
} SSuperTableMetaMsg;
//typedef struct {
// int16_t elemLen;
//
// char tableId[TSDB_TABLE_ID_LEN + 1];
// int16_t orderIndex;
// int16_t orderType; // used in group by xx order by xxx
//
// int16_t rel; // denotes the relation between condition and table list
//
// int32_t tableCond; // offset value of table name condition
// int32_t tableCondLen;
//
// int32_t cond; // offset of column query condition
// int32_t condLen;
//
// int16_t tagCols[TSDB_MAX_TAGS + 1]; // required tag columns, plus one is for table name
// int16_t numOfTags; // required number of tags
//
// int16_t numOfGroupCols; // num of group by columns
// int32_t groupbyTagColumnList;
//} SSuperTableMetaElemMsg;
//
//typedef struct {
// int32_t numOfTables;
// int32_t join;
// int32_t joinCondLen; // for join condition
// int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM];
//} SSuperTableMetaMsg;
typedef struct {
int32_t nodeId;
@ -684,24 +680,17 @@ typedef struct {
} SVnodeDesc;
typedef struct {
SVnodeDesc vpeerDesc[TSDB_REPLICA_MAX_NUM];
SVnodeDesc vpeerDesc[TSDB_MAX_REPLICA_NUM];
int16_t index; // used locally
int32_t vgId;
int32_t numOfSids;
int32_t pSidExtInfoList[]; // offset value of STableIdInfo
} SVnodeSidList;
typedef struct {
int32_t numOfTables;
int32_t numOfVnodes;
uint16_t tagLen; /* tag value length */
int32_t list[]; /* offset of SVnodeSidList, compared to the SSuperTableMeta struct */
} SSuperTableMeta;
typedef struct STableMetaMsg {
int32_t contLen;
char tableId[TSDB_TABLE_ID_LEN]; // table id
char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table
char tableId[TSDB_TABLE_ID_LEN + 1]; // table id
char stableId[TSDB_TABLE_ID_LEN + 1]; // stable name if it is created according to super table
uint8_t numOfTags;
uint8_t precision;
uint8_t tableType;
@ -742,7 +731,7 @@ typedef struct SCMShowRsp {
} SCMShowRsp;
typedef struct {
char ip[32];
char ep[TSDB_FQDN_LEN]; // end point, hostname:port
} SCMCreateDnodeMsg, SCMDropDnodeMsg;
typedef struct {
@ -757,7 +746,7 @@ typedef struct {
} SDMConfigVnodeMsg;
typedef struct {
char ip[32];
char ep[TSDB_FQDN_LEN]; // end point, hostname:port
char config[64];
} SMDCfgDnodeMsg, SCMCfgDnodeMsg;

5
src/inc/trpc.h
View File

@ -31,8 +31,8 @@ extern int tsRpcHeadSize;
typedef struct {
int8_t inUse;
int8_t numOfIps;
uint16_t port;
uint32_t ip[TSDB_MAX_MPEERS];
uint16_t port[TSDB_MAX_REPLICA];
char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN];
} SRpcIpSet;
typedef struct {
@ -51,7 +51,6 @@ typedef struct {
} SRpcMsg;
typedef struct {
char *localIp; // local IP used
uint16_t localPort; // local port
char *label; // for debug purpose
int numOfThreads; // number of threads to handle connections

81
src/inc/tsdb.h
View File

@ -45,15 +45,19 @@ typedef struct {
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
typedef struct {
int8_t precision;
int8_t compression;
int32_t tsdbId;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t maxTables; // maximum number of tables this repository can have
int32_t daysPerFile; // day per file sharding policy
int32_t keep; // day of data to keep
int32_t keep1;
int32_t keep2;
int32_t minRowsPerFileBlock; // minimum rows per file block
int32_t maxRowsPerFileBlock; // maximum rows per file block
int32_t keep; // day of data to keep
int64_t maxCacheSize; // maximum cache size this TSDB can use
int32_t commitTime;
int8_t precision;
int8_t compression;
} STsdbCfg;
void tsdbSetDefaultCfg(STsdbCfg *pCfg);
@ -137,15 +141,10 @@ int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg);
typedef void *TsdbQueryHandleT; // Use void to hide implementation details
typedef struct STableGroupList { // qualified table object list in group
SArray *pGroupList;
int32_t numOfTables;
} STableGroupList;
// query condition to build vnode iterator
typedef struct STsdbQueryCond {
STimeWindow twindow;
int32_t order; // desc/asc order to iterate the data block
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo *colList;
} STsdbQueryCond;
@ -178,14 +177,29 @@ typedef void *TsdbPosT;
/**
* Get the data block iterator, starting from position according to the query condition
* @param pCond query condition, only includes the filter on primary time stamp
* @param pTableList table sid list
*
* @param tsdb tsdb handle
* @param pCond query condition, including time window, result set order, and basic required columns for each block
* @param groupInfo tableId list in the form of set, seperated into different groups according to group by condition
* @return
*/
TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
/**
* move to next block
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
* Note that only one data block with only row will be returned while invoking retrieve data block function for
* all tables in this group.
*
* @param tsdb tsdb handle
* @param pCond query condition, including time window, result set order, and basic required columns for each block
* @param groupInfo tableId list.
* @return
*/
TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
/**
* move to next block if exists
*
* @param pQueryHandle
* @return
*/
@ -212,26 +226,16 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle);
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis);
/**
*
* The query condition with primary timestamp is passed to iterator during its constructor function,
* the returned data block must be satisfied with the time window condition in any cases,
* which means the SData data block is not actually the completed disk data blocks.
*
* @param pQueryHandle
* @param pQueryHandle query handle
* @param pColumnIdList required data columns id list
* @return
*/
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pIdList);
/**
* todo remove the parameter of position, and order type
*
* Reset to the start(end) position of current query, from which the iterator starts.
*
* @param pQueryHandle
* @param position set the iterator traverses position
* @param order ascending order or descending order
* @return
*/
int32_t tsdbResetQuery(TsdbQueryHandleT *pQueryHandle, STimeWindow *window, TsdbPosT position, int16_t order);
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList);
/**
* todo remove this function later
@ -268,20 +272,19 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
* Get the qualified table id for a super table according to the tag query expression.
* @param stableid. super table sid
* @param pTagCond. tag query condition
*
*/
int32_t tsdbQuerySTableByTagCond(
TsdbRepoT *tsdb,
int64_t uid,
const char *pTagCond,
size_t len,
int16_t tagNameRelType,
const char* tbnameCond,
STableGroupInfo *pGroupList,
SColIndex *pColIndex,
int32_t numOfCols
);
int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols);
/**
* create the table group result including only one table, used to handle the normal table query
*
* @param tsdb tsdbHandle
* @param uid table uid
* @param pGroupInfo the generated result
* @return
*/
int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo);
/**

7
src/inc/tsync.h
View File

@ -38,14 +38,15 @@ typedef enum _TAOS_SYNC_STATUS {
typedef struct {
uint32_t nodeId; // node ID assigned by TDengine
uint32_t nodeIp; // node IP address
char name[TSDB_FILENAME_LEN]; // external node name
uint16_t nodePort; // node sync Port
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN
} SNodeInfo;
typedef struct {
uint32_t arbitratorIp; // arbitrator IP address
int8_t quorum; // number of confirms required, >=1
int8_t replica; // number of replications, >=1
uint16_t arbitratorPort; // arbitrator port
char arbitratorFqdn[TSDB_FQDN_LEN]; // arbitrator IP address
SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA];
} SSyncCfg;

2
src/kit/shell/src/shellEngine.c
View File

@ -40,7 +40,7 @@ History history;
*/
TAOS *shellInit(struct arguments *args) {
printf("\n");
printf(CLIENT_VERSION, osName, taos_get_client_info());
printf(CLIENT_VERSION, tsOsName, taos_get_client_info());
fflush(stdout);
// set options before initializing

21
src/mnode/inc/mgmtDef.h
View File

@ -31,13 +31,9 @@ struct SMnodeObj;
typedef struct SDnodeObj {
int32_t dnodeId;
uint32_t privateIp;
uint32_t publicIp;
uint16_t mnodeShellPort;
uint16_t mnodeDnodePort;
uint16_t dnodeShellPort;
uint16_t dnodeMnodePort;
uint16_t syncPort;
uint16_t dnodePort;
char dnodeFqdn[TSDB_FQDN_LEN];
char dnodeEp[TSDB_FQDN_LEN];
int64_t createdTime;
uint32_t lastAccess;
int32_t openVnodes;
@ -47,7 +43,6 @@ typedef struct SDnodeObj {
int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode
int8_t status; // set in balance function
int8_t isMgmt;
char dnodeName[TSDB_NODE_NAME_LEN + 1];
int8_t reserved[15];
int8_t updateEnd[1];
int32_t refCount;
@ -123,7 +118,7 @@ typedef struct SVgObj {
uint32_t vgId;
char dbName[TSDB_DB_NAME_LEN + 1];
int64_t createdTime;
SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT];
SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
int32_t numOfVnodes;
int32_t lbDnodeId;
int32_t lbTime;
@ -143,14 +138,15 @@ typedef struct SVgObj {
} SVgObj;
typedef struct {
int64_t maxCacheSize;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t maxTables;
int32_t daysPerFile;
int32_t daysToKeep;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t minRowsPerFileBlock; // minimum rows per file block
int32_t maxRowsPerFileBlock; // maximum rows per file block
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t commitTime;
int8_t precision;
int8_t compression;
@ -163,6 +159,7 @@ typedef struct SDbObj {
char name[TSDB_DB_NAME_LEN + 1];
char acct[TSDB_USER_LEN + 1];
int64_t createdTime;
int32_t cfgVersion;
SDbCfg cfg;
int8_t status;
int8_t reserved[14];

2
src/mnode/inc/mgmtDnode.h
View File

@ -38,7 +38,7 @@ void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
void mgmtIncDnodeRef(SDnodeObj *pDnode);
void mgmtDecDnodeRef(SDnodeObj *pDnode);
void * mgmtGetDnode(int32_t dnodeId);
void * mgmtGetDnodeByIp(uint32_t ip);
void * mgmtGetDnodeByIp(char *ep);
void mgmtUpdateDnode(SDnodeObj *pDnode);
int32_t mgmtDropDnode(SDnodeObj *pDnode);

2
src/mnode/inc/mgmtMnode.h
View File

@ -40,7 +40,7 @@ void * mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
void mgmtReleaseMnode(struct SMnodeObj *pMnode);
char * mgmtGetMnodeRoleStr();
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet, bool usePublicIp);
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
void mgmtGetMnodeInfos(void *mnodes);
#ifdef __cplusplus

2
src/mnode/inc/mgmtVgroup.h
View File

@ -50,7 +50,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle);
void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup);
SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip);
SRpcIpSet mgmtGetIpSetFromIp(char *ep);
#ifdef __cplusplus
}

6
src/mnode/src/mgmtBalance.c
View File

@ -17,6 +17,7 @@
#include "os.h"
#include "trpc.h"
#include "tbalance.h"
#include "tglobal.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtMnode.h"
@ -41,7 +42,10 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
if (pDnode == NULL) break;
if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) {
float usage = (float)pDnode->openVnodes / pDnode->totalVnodes;
float openVnodes = pDnode->openVnodes;
if (pDnode->isMgmt) openVnodes += tsMgmtEqualVnodeNum;
float usage = openVnodes / pDnode->totalVnodes;
if (usage <= vnodeUsage) {
pSelDnode = pDnode;
vnodeUsage = usage;

1
src/mnode/src/mgmtDClient.c
View File

@ -38,7 +38,6 @@ static void *tsMgmtDClientRpc = NULL;
int32_t mgmtInitDClient() {
SRpcInit rpcInit = {0};
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = 0;
rpcInit.label = "MND-DC";
rpcInit.numOfThreads = 1;

7
src/mnode/src/mgmtDServer.c
View File

@ -43,7 +43,6 @@ static void *tsMgmtDServerQhandle = NULL;
int32_t mgmtInitDServer() {
SRpcInit rpcInit = {0};
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;;
rpcInit.localPort = tsMnodeDnodePort;
rpcInit.label = "MND-DS";
rpcInit.numOfThreads = 1;
@ -105,14 +104,12 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
if (!sdbIsMaster()) {
SRpcConnInfo connInfo;
rpcGetConnInfo(rpcMsg->handle, &connInfo);
bool usePublicIp = false;
SRpcIpSet ipSet = {0};
ipSet.port = tsMnodeDnodePort;
dnodeGetMnodeIpSet(&ipSet, usePublicIp);
dnodeGetMnodeDnodeIpSet(&ipSet);
mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i]));
mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
}
rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
return;

230
src/mnode/src/mgmtDb.c
View File

@ -22,6 +22,7 @@
#include "tglobal.h"
#include "ttime.h"
#include "tname.h"
#include "tbalance.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
#include "mgmtAcct.h"
@ -177,78 +178,81 @@ SDbObj *mgmtGetDbByTableId(char *tableId) {
}
static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
if (pCfg->maxCacheSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_BLOCK_SIZE) {
mError("invalid db option maxCacheSize:%d valid range: [%d, %d]", pCfg->maxCacheSize, TSDB_MIN_CACHE_BLOCK_SIZE,
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) {
mError("invalid db option cacheBlockSize:%d valid range: [%d, %d]", pCfg->cacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE,
TSDB_MAX_CACHE_BLOCK_SIZE);
}
if (pCfg->maxTables < TSDB_MIN_TABLES_PER_VNODE || pCfg->maxTables > TSDB_MAX_TABLES_PER_VNODE) {
mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES_PER_VNODE,
TSDB_MAX_TABLES_PER_VNODE);
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) {
mError("invalid db option totalBlocks:%d valid range: [%d, %d]", pCfg->totalBlocks, TSDB_MIN_TOTAL_BLOCKS,
TSDB_MAX_TOTAL_BLOCKS);
}
if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) {
mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) {
mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE,
TSDB_FILE_MAX_PARTITION_RANGE);
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE,
TSDB_MAX_DAYS_PER_FILE);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysToKeep1 < pCfg->daysPerFile) {
mError("invalid db option daystokeep:%d", pCfg->daysToKeep);
if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) {
mError("invalid db option daysToKeep:%d", pCfg->daysToKeep);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->daysToKeep2 > pCfg->daysToKeep || pCfg->daysToKeep2 < pCfg->daysToKeep1) {
mError("invalid db option daystokeep1:%d, daystokeep2:%d, daystokeep:%d", pCfg->daysToKeep1,
pCfg->daysToKeep2, pCfg->daysToKeep);
if (pCfg->daysToKeep < pCfg->daysPerFile) {
mError("invalid db option daysToKeep:%d daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->minRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) {
if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock,
TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK);
TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->maxRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) {
if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock,
TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK);
TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->maxRowsPerFileBlock < pCfg->minRowsPerFileBlock) {
if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) {
mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock,
pCfg->maxRowsPerFileBlock);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCfg->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) {
mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL,
TSDB_MAX_COMMIT_TIME_INTERVAL);
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) {
mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME,
TSDB_MAX_COMMIT_TIME);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->precision != TSDB_TIME_PRECISION_MILLI && pCfg->precision != TSDB_TIME_PRECISION_MICRO) {
mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_TIME_PRECISION_MILLI,
TSDB_TIME_PRECISION_MICRO);
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) {
mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_MIN_PRECISION,
TSDB_MAX_PRECISION);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->compression < TSDB_MIN_COMPRESSION_LEVEL || pCfg->compression > TSDB_MAX_COMPRESSION_LEVEL) {
mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMPRESSION_LEVEL,
TSDB_MAX_COMPRESSION_LEVEL);
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) {
mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMP_LEVEL,
TSDB_MAX_COMP_LEVEL);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->commitLog < 0 || pCfg->commitLog > 2) {
if (pCfg->commitLog < TSDB_MIN_CLOG_LEVEL || pCfg->commitLog > TSDB_MAX_CLOG_LEVEL) {
mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog);
return TSDB_CODE_INVALID_OPTION;
}
if (pCfg->replications < TSDB_REPLICA_MIN_NUM || pCfg->replications > TSDB_REPLICA_MAX_NUM) {
mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_REPLICA_MIN_NUM,
TSDB_REPLICA_MAX_NUM);
if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) {
mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM,
TSDB_MAX_REPLICA_NUM);
return TSDB_CODE_INVALID_OPTION;
}
@ -256,14 +260,15 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
}
static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->maxCacheSize < 0) pCfg->maxCacheSize = tsMaxCacheSize;
if (pCfg->maxTables < 0) pCfg->maxTables = tsSessionsPerVnode;
if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = tsCacheBlockSize;
if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsTotalBlocks;
if (pCfg->maxTables < 0) pCfg->maxTables = tsTablesPerVnode;
if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile;
if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep;
if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep;
if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep;
if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsRowsInFileBlock;
if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = pCfg->minRowsPerFileBlock * 2;
if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock;
if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock;
if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime;
if (pCfg->precision < 0) pCfg->precision = tsTimePrecision;
if (pCfg->compression < 0) pCfg->compression = tsCompression;
@ -293,14 +298,15 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN);
pDb->createdTime = taosGetTimestampMs();
pDb->cfg = (SDbCfg) {
.maxCacheSize = 64,//(int64_t)pCreate->cacheBlockSize * pCreate->cacheNumOfBlocks.totalBlocks,
.cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->totalBlocks,
.maxTables = pCreate->maxSessions,
.daysPerFile = pCreate->daysPerFile,
.daysToKeep = pCreate->daysToKeep,
.daysToKeep1 = pCreate->daysToKeep1,
.daysToKeep2 = pCreate->daysToKeep2,
.minRowsPerFileBlock = pCreate->rowsInFileBlock * 1,
.maxRowsPerFileBlock = pCreate->rowsInFileBlock * 2,
.minRowsPerFileBlock = pCreate->maxRowsPerFileBlock,
.maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock,
.commitTime = pCreate->commitTime,
.precision = pCreate->precision,
.compression = pCreate->compression,
@ -459,13 +465,25 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "rows");
strcpy(pSchema[cols].name, "cache(MB)");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "blocks");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "minrows");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "cache(Mb)");
strcpy(pSchema[cols].name, "maxrows");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -492,7 +510,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
pShow->bytes[cols] = 3;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "time precision");
strcpy(pSchema[cols].name, "precision");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -583,14 +601,22 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
*(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.cacheBlockSize;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.totalBlocks;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.maxCacheSize;
*(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.commitTime;
cols++;
@ -664,14 +690,15 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
SCMCreateDbMsg *pCreate = pMsg->pCont;
pCreate->maxSessions = htonl(pCreate->maxSessions);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
pCreate->daysPerFile = htonl(pCreate->daysPerFile);
pCreate->daysToKeep = htonl(pCreate->daysToKeep);
pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2);
pCreate->commitTime = htonl(pCreate->commitTime);
pCreate->blocksPerTable = htons(pCreate->blocksPerTable);
pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock);
pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
int32_t code;
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_GRANT_EXPIRED;
@ -688,38 +715,70 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
}
static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
SDbCfg newCfg = pDb->cfg;
int32_t daysToKeep = htonl(pAlter->daysToKeep);
int32_t maxTables = htonl(pAlter->maxSessions);
SDbCfg newCfg = pDb->cfg;
int32_t cacheBlockSize = htonl(pAlter->daysToKeep);
int32_t totalBlocks = htonl(pAlter->totalBlocks);
int32_t maxTables = htonl(pAlter->maxSessions);
int32_t daysToKeep = htonl(pAlter->daysToKeep);
int32_t daysToKeep1 = htonl(pAlter->daysToKeep1);
int32_t daysToKeep2 = htonl(pAlter->daysToKeep2);
int8_t compression = pAlter->compression;
int8_t replications = pAlter->replications;
terrno = TSDB_CODE_SUCCESS;
if (cacheBlockSize > 0 && cacheBlockSize != pDb->cfg.cacheBlockSize) {
mTrace("db:%s, cache:%d change to %d", pDb->name, pDb->cfg.cacheBlockSize, cacheBlockSize);
newCfg.cacheBlockSize = cacheBlockSize;
}
if (totalBlocks > 0 && totalBlocks != pDb->cfg.totalBlocks) {
mTrace("db:%s, blocks:%d change to %d", pDb->name, pDb->cfg.totalBlocks, totalBlocks);
newCfg.totalBlocks = totalBlocks;
}
if (maxTables > 0 && maxTables != pDb->cfg.maxTables) {
mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables);
newCfg.maxTables = maxTables;
if (newCfg.maxTables < pDb->cfg.maxTables) {
mTrace("db:%s, tables:%d should larger than origin:%d", pDb->name, newCfg.maxTables, pDb->cfg.maxTables);
terrno = TSDB_CODE_INVALID_OPTION;
}
}
if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) {
mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep);
newCfg.daysToKeep = daysToKeep;
}
}
if (daysToKeep1 > 0 && daysToKeep1 != pDb->cfg.daysToKeep1) {
mTrace("db:%s, daysToKeep1:%d change to %d", pDb->name, pDb->cfg.daysToKeep1, daysToKeep1);
newCfg.daysToKeep1 = daysToKeep1;
}
if (daysToKeep2 > 0 && daysToKeep2 != pDb->cfg.daysToKeep2) {
mTrace("db:%s, daysToKeep2:%d change to %d", pDb->name, pDb->cfg.daysToKeep2, daysToKeep2);
newCfg.daysToKeep2 = daysToKeep2;
}
if (compression > 0 && compression != pDb->cfg.compression) {
mTrace("db:%s, compression:%d change to %d", pDb->name, pDb->cfg.compression, compression);
newCfg.compression = compression;
}
if (replications > 0 && replications != pDb->cfg.replications) {
mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications);
if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) {
mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM);
terrno = TSDB_CODE_INVALID_OPTION;
}
mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications);
newCfg.replications = replications;
}
if (maxTables > 0 && maxTables != pDb->cfg.maxTables) {
mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables);
if (maxTables < TSDB_MIN_TABLES_PER_VNODE || maxTables > TSDB_MAX_TABLES_PER_VNODE) {
mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE);
terrno = TSDB_CODE_INVALID_OPTION;
}
if (maxTables < pDb->cfg.maxTables) {
mError("invalid db option tables: %d should larger than original:%d", maxTables, pDb->cfg.maxTables);
terrno = TSDB_CODE_INVALID_OPTION;
}
newCfg.maxTables = maxTables;
if (replications > mgmtGetDnodesNum()) {
mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications);
terrno = TSDB_CODE_NO_ENOUGH_DNODES;
}
if (pDb->cfg.replications - replications >= 2) {
mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications);
terrno = TSDB_CODE_INVALID_OPTION;
}
return newCfg;
@ -731,8 +790,16 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
return terrno;
}
int32_t code = mgmtCheckDbCfg(&newCfg);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
int32_t oldReplica = pDb->cfg.replications;
if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
pDb->cfg = newCfg;
pDb->cfgVersion++;
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
@ -745,7 +812,20 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
return TSDB_CODE_SDB_ERROR;
}
}
void *pNode = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pNode = mgmtGetNextVgroup(pNode, &pVgroup);
if (pVgroup == NULL) break;
mgmtSendCreateVgroupMsg(pVgroup, NULL);
mgmtDecVgroupRef(pVgroup);
}
if (oldReplica != pDb->cfg.replications) {
balanceNotify();
}
return TSDB_CODE_SUCCESS;
}
@ -773,16 +853,6 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
return;
}
SVgObj *pVgroup = pDb->pHead;
if (pVgroup != NULL) {
mPrint("vgroup:%d, will be altered", pVgroup->vgId);
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
newMsg->ahandle = pVgroup;
newMsg->expected = pVgroup->numOfVnodes;
mgmtAlterVgroup(pVgroup, newMsg);
return;
}
mTrace("db:%s, all vgroups is altered", pDb->name);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
}

115
src/mnode/src/mgmtDnode.c
View File

@ -42,7 +42,7 @@ int32_t tsAccessSquence = 0;
extern void * tsMnodeSdb;
extern void * tsVgroupSdb;
static int32_t mgmtCreateDnode(uint32_t ip);
static int32_t mgmtCreateDnode(char *ep);
static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg);
static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
@ -68,12 +68,6 @@ static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
}
pDnode->mnodeShellPort = tsMnodeShellPort;
pDnode->mnodeDnodePort = tsMnodeDnodePort;
pDnode->dnodeShellPort = tsDnodeShellPort;
pDnode->dnodeMnodePort = tsDnodeMnodePort;
pDnode->syncPort = tsSyncPort;
return TSDB_CODE_SUCCESS;
}
@ -144,9 +138,8 @@ static int32_t mgmtDnodeActionDecode(SSdbOper *pOper) {
static int32_t mgmtDnodeActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
uint32_t ip = inet_addr(tsPrivateIp);
mgmtCreateDnode(ip);
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
mgmtCreateDnode(tsLocalEp);
SDnodeObj *pDnode = mgmtGetDnodeByIp(tsLocalEp);
mgmtAddMnode(pDnode->dnodeId);
mgmtDecDnodeRef(pDnode);
}
@ -214,14 +207,14 @@ void *mgmtGetDnode(int32_t dnodeId) {
return sdbGetRow(tsDnodeSdb, &dnodeId);
}
void *mgmtGetDnodeByIp(uint32_t ip) {
void *mgmtGetDnodeByIp(char *ep) {
SDnodeObj *pDnode = NULL;
void * pNode = NULL;
while (1) {
pNode = sdbFetchRow(tsDnodeSdb, pNode, (void**)&pDnode);
if (pDnode == NULL) break;
if (ip == pDnode->privateIp) {
if (strcmp(ep, pDnode->dnodeEp) == 0) {
return pDnode;
}
mgmtDecDnodeRef(pDnode);
@ -253,19 +246,18 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont;
if (pCmCfgDnode->ip[0] == 0) {
strcpy(pCmCfgDnode->ip, tsPrivateIp);
if (pCmCfgDnode->ep[0] == 0) {
strcpy(pCmCfgDnode->ep, tsLocalEp);
} else {
strcpy(pCmCfgDnode->ip, pCmCfgDnode->ip);
strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep);
}
uint32_t dnodeIp = inet_addr(pCmCfgDnode->ip);
if (strcmp(pMsg->pUser->user, "root") != 0) {
rpcRsp.code = TSDB_CODE_NO_RIGHTS;
} else {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(dnodeIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCmCfgDnode->ep);
SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
strcpy(pMdCfgDnode->ip, pCmCfgDnode->ip);
strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
strcpy(pMdCfgDnode->config, pCmCfgDnode->config);
SRpcMsg rpcMdCfgDnodeMsg = {
.handle = 0,
@ -279,7 +271,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
}
if (rpcRsp.code == TSDB_CODE_SUCCESS) {
mPrint("dnode:%s, is configured by %s", pCmCfgDnode->ip, pMsg->pUser->user);
mPrint("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
}
rpcSendResponse(&rpcRsp);
@ -292,8 +284,6 @@ static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
SDMStatusMsg *pStatus = rpcMsg->pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->privateIp = htonl(pStatus->privateIp);
pStatus->publicIp = htonl(pStatus->publicIp);
pStatus->moduleStatus = htonl(pStatus->moduleStatus);
pStatus->lastReboot = htonl(pStatus->lastReboot);
pStatus->numOfCores = htons(pStatus->numOfCores);
@ -308,23 +298,21 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
SDnodeObj *pDnode = NULL;
if (pStatus->dnodeId == 0) {
pDnode = mgmtGetDnodeByIp(pStatus->privateIp);
pDnode = mgmtGetDnodeByIp(pStatus->dnodeEp);
if (pDnode == NULL) {
mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp));
mTrace("dnode %s not created", pStatus->dnodeEp);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
return;
}
} else {
pDnode = mgmtGetDnode(pStatus->dnodeId);
if (pDnode == NULL) {
mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp));
mError("dnode id:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
return;
}
}
pDnode->privateIp = pStatus->privateIp;
pDnode->publicIp = pStatus->publicIp;
pDnode->lastReboot = pStatus->lastReboot;
pDnode->numOfCores = pStatus->numOfCores;
pDnode->diskAvailable = pStatus->diskAvailable;
@ -334,7 +322,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
pDnode->lastAccess = tsAccessSquence;
if (pStatus->dnodeId == 0) {
mTrace("dnode:%d, first access, privateIp:%s, name:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName);
mTrace("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp);
} else {
//mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
}
@ -343,10 +331,11 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
for (int32_t j = 0; j < openVnodes; ++j) {
SVnodeLoad *pVload = &pStatus->load[j];
pVload->vgId = htonl(pVload->vgId);
pVload->cfgVersion = htonl(pVload->cfgVersion);
SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId);
if (pVgroup == NULL) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
} else {
@ -390,25 +379,24 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
rpcSendResponse(&rpcRsp);
}
static int32_t mgmtCreateDnode(uint32_t ip) {
static int32_t mgmtCreateDnode(char *ep) {
int32_t grantCode = grantCheck(TSDB_GRANT_DNODE);
if (grantCode != TSDB_CODE_SUCCESS) {
return grantCode;
}
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
SDnodeObj *pDnode = mgmtGetDnodeByIp(ep);
if (pDnode != NULL) {
mError("dnode:%d is alredy exist, ip:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp));
mError("dnode:%d is alredy exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort);
return TSDB_CODE_DNODE_ALREADY_EXIST;
}
pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
pDnode->privateIp = ip;
pDnode->publicIp = ip;
pDnode->createdTime = taosGetTimestampMs();
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
sprintf(pDnode->dnodeName, "n%d", sdbGetId(tsDnodeSdb) + 1);
strcpy(pDnode->dnodeEp, ep);
taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -445,15 +433,15 @@ int32_t mgmtDropDnode(SDnodeObj *pDnode) {
return code;
}
static int32_t mgmtDropDnodeByIp(uint32_t ip) {
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
static int32_t mgmtDropDnodeByIp(char *ep) {
SDnodeObj *pDnode = mgmtGetDnodeByIp(ep);
if (pDnode == NULL) {
mError("dnode:%s, is not exist", taosIpStr(ip));
mError("dnode:%s, is not exist", ep);
return TSDB_CODE_DNODE_NOT_EXIST;
}
if (pDnode->privateIp == dnodeGetMnodeMasteIp()) {
mError("dnode:%d, can't drop dnode which is master", pDnode->dnodeId);
if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) {
mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
return TSDB_CODE_NO_REMOVE_MASTER;
}
@ -472,13 +460,12 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) {
if (strcmp(pMsg->pUser->user, "root") != 0) {
rpcRsp.code = TSDB_CODE_NO_RIGHTS;
} else {
uint32_t ip = inet_addr(pCreate->ip);
rpcRsp.code = mgmtCreateDnode(ip);
rpcRsp.code = mgmtCreateDnode(pCreate->ep);
if (rpcRsp.code == TSDB_CODE_SUCCESS) {
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
mLPrint("dnode:%d, ip:%s is created by %s", pDnode->dnodeId, pCreate->ip, pMsg->pUser->user);
SDnodeObj *pDnode = mgmtGetDnodeByIp(pCreate->ep);
mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user);
} else {
mError("failed to create dnode:%s, reason:%s", pCreate->ip, tstrerror(rpcRsp.code));
mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(rpcRsp.code));
}
}
rpcSendResponse(&rpcRsp);
@ -489,15 +476,15 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SCMDropDnodeMsg *pDrop = pMsg->pCont;
if (strcmp(pMsg->pUser->user, "root") != 0) {
rpcRsp.code = TSDB_CODE_NO_RIGHTS;
} else {
uint32_t ip = inet_addr(pDrop->ip);
rpcRsp.code = mgmtDropDnodeByIp(ip);
rpcRsp.code = mgmtDropDnodeByIp(pDrop->ep);
if (rpcRsp.code == TSDB_CODE_SUCCESS) {
mLPrint("dnode:%s is dropped by %s", pDrop->ip, pMsg->pUser->user);
mLPrint("dnode:%s is dropped by %s", pDrop->ep, pMsg->pUser->user);
} else {
mError("failed to drop dnode:%s, reason:%s", pDrop->ip, tstrerror(rpcRsp.code));
mError("failed to drop dnode:%s, reason:%s", pDrop->ep, tstrerror(rpcRsp.code));
}
}
@ -522,15 +509,9 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "private ip");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "public ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -580,7 +561,6 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
int32_t cols = 0;
SDnodeObj *pDnode = NULL;
char *pWrite;
char ipstr[32];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode);
@ -592,14 +572,8 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
*(int16_t *)pWrite = pDnode->dnodeId;
cols++;
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
cols++;
tinet_ntoa(ipstr, pDnode->publicIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -651,9 +625,9 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -701,10 +675,8 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
*(int16_t *)pWrite = pDnode->dnodeId;
cols++;
char ipstr[20];
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -864,8 +836,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
SDnodeObj *pDnode = NULL;
if (pShow->payloadLen > 0 ) {
uint32_t ip = ip2uint(pShow->payload);
pDnode = mgmtGetDnodeByIp(ip);
pDnode = mgmtGetDnodeByIp(pShow->payload);
} else {
mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode);
}

2
src/mnode/src/mgmtMain.c
View File

@ -130,7 +130,7 @@ int32_t mgmtInitSystem() {
struct stat dirstat;
bool fileExist = (stat(tsMnodeDir, &dirstat) == 0);
bool asMaster = (strcmp(tsMasterIp, tsPrivateIp) == 0);
bool asMaster = (strcmp(tsMaster, tsLocalEp) == 0);
if (asMaster || fileExist) {
if (mgmtStartSystem() != 0) {

34
src/mnode/src/mgmtMnode.c
View File

@ -171,25 +171,21 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
}
}
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet, bool usePublicIp) {
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
void *pNode = NULL;
while (1) {
SMnodeObj *pMnode = NULL;
pNode = mgmtGetNextMnode(pNode, &pMnode);
if (pMnode == NULL) break;
if (usePublicIp) {
ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->publicIp);
} else {
ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->privateIp);
}
strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
ipSet->inUse = ipSet->numOfIps;
}
ipSet->numOfIps++;
ipSet->port = htons(pMnode->pDnode->mnodeShellPort);
mgmtReleaseMnode(pMnode);
}
@ -207,10 +203,7 @@ void mgmtGetMnodeInfos(void *param) {
if (pMnode == NULL) break;
mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
mnodes->nodeInfos[index].nodeIp = htonl(pMnode->pDnode->privateIp);
mnodes->nodeInfos[index].nodePort = htons(pMnode->pDnode->mnodeDnodePort);
mnodes->nodeInfos[index].syncPort = htons(pMnode->pDnode->syncPort);
strcpy(mnodes->nodeInfos[index].nodeName, pMnode->pDnode->dnodeName);
strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
mnodes->inUse = index;
}
@ -282,15 +275,9 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "private ip");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "public ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -327,7 +314,6 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
int32_t cols = 0;
SMnodeObj *pMnode = NULL;
char *pWrite;
char ipstr[32];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode);
@ -339,14 +325,8 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
*(int16_t *)pWrite = pMnode->mnodeId;
cols++;
tinet_ntoa(ipstr, pMnode->pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
cols++;
tinet_ntoa(ipstr, pMnode->pDnode->publicIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

16
src/mnode/src/mgmtSdb.c
View File

@ -231,8 +231,8 @@ void sdbUpdateSync() {
for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
SDMMnodeInfo *node = &mnodes->nodeInfos[i];
syncCfg.nodeInfo[i].nodeId = node->nodeId;
syncCfg.nodeInfo[i].nodeIp = node->nodeIp;
strcpy(syncCfg.nodeInfo[i].name, node->nodeName);
taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[i].nodeFqdn, &syncCfg.nodeInfo[i].nodePort);
syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
index++;
}
@ -244,8 +244,8 @@ void sdbUpdateSync() {
if (pMnode == NULL) break;
syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
syncCfg.nodeInfo[index].nodeIp = pMnode->pDnode->privateIp;
strcpy(syncCfg.nodeInfo[index].name, pMnode->pDnode->dnodeName);
syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC;
strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
index++;
mgmtReleaseMnode(pMnode);
@ -253,7 +253,8 @@ void sdbUpdateSync() {
}
syncCfg.replica = index;
syncCfg.arbitratorIp = syncCfg.nodeInfo[0].nodeIp;
syncCfg.arbitratorPort = syncCfg.nodeInfo[0].nodePort;
strcpy(syncCfg.arbitratorFqdn, syncCfg.nodeInfo[0].nodeFqdn);
if (syncCfg.replica == 1) {
syncCfg.quorum = 1;
} else {
@ -271,10 +272,9 @@ void sdbUpdateSync() {
if (!hasThisDnode) return;
if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
sdbPrint("work as mnode, replica:%d arbitratorIp:%s", syncCfg.replica, taosIpStr(syncCfg.arbitratorIp));
sdbPrint("work as mnode, replica:%d arbitrator:%s", syncCfg.replica, syncCfg.arbitratorFqdn);
for (int32_t i = 0; i < syncCfg.replica; ++i) {
sdbPrint("mnode:%d, ip:%s name:%s", syncCfg.nodeInfo[i].nodeId, taosIpStr(syncCfg.nodeInfo[i].nodeIp),
syncCfg.nodeInfo[i].name);
sdbPrint("mnode:%d, ip:%s", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn);
}
SSyncInfo syncInfo;

11
src/mnode/src/mgmtShell.c
View File

@ -72,7 +72,6 @@ int32_t mgmtInitShell() {
}
SRpcInit rpcInit = {0};
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = tsMnodeShellPort;
rpcInit.label = "MND-shell";
rpcInit.numOfThreads = numOfThreads;
@ -148,14 +147,12 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
if (!sdbIsMaster()) {
SRpcConnInfo connInfo;
rpcGetConnInfo(rpcMsg->handle, &connInfo);
bool usePublicIp = (connInfo.serverIp == tsPublicIpInt);
SRpcIpSet ipSet = {0};
ipSet.port = tsMnodeShellPort;
dnodeGetMnodeIpSet(&ipSet, usePublicIp);
mgmtGetMnodeIpSet(&ipSet);
mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i]));
mTrace("index:%d ip:%s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
}
rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
@ -343,7 +340,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
return;
}
mgmtGetMnodeIpSet(&pHBRsp->ipList, pMsg->usePublicIp);
mgmtGetMnodeIpSet(&pHBRsp->ipList);
/*
* TODO
@ -429,7 +426,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
pConnectRsp->writeAuth = pUser->writeAuth;
pConnectRsp->superAuth = pUser->superAuth;
mgmtGetMnodeIpSet(&pConnectRsp->ipList, pMsg->usePublicIp);
mgmtGetMnodeIpSet(&pConnectRsp->ipList);
connect_over:
rpcRsp.code = code;

310
src/mnode/src/mgmtTable.c
View File

@ -74,10 +74,13 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg);
static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg);
static void mgmtGetChildTableMeta(SQueuedMsg *pMsg);
static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg);
static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg);
static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg);
static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName);
static void mgmtDestroyChildTable(SChildTableObj *pTable) {
tfree(pTable->schema);
tfree(pTable->sql);
@ -610,9 +613,12 @@ static void mgmtExtractTableName(char* tableId, char* name) {
static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->pCont;
pMsg->pTable = mgmtGetTable(pCreate->tableId);
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pCreate->tableId);
if (pMsg->pTable != NULL && pMsg->retry == 0) {
if (pCreate->igExists) {
if (pCreate->getMeta) {
mTrace("table:%s, continue to get meta", pCreate->tableId);
mgmtProcessTableMetaMsg(pMsg);
} else if (pCreate->igExists) {
mTrace("table:%s, is already exist", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
} else {
@ -622,7 +628,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
return;
}
pMsg->pDb = mgmtGetDb(pCreate->db);
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pCreate->db);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to create, db not selected", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
@ -679,15 +685,21 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle);
pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to get table meta, db not selected", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId);
if (pMsg->pTable == NULL) {
mgmtGetChildTableMeta(pMsg);
if (htons(pInfo->createFlag) != 1) {
mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
} else {
mgmtAutoCreateChildTable(pMsg);
}
} else {
if (pMsg->pTable->type != TSDB_SUPER_TABLE) {
mgmtGetChildTableMeta(pMsg);
@ -756,8 +768,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
if (pStable->numOfTables != 0) {
mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
mgmtDropAllChildTablesInStable(pStable);
for (int32_t vg = 0; vg < pStable->vgLen; ++vg) {
int32_t vgId = pStable->vgList[vg];
if (vgId == 0) break;
SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
pDrop->vgId = htonl(vgId);
pDrop->uid = htobe64(pStable->uid);
mgmtExtractTableName(pStable->info.tableId, pDrop->tableId);
SVgObj *pVgroup = mgmtGetVgroup(vgId);
if (pVgroup != NULL) {
SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
mgmtSendMsgToDnode(&ipSet, &rpcMsg);
mgmtDecVgroupRef(pVgroup);
}
}
//mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
//mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -783,31 +813,33 @@ static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *t
static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) {
if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) {
return TSDB_CODE_APP_ERROR;
mError("stable:%s, add tag, too many tags", pStable->info.tableId);
return TSDB_CODE_TOO_MANY_TAGS;
}
// check if schemas have the same name
for (int32_t i = 1; i < ntags; i++) {
for (int32_t j = 0; j < i; j++) {
if (strcasecmp(schema[i].name, schema[j].name) == 0) {
return TSDB_CODE_APP_ERROR;
}
for (int32_t i = 0; i < ntags; i++) {
if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
mError("stable:%s, add tag, column:%s already exist", pStable->info.tableId, schema[i].name);
return TSDB_CODE_TAG_ALREAY_EXIST;
}
if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
mError("stable:%s, add tag, tag:%s already exist", pStable->info.tableId, schema[i].name);
return TSDB_CODE_FIELD_ALREAY_EXIST;
}
}
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ntags);
memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ntags),
pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags);
memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ntags);
memcpy(pStable->schema + pStable->numOfColumns + pStable->numOfTags, schema, sizeof(SSchema) * ntags);
SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns);
SSchema *tschema = (SSchema *)(pStable->schema + pStable->numOfColumns + pStable->numOfTags);
for (int32_t i = 0; i < ntags; i++) {
tschema[i].colId = pStable->nextColId++;
}
pStable->numOfColumns += ntags;
pStable->numOfTags += ntags;
pStable->sversion++;
SSdbOper oper = {
@ -822,25 +854,22 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
return TSDB_CODE_SDB_ERROR;
}
mPrint("table %s, succeed to add tag %s", pStable->info.tableId, schema[0].name);
mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName);
if (col <= 0 || col >= pStable->numOfTags) {
return TSDB_CODE_APP_ERROR;
if (col < 0) {
mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName);
return TSDB_CODE_TAG_NOT_EXIST;
}
memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1),
sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1));
memmove(pStable->schema + pStable->numOfColumns + col, pStable->schema + pStable->numOfColumns + col + 1,
sizeof(SSchema) * (pStable->numOfTags - col - 1));
pStable->numOfTags--;
pStable->sversion++;
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
pStable->schema = realloc(pStable->schema, schemaSize);
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
@ -853,27 +882,29 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
return TSDB_CODE_SDB_ERROR;
}
mPrint("table %s, succeed to drop tag %s", pStable->info.tableId, tagName);
mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) {
int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName);
if (col < 0) {
// Tag name does not exist
mError("table:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName);
return TSDB_CODE_INVALID_MSG_TYPE;
mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName);
return TSDB_CODE_TAG_NOT_EXIST;
}
// int32_t rowSize = 0;
uint32_t len = strlen(newTagName);
if (len >= TSDB_COL_NAME_LEN) {
return TSDB_CODE_COL_NAME_TOO_LONG;
}
if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) {
return TSDB_CODE_APP_ERROR;
if (mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) {
return TSDB_CODE_TAG_ALREAY_EXIST;
}
// update
SSchema *schema = (SSchema *) (pStable->schema + (pStable->numOfColumns + col) * sizeof(SSchema));
SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col);
strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN);
SSdbOper oper = {
@ -888,15 +919,15 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
return TSDB_CODE_SDB_ERROR;
}
mPrint("table %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName);
mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) {
SSchema *schema = (SSchema *) pStable->schema;
for (int32_t i = 0; i < pStable->numOfColumns; i++) {
if (strcasecmp(schema[i].name, colName) == 0) {
return i;
for (int32_t col = 0; col < pStable->numOfColumns; col++) {
if (strcasecmp(schema[col].name, colName) == 0) {
return col;
}
}
@ -905,21 +936,28 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN
static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) {
if (ncols <= 0) {
mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
for (int32_t i = 0; i < ncols; i++) {
if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
return TSDB_CODE_APP_ERROR;
mError("stable:%s, add column, column:%s already exist", pStable->info.tableId, schema[i].name);
return TSDB_CODE_FIELD_ALREAY_EXIST;
}
if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
mError("stable:%s, add column, tag:%s already exist", pStable->info.tableId, schema[i].name);
return TSDB_CODE_TAG_ALREAY_EXIST;
}
}
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ncols);
memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ncols),
pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags);
memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ncols);
memmove(pStable->schema + pStable->numOfColumns + ncols, pStable->schema + pStable->numOfColumns,
sizeof(SSchema) * pStable->numOfTags);
memcpy(pStable->schema + pStable->numOfColumns, schema, sizeof(SSchema) * ncols);
SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns);
for (int32_t i = 0; i < ncols; i++) {
@ -947,17 +985,18 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
return TSDB_CODE_SDB_ERROR;
}
mPrint("table %s, succeed to add column", pStable->info.tableId);
mPrint("stable %s, succeed to add column", pStable->info.tableId);
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, char *colName) {
int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName);
if (col < 0) {
return TSDB_CODE_APP_ERROR;
if (col <= 0) {
mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName);
return TSDB_CODE_FIELD_NOT_EXIST;
}
memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1),
memmove(pStable->schema + col, pStable->schema + col + 1,
sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1));
pStable->numOfColumns--;
@ -984,7 +1023,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
return TSDB_CODE_SDB_ERROR;
}
mPrint("table %s, succeed to delete column", pStable->info.tableId);
mPrint("stable %s, succeed to delete column", pStable->info.tableId);
return TSDB_CODE_SUCCESS;
}
@ -1173,47 +1212,64 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
SCMSTableVgroupMsg *pInfo = pMsg->pCont;
SSuperTableObj *pTable = mgmtGetSuperTable(pInfo->tableId);
pMsg->pTable = (STableObj *)pTable;
if (pMsg->pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * pTable->vgLen;
SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
int32_t numOfTable = htonl(pInfo->numOfTables);
char* name = (char*) pInfo + sizeof(struct SCMSTableVgroupMsg);
SCMSTableVgroupRspMsg *pRsp = NULL;
// todo set the initial size to be 10, fix me
int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + (sizeof(SCMVgroupInfo) * 10 + sizeof(SVgroupsInfo))*numOfTable;
pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
int32_t vg = 0;
for (; vg < pTable->vgLen; ++vg) {
int32_t vgId = pTable->vgList[vg];
if (vgId == 0) break;
SVgObj *pVgroup = mgmtGetVgroup(vgId);
if (pVgroup == NULL) break;
pRsp->vgroups[vg].vgId = htonl(vgId);
for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) {
SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
if (pDnode == NULL) break;
pRsp->vgroups[vg].ipAddr[vn].ip = htonl(pDnode->privateIp);
pRsp->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort);
pRsp->vgroups[vg].numOfIps++;
pRsp->numOfTables = htonl(numOfTable);
char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg);
for(int32_t i = 0; i < numOfTable; ++i) {
SSuperTableObj *pTable = mgmtGetSuperTable(name);
pMsg->pTable = (STableObj *)pTable;
if (pMsg->pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
mgmtDecVgroupRef(pVgroup);
SVgroupsInfo* pVgroup = (SVgroupsInfo*) msg;
int32_t vg = 0;
for (; vg < pTable->vgLen; ++vg) {
int32_t vgId = pTable->vgList[vg];
if (vgId == 0) break;
SVgObj *vgItem = mgmtGetVgroup(vgId);
if (vgItem == NULL) break;
pVgroup->vgroups[vg].vgId = htonl(vgId);
for (int32_t vn = 0; vn < vgItem->numOfVnodes; ++vn) {
SDnodeObj *pDnode = vgItem->vnodeGid[vn].pDnode;
if (pDnode == NULL) break;
pVgroup->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort);
pVgroup->vgroups[vg].numOfIps++;
}
mgmtDecVgroupRef(vgItem);
}
pVgroup->numOfVgroups = htonl(vg);
// one table is done, try the next table
msg += sizeof(SVgroupsInfo) + vg * sizeof(SCMVgroupInfo);
}
pRsp->numOfVgroups = htonl(vg);
SRpcMsg rpcRsp = {0};
rpcRsp.handle = pMsg->thandle;
rpcRsp.pCont = pRsp;
rpcRsp.contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * vg;
rpcRsp.contLen = msg - (char*) pRsp;
rpcSendResponse(&rpcRsp);
}
@ -1472,9 +1528,9 @@ static int32_t mgmtModifyChildTableTagValue(SChildTableObj *pTable, char *tagNam
static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) {
SSchema *schema = (SSchema *) pTable->schema;
for (int32_t i = 0; i < pTable->numOfColumns; i++) {
if (strcasecmp(schema[i].name, colName) == 0) {
return i;
for (int32_t col = 0; col < pTable->numOfColumns; col++) {
if (strcasecmp(schema[col].name, colName) == 0) {
return col;
}
}
@ -1483,21 +1539,23 @@ static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colN
static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSchema schema[], int32_t ncols) {
if (ncols <= 0) {
mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId);
return TSDB_CODE_APP_ERROR;
}
for (int32_t i = 0; i < ncols; i++) {
if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) {
return TSDB_CODE_APP_ERROR;
mError("table:%s, add column, column:%s already exist", pTable->info.tableId, schema[i].name);
return TSDB_CODE_FIELD_ALREAY_EXIST;
}
}
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols);
memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols);
memcpy(pTable->schema + pTable->numOfColumns, schema, sizeof(SSchema) * ncols);
SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns);
SSchema *tschema = (SSchema *) (pTable->schema + pTable->numOfColumns);
for (int32_t i = 0; i < ncols; i++) {
tschema[i].colId = pTable->nextColId++;
}
@ -1507,7 +1565,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
SAcctObj *pAcct = mgmtGetAcct(pDb->acct);
if (pAcct != NULL) {
pAcct->acctInfo.numOfTimeSeries += ncols;
pAcct->acctInfo.numOfTimeSeries += ncols;
mgmtDecAcctRef(pAcct);
}
@ -1529,13 +1587,12 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, char *colName) {
int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName);
if (col < 0) {
return TSDB_CODE_APP_ERROR;
if (col <= 0) {
mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName);
return TSDB_CODE_FIELD_NOT_EXIST;
}
memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1),
sizeof(SSchema) * (pTable->numOfColumns - col - 1));
memmove(pTable->schema + col, pTable->schema + col + 1, sizeof(SSchema) * (pTable->numOfColumns - col - 1));
pTable->numOfColumns--;
pTable->sversion++;
@ -1557,7 +1614,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
return TSDB_CODE_SDB_ERROR;
}
mPrint("table %s, succeed to add column %s", pTable->info.tableId, colName);
mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName);
return TSDB_CODE_SUCCESS;
}
@ -1577,7 +1634,6 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pT
static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
SDbObj *pDb = pMsg->pDb;
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
int8_t usePublicIp = pMsg->usePublicIp;
pMeta->uid = htobe64(pTable->uid);
pMeta->sid = htonl(pTable->sid);
@ -1607,13 +1663,8 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
if (pDnode == NULL) break;
if (usePublicIp) {
pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->publicIp);
pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort);
} else {
pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->privateIp);
pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort);
}
strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn);
pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL);
pMeta->vgroup.numOfIps++;
mgmtDecDnodeRef(pDnode);
}
@ -1624,42 +1675,34 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
return TSDB_CODE_SUCCESS;
}
void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
if (pTable == NULL) {
if (htons(pInfo->createFlag) != 1) {
mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
} else {
//TODO: on demand create table from super table if table does not exists
int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId));
SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
memcpy(newMsg, pMsg, sizeof(SQueuedMsg));
pMsg->pCont = NULL;
newMsg->ahandle = newMsg->pCont;
newMsg->pCont = pCreateMsg;
mTrace("table:%s, start to create in demand", pInfo->tableId);
mgmtAddToShellQueue(newMsg);
return;
}
int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId));
strcpy(pCreateMsg->db, pMsg->pDb->name);
pCreateMsg->igExists = 1;
pCreateMsg->getMeta = 1;
memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
pMsg->pCont = newMsg->pCont;
newMsg->pCont = pCreateMsg;
mTrace("table:%s, start to create on demand", pInfo->tableId);
mgmtAddToShellQueue(newMsg);
}
static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS);
if (pMeta == NULL) {
mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId);
mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
@ -1769,8 +1812,8 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
mgmtDecTableRef(pTable);
return;
}
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode);
SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnode);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
SRpcMsg rpcRsp = {
.handle = NULL,
.pCont = pMDCreate,
@ -2078,7 +2121,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
return;
}
pAlter->numOfCols = htons(pAlter->numOfCols);
pAlter->type = htons(pAlter->type);
if (pAlter->numOfCols > 2) {
mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR);

83
src/mnode/src/mgmtVgroup.c
View File

@ -162,7 +162,7 @@ static int32_t mgmtVgroupActionEncode(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
SVgObj *pTmpVgroup = pOper->rowData;
for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) {
for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
pTmpVgroup->vnodeGid[i].pDnode = NULL;
pTmpVgroup->vnodeGid[i].role = 0;
}
@ -260,7 +260,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
}
if (!dnodeExist) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
mError("vgroup:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
return;
@ -272,8 +272,9 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
pVgroup->pointsWritten = htobe64(pVload->pointsWritten);
}
if (pVload->replica != pVgroup->numOfVnodes) {
mError("dnode:%d, vgroup:%d replica:%d not match with mgmt:%d", pDnode->dnodeId, pVload->vgId, pVload->replica,
if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) {
mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion,
pVgroup->numOfVnodes);
mgmtSendCreateVgroupMsg(pVgroup, NULL);
}
@ -400,9 +401,9 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -439,7 +440,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
SVgObj *pVgroup = NULL;
int32_t maxReplica = 0;
int32_t cols = 0;
char ipstr[20];
char * pWrite;
SDbObj *pDb = mgmtGetDb(pShow->db);
@ -478,10 +478,10 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
SDnodeObj *pDnode = pVgroup->vnodeGid[i].pDnode;
if (pDnode != NULL) {
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role));
cols++;
@ -535,23 +535,22 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
SMDVnodeCfg *pCfg = &pVnode->cfg;
pCfg->vgId = htonl(pVgroup->vgId);
pCfg->cfgVersion = htonl(pDb->cfgVersion);
pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks);
pCfg->maxTables = htonl(pDb->cfg.maxTables);
pCfg->maxCacheSize = htobe64(pDb->cfg.maxCacheSize);
pCfg->maxCacheSize = htobe64(-1); //TODO
pCfg->minRowsPerFileBlock = htonl(-1); //TODO
pCfg->maxRowsPerFileBlock = htonl(-1); //TODO
pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile);
pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1);
pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2);
pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep);
pCfg->daysToKeep = htonl(-1); //TODO
pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1);
pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2);
pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock);
pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock);
pCfg->commitTime = htonl(pDb->cfg.commitTime);
pCfg->precision = pDb->cfg.precision;
pCfg->compression = pDb->cfg.compression;
pCfg->compression = -1;
pCfg->wals = 3;
pCfg->commitLog = pDb->cfg.commitLog;
pCfg->replications = (int8_t) pVgroup->numOfVnodes;
pCfg->wals = 3;
pCfg->quorum = 1;
SMDVnodeDesc *pNodes = pVnode->nodes;
@ -559,11 +558,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode;
if (pDnode != NULL) {
pNodes[j].nodeId = htonl(pDnode->dnodeId);
pNodes[j].nodeIp = htonl(pDnode->privateIp);
strcpy(pNodes[j].nodeName, pDnode->dnodeName);
if (j == 0) {
pCfg->arbitratorIp = htonl(pDnode->privateIp);
}
strcpy(pNodes[j].nodeEp, pDnode->dnodeEp);
}
}
@ -574,21 +569,21 @@ SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) {
SRpcIpSet ipSet = {
.numOfIps = pVgroup->numOfVnodes,
.inUse = 0,
.port = tsDnodeMnodePort
};
for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
ipSet.ip[i] = pVgroup->vnodeGid[i].pDnode->privateIp;
strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn);
ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEMNODE;
}
return ipSet;
}
SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip) {
SRpcIpSet ipSet = {
.ip[0] = ip,
.numOfIps = 1,
.inUse = 0,
.port = tsDnodeMnodePort
};
SRpcIpSet mgmtGetIpSetFromIp(char *ep) {
SRpcIpSet ipSet;
ipSet.numOfIps = 1;
ipSet.inUse = 0;
taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]);
ipSet.port[0] += TSDB_PORT_DNODEMNODE;
return ipSet;
}
@ -608,7 +603,7 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mgmtSendCreateVnodeMsg(pVgroup, &ipSet, ahandle);
}
}
@ -674,7 +669,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
mTrace("vgroup:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle);
}
}
@ -737,7 +732,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
}
@ -745,16 +740,14 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfVgroups = 0;
int32_t dbNameLen = strlen(pDropDb->name);
SVgObj *pVgroup = NULL;
mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
while (1) {
pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
if (pVgroup == NULL) break;
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
if (pVgroup->pDb == pDropDb) {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
@ -763,23 +756,11 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfVgroups++;
mgmtSendDropVgroupMsg(pVgroup, NULL);
}
mgmtSendDropVgroupMsg(pVgroup, NULL);
mgmtDecVgroupRef(pVgroup);
}
mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
}
void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) {
assert(ahandle != NULL);
if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) {
// TODO:
// mgmtSendAlterVgroupMsg(pVgroup, NULL);
} else {
mgmtAddToShellQueue(ahandle);
}
}

8
src/os/darwin/src/darwinEnv.c
View File

@ -23,8 +23,8 @@ void osInit() {
strcpy(tsVnodeDir, "");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(dataDir, "/var/lib/taos");
strcpy(logDir, "~/TDengineLog");
strcpy(scriptDir, "/etc/taos");
strcpy(osName, "Darwin");
strcpy(tsDataDir, "/var/lib/taos");
strcpy(tsLogDir, "~/TDengineLog");
strcpy(tsScriptDir, "/etc/taos");
strcpy(tsOsName, "Darwin");
}

8
src/os/linux/src/linuxEnv.c
View File

@ -23,8 +23,8 @@ void osInit() {
strcpy(tsVnodeDir, "");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(dataDir, "/var/lib/taos");
strcpy(logDir, "/var/log/taos");
strcpy(scriptDir, "/etc/taos");
strcpy(osName, "Linux");
strcpy(tsDataDir, "/var/lib/taos");
strcpy(tsLogDir, "/var/log/taos");
strcpy(tsScriptDir, "/etc/taos");
strcpy(tsOsName, "Linux");
}

23
src/os/linux/src/linuxSysPara.c
View File

@ -290,11 +290,12 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
bool taosGetDisk() {
struct statvfs info;
const double unit = 1024 * 1024 * 1024;
if (tscEmbedded) {
if (statvfs(dataDir, &info)) {
tsTotalDataDirGB = 0;
tsAvailDataDirGB = 0;
if (statvfs(tsDataDir, &info)) {
//tsTotalDataDirGB = 0;
//tsAvailDataDirGB = 0;
uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
return false;
} else {
tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
@ -302,9 +303,10 @@ bool taosGetDisk() {
}
}
if (statvfs(logDir, &info)) {
tsTotalLogDirGB = 0;
tsAvailLogDirGB = 0;
if (statvfs(tsLogDir, &info)) {
//tsTotalLogDirGB = 0;
//tsAvailLogDirGB = 0;
uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
return false;
} else {
tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
@ -312,8 +314,9 @@ bool taosGetDisk() {
}
if (statvfs("/tmp", &info)) {
tsTotalTmpDirGB = 0;
tsAvailTmpDirGB = 0;
//tsTotalTmpDirGB = 0;
//tsAvailTmpDirGB = 0;
uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
return false;
} else {
tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
@ -361,6 +364,8 @@ static bool taosGetCardName(char *ip, char *name) {
static bool taosGetCardInfo(int64_t *bytes) {
static char tsPublicCard[1000] = {0};
static char tsPrivateIp[40];
if (tsPublicCard[0] == 0) {
if (!taosGetCardName(tsPrivateIp, tsPublicCard)) {
uError("can't get card name from ip:%s", tsPrivateIp);

8
src/os/windows/src/twinenv.c
View File

@ -23,8 +23,8 @@ void osInit() {
strcpy(tsVnodeDir, "C:/TDengine/data");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(dataDir, "C:/TDengine/data");
strcpy(logDir, "C:/TDengine/log");
strcpy(scriptDir, "C:/TDengine/script");
strcpy(osName, "Windows");
strcpy(tsDataDir, "C:/TDengine/data");
strcpy(tsLogDir, "C:/TDengine/log");
strcpy(tsScriptDir, "C:/TDengine/script");
strcpy(tsOsName, "Windows");
}

2
src/plugins/http/inc/httpHandle.h
View File

@ -210,7 +210,7 @@ typedef struct HttpThread {
typedef struct HttpServer {
char label[HTTP_LABEL_SIZE];
char serverIp[16];
uint32_t serverIp;
uint16_t serverPort;
int cacheContext;
int sessionExpire;

2
src/plugins/http/src/httpResp.c
View File

@ -199,7 +199,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) {
} else {}
}
httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_INVALID_SQL, temp);
httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, temp);
}
void httpSendSuccResp(HttpContext *pContext, char *desc) {

2
src/plugins/http/src/httpSql.c
View File

@ -239,7 +239,7 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) {
if (code < 0) {
SSqlObj *pObj = (SSqlObj *)result;
if (code == TSDB_CODE_INVALID_SQL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:invalidsql, sqlObj:%p, error:%s",
httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload);
httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload);
} else {

4
src/plugins/http/src/httpSystem.c
View File

@ -48,7 +48,7 @@ int httpInitSystem() {
memset(httpServer, 0, sizeof(HttpServer));
strcpy(httpServer->label, "rest");
strcpy(httpServer->serverIp, tsHttpIp);
httpServer->serverIp = 0;
httpServer->serverPort = tsHttpPort;
httpServer->cacheContext = tsHttpCacheSessions;
httpServer->sessionExpire = tsHttpSessionExpire;
@ -117,7 +117,7 @@ void httpCleanUpSystem() {
httpPrint("http service cleanup");
httpStopSystem();
#if 1
#if 0
if (httpServer == NULL) {
return;
}

20
src/plugins/monitor/src/monitorMain.c
View File

@ -68,7 +68,7 @@ typedef enum {
typedef struct {
void * conn;
void * timer;
char privateIpStr[TSDB_IPv4ADDR_LEN];
char ep[TSDB_FQDN_LEN];
int8_t cmdIndex;
int8_t state;
char sql[SQL_LENGTH];
@ -112,14 +112,8 @@ static void monitorInitConn(void *para, void *unused) {
monitorPrint("starting to initialize monitor service ..");
tsMonitorConn.state = MONITOR_STATE_INITIALIZING;
if (tsMonitorConn.privateIpStr[0] == 0) {
strcpy(tsMonitorConn.privateIpStr, tsPrivateIp);
for (int32_t i = 0; i < TSDB_IPv4ADDR_LEN; ++i) {
if (tsMonitorConn.privateIpStr[i] == '.') {
tsMonitorConn.privateIpStr[i] = '_';
}
}
}
if (tsMonitorConn.ep[0] == 0)
strcpy(tsMonitorConn.ep, tsLocalEp);
if (tsMonitorConn.conn == NULL) {
taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, &tsMonitorConn, &(tsMonitorConn.conn));
@ -163,7 +157,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
tsMonitorDbName, IP_LEN_STR + 1);
} else if (cmd == MONITOR_CMD_CREATE_TB_DN) {
snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName,
tsMonitorConn.privateIpStr, tsMonitorDbName, tsPrivateIp);
tsMonitorConn.ep, tsMonitorDbName, tsLocalEp);
} else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) {
snprintf(sql, SQL_LENGTH,
"create table if not exists %s.acct(ts timestamp "
@ -214,7 +208,7 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) {
if (-code == TSDB_CODE_TABLE_ALREADY_EXIST || -code == TSDB_CODE_DB_ALREADY_EXIST || code >= 0) {
monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql);
if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) {
monitorPrint("dnode:%s is started", tsPrivateIp);
monitorPrint("dnode:%s is started", tsLocalEp);
}
tsMonitorConn.cmdIndex++;
monitorInitDatabase();
@ -346,7 +340,7 @@ static void monitorSaveSystemInfo() {
int64_t ts = taosGetTimestampUs();
char * sql = tsMonitorConn.sql;
int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.privateIpStr, ts);
int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.ep, ts);
pos += monitorBuildCpuSql(sql + pos);
pos += monitorBuildMemorySql(sql + pos);
@ -414,7 +408,7 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
va_end(argpointer);
if (len > max_length) len = max_length;
len += sprintf(sql + len, "', '%s')", tsPrivateIp);
len += sprintf(sql + len, "', '%s')", tsLocalEp);
sql[len++] = 0;
monitorTrace("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql);

5
src/query/inc/qast.h
View File

@ -91,9 +91,10 @@ void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res);
uint8_t getBinaryExprOptr(SSQLToken *pToken);
SBuffer exprTreeToBinary(tExprNode* pExprTree);
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
tExprNode* exprTreeFromBinary(const void* pBuf, size_t size);
tExprNode* exprTreeFromBinary(const void* data, size_t size);
tExprNode* exprTreeFromTableName(const char* tbnameCond);
#ifdef __cplusplus

14
src/query/inc/tsqlfunction.h
View File

@ -108,17 +108,17 @@ enum {
SECONDARY_STAGE_MERGE = 0x20u,
};
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (((type)&TSDB_QUERY_TYPE_JOIN_QUERY) != 0)
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY))
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
typedef struct SArithmeticSupport {
SExprInfo *pArithExpr;
int32_t numOfCols;
SColumnInfo* colList;
int32_t offset;
char** data;
int32_t numOfCols;
SColumnInfo *colList;
int32_t offset;
char** data;
} SArithmeticSupport;
typedef struct SQLPreAggVal {

184
src/query/src/qast.c
View File

@ -31,6 +31,7 @@
#include "tskiplist.h"
#include "queryLog.h"
#include "tsdbMain.h"
#include "exception.h"
/*
*
@ -43,7 +44,6 @@
*
*/
static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken);
static void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i);
static void destroySyntaxTree(tExprNode *);
@ -427,7 +427,7 @@ void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len) {
static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); }
static void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
if (pNode == NULL) {
return;
}
@ -1015,104 +1015,116 @@ void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) {
}
}
static void exprTreeToBinaryImpl(tExprNode* pExprTree, SBuffer* pBuf) {
tbufWrite(pBuf, &pExprTree->nodeType, sizeof(pExprTree->nodeType));
static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
tbufWriteUint8(bw, expr->nodeType);
if (pExprTree->nodeType == TSQL_NODE_VALUE) {
tVariant* pVal = pExprTree->pVal;
if (expr->nodeType == TSQL_NODE_VALUE) {
tVariant* pVal = expr->pVal;
tbufWrite(pBuf, &pVal->nType, sizeof(pVal->nType));
tbufWriteUint32(bw, pVal->nType);
if (pVal->nType == TSDB_DATA_TYPE_BINARY) {
tbufWrite(pBuf, &pVal->nLen, sizeof(pVal->nLen));
tbufWrite(pBuf, pVal->pz, pVal->nLen);
tbufWriteInt32(bw, pVal->nLen);
tbufWrite(bw, pVal->pz, pVal->nLen);
} else {
tbufWrite(pBuf, &pVal->pz, sizeof(pVal->i64Key));
tbufWriteInt64(bw, pVal->i64Key);
}
} else if (pExprTree->nodeType == TSQL_NODE_COL) {
SSchema* pSchema = pExprTree->pSchema;
tbufWrite(pBuf, &pSchema->colId, sizeof(pSchema->colId));
tbufWrite(pBuf, &pSchema->bytes, sizeof(pSchema->bytes));
tbufWrite(pBuf, &pSchema->type, sizeof(pSchema->type));
} else if (expr->nodeType == TSQL_NODE_COL) {
SSchema* pSchema = expr->pSchema;
tbufWriteInt16(bw, pSchema->colId);
tbufWriteInt16(bw, pSchema->bytes);
tbufWriteUint8(bw, pSchema->type);
tbufWriteString(bw, pSchema->name);
int32_t len = strlen(pSchema->name);
tbufWriteStringLen(pBuf, pSchema->name, len);
} else if (pExprTree->nodeType == TSQL_NODE_EXPR) {
tbufWrite(pBuf, &pExprTree->_node.optr, sizeof(pExprTree->_node.optr));
tbufWrite(pBuf, &pExprTree->_node.hasPK, sizeof(pExprTree->_node.hasPK));
exprTreeToBinaryImpl(pExprTree->_node.pLeft, pBuf);
exprTreeToBinaryImpl(pExprTree->_node.pRight, pBuf);
} else if (expr->nodeType == TSQL_NODE_EXPR) {
tbufWriteUint8(bw, expr->_node.optr);
tbufWriteUint8(bw, expr->_node.hasPK);
exprTreeToBinaryImpl(bw, expr->_node.pLeft);
exprTreeToBinaryImpl(bw, expr->_node.pRight);
}
}
SBuffer exprTreeToBinary(tExprNode* pExprTree) {
SBuffer buf = {0};
if (pExprTree == NULL) {
return buf;
void exprTreeToBinary(SBufferWriter* bw, tExprNode* expr) {
if (expr != NULL) {
exprTreeToBinaryImpl(bw, expr);
}
int32_t code = tbufBeginWrite(&buf);
if (code != 0) {
return buf;
}
exprTreeToBinaryImpl(pExprTree, &buf);
return buf;
}
static tExprNode* exprTreeFromBinaryImpl(SBuffer* pBuf) {
tExprNode* pExpr = calloc(1, sizeof(tExprNode));
pExpr->nodeType = tbufReadUint8(pBuf);
// TODO: these three functions should be made global
static void* exception_calloc(size_t nmemb, size_t size) {
void* p = calloc(nmemb, size);
if (p == NULL) {
THROW(TSDB_CODE_SERV_OUT_OF_MEMORY);
}
return p;
}
static void* exception_malloc(size_t size) {
void* p = malloc(size);
if (p == NULL) {
THROW(TSDB_CODE_SERV_OUT_OF_MEMORY);
}
return p;
}
static char* exception_strdup(const char* str) {
char* p = strdup(str);
if (p == NULL) {
THROW(TSDB_CODE_SERV_OUT_OF_MEMORY);
}
return p;
}
static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
int32_t anchor = CLEANUP_GET_ANCHOR();
tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL);
pExpr->nodeType = tbufReadUint8(br);
if (pExpr->nodeType == TSQL_NODE_VALUE) {
tVariant* pVal = calloc(1, sizeof(tVariant));
if (pVal == NULL) {
// TODO:
}
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
pExpr->pVal = pVal;
pVal->nType = tbufReadUint32(pBuf);
pVal->nType = tbufReadUint32(br);
if (pVal->nType == TSDB_DATA_TYPE_BINARY) {
tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen));
tbufReadToBuffer(br, &pVal->nLen, sizeof(pVal->nLen));
pVal->pz = calloc(1, pVal->nLen + 1);
tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen);
tbufReadToBuffer(br, pVal->pz, pVal->nLen);
} else {
pVal->i64Key = tbufReadInt64(pBuf);
pVal->i64Key = tbufReadInt64(br);
}
} else if (pExpr->nodeType == TSQL_NODE_COL) {
SSchema* pSchema = calloc(1, sizeof(SSchema));
if (pSchema == NULL) {
// TODO:
}
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
pExpr->pSchema = pSchema;
pSchema->colId = tbufReadInt16(pBuf);
pSchema->bytes = tbufReadInt16(pBuf);
pSchema->type = tbufReadUint8(pBuf);
tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN);
pSchema->colId = tbufReadInt16(br);
pSchema->bytes = tbufReadInt16(br);
pSchema->type = tbufReadUint8(br);
tbufReadToString(br, pSchema->name, TSDB_COL_NAME_LEN);
} else if (pExpr->nodeType == TSQL_NODE_EXPR) {
pExpr->_node.optr = tbufReadUint8(pBuf);
pExpr->_node.hasPK = tbufReadUint8(pBuf);
pExpr->_node.pLeft = exprTreeFromBinaryImpl(pBuf);
pExpr->_node.pRight = exprTreeFromBinaryImpl(pBuf);
pExpr->_node.optr = tbufReadUint8(br);
pExpr->_node.hasPK = tbufReadUint8(br);
pExpr->_node.pLeft = exprTreeFromBinaryImpl(br);
pExpr->_node.pRight = exprTreeFromBinaryImpl(br);
assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL);
}
CLEANUP_EXECUTE_TO(anchor, false);
return pExpr;
}
tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) {
tExprNode* exprTreeFromBinary(const void* data, size_t size) {
if (size == 0) {
return NULL;
}
SBuffer rbuf = {0};
tbufBeginRead(&rbuf, pBuf, size);
return exprTreeFromBinaryImpl(&rbuf);
SBufferReader br = tbufInitReader(data, size, false);
return exprTreeFromBinaryImpl(&br);
}
tExprNode* exprTreeFromTableName(const char* tbnameCond) {
@ -1120,23 +1132,18 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
return NULL;
}
tExprNode* expr = calloc(1, sizeof(tExprNode));
if (expr == NULL) {
// TODO:
}
int32_t anchor = CLEANUP_GET_ANCHOR();
tExprNode* expr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL);
expr->nodeType = TSQL_NODE_EXPR;
tExprNode* left = calloc(1, sizeof(tExprNode));
if (left == NULL) {
// TODO:
}
tExprNode* left = exception_calloc(1, sizeof(tExprNode));
expr->_node.pLeft = left;
left->nodeType = TSQL_NODE_COL;
SSchema* pSchema = calloc(1, sizeof(SSchema));
if (pSchema == NULL) {
// TODO:
}
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
left->pSchema = pSchema;
pSchema->type = TSDB_DATA_TYPE_BINARY;
@ -1144,36 +1151,24 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
strcpy(pSchema->name, TSQL_TBNAME_L);
pSchema->colId = -1;
tExprNode* right = calloc(1, sizeof(tExprNode));
if (right == NULL) {
// TODO
}
tExprNode* right = exception_calloc(1, sizeof(tExprNode));
expr->_node.pRight = right;
if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
expr->_node.optr = TSDB_RELATION_LIKE;
tVariant* pVal = calloc(1, sizeof(tVariant));
if (pVal == NULL) {
// TODO:
}
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
right->pVal = pVal;
pVal->nType = TSDB_DATA_TYPE_BINARY;
size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1;
pVal->pz = malloc(len);
if (pVal->pz == NULL) {
// TODO:
}
pVal->pz = exception_malloc(len);
memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len);
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
expr->_node.optr = TSDB_RELATION_IN;
tVariant* pVal = calloc(1, sizeof(tVariant));
if (pVal == NULL) {
// TODO:
}
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
right->pVal = pVal;
pVal->nType = TSDB_DATA_TYPE_ARRAY;
pVal->arr = taosArrayInit(2, sizeof(char*));
@ -1184,7 +1179,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
cond = e + 1;
} else if (*e == ',') {
size_t len = e - cond + 1;
char* p = malloc( len );
char* p = exception_malloc( len );
memcpy(p, cond, len);
p[len - 1] = 0;
cond += len;
@ -1193,12 +1188,13 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
}
if (*cond != 0) {
char* p = strdup( cond );
char* p = exception_strdup( cond );
taosArrayPush(pVal->arr, &p);
}
taosArraySortString(pVal->arr);
}
CLEANUP_EXECUTE_TO(anchor, false);
return expr;
}

58
src/query/src/queryExecutor.c
View File

@ -55,8 +55,8 @@
typedef struct SPointInterpoSupporter {
int32_t numOfCols;
char ** pPrevPoint;
char ** pNextPoint;
SArray* prev;
SArray* next;
} SPointInterpoSupporter;
typedef enum {
@ -1627,7 +1627,7 @@ static bool isFirstLastRowQuery(SQuery *pQuery) {
return false;
}
static bool notHasQueryTimeRange(SQuery *pQuery) {
static UNUSED_FUNC bool notHasQueryTimeRange(SQuery *pQuery) {
return (pQuery->window.skey == 0 && pQuery->window.ekey == INT64_MAX && QUERY_IS_ASC_QUERY(pQuery)) ||
(pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery)));
}
@ -1950,7 +1950,7 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) {
}
}
static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) {
static UNUSED_FUNC void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) {
assert(pCtx->param[index].pz == NULL);
int32_t len = 0;
@ -2015,6 +2015,7 @@ static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t
* @param pInterpoRaw
*/
void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupport) {
#if 0
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery * pQuery = pRuntimeEnv->pQuery;
@ -2024,7 +2025,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI
}
int32_t count = 1;
TSKEY key = *(TSKEY *)pPointInterpSupport->pNextPoint[0];
TSKEY key = *(TSKEY *)pPointInterpSupport->next[0];
if (key == pQuery->window.skey) {
// the queried timestamp has value, return it directly without interpolation
@ -2107,9 +2108,11 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI
}
}
}
#endif
}
void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) {
#if 0
if (isPointInterpoQuery(pQuery)) {
pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES);
pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES);
@ -2136,9 +2139,11 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu
offset += pQuery->colList[i].bytes;
}
}
#endif
}
void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) {
#if 0
if (pPointInterpSupport->numOfCols <= 0 || pPointInterpSupport->pPrevPoint == NULL) {
return;
}
@ -2150,6 +2155,7 @@ void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) {
tfree(pPointInterpSupport->pNextPoint);
pPointInterpSupport->numOfCols = 0;
#endif
}
static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void *pMeterObj) {
@ -4223,7 +4229,11 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery)
.numOfCols = pQuery->numOfCols,
};
if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
// normal query setup the queryhandle here
if (isFirstLastRowQuery(pQuery)) { // in case of last_row query, invoke a different API.
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo);
} else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) {
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo);
}
@ -4285,36 +4295,36 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery)
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
SPointInterpoSupporter interpInfo = {0};
pointInterpSupporterInit(pQuery, &interpInfo);
// SPointInterpoSupporter interpInfo = {0};
// pointInterpSupporterInit(pQuery, &interpInfo);
/*
* in case of last_row query without query range, we set the query timestamp to
* pMeterObj->lastKey. Otherwise, keep the initial query time range unchanged.
* in case of last_row query without query range, we set the query timestamp to be
* STable->lastKey. Otherwise, keep the initial query time range unchanged.
*/
if (isFirstLastRowQuery(pQuery) && notHasQueryTimeRange(pQuery)) {
if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) {
sem_post(&pQInfo->dataReady);
pointInterpSupporterDestroy(&interpInfo);
return TSDB_CODE_SUCCESS;
}
}
// if (isFirstLastRowQuery(pQuery)) {
// if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) {
// sem_post(&pQInfo->dataReady);
// pointInterpSupporterDestroy(&interpInfo);
// return TSDB_CODE_SUCCESS;
// }
// }
/*
* here we set the value for before and after the specified time into the
* parameter for interpolation query
*/
pointInterpSupporterSetData(pQInfo, &interpInfo);
pointInterpSupporterDestroy(&interpInfo);
// pointInterpSupporterSetData(pQInfo, &interpInfo);
// pointInterpSupporterDestroy(&interpInfo);
int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit,
pQuery->precision);
taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0);
// int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit,
// pQuery->precision);
// taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0);
// allocMemForInterpo(pQInfo, pQuery, pMeterObj);
if (!isPointInterpoQuery(pQuery)) {
// if (!isPointInterpoQuery(pQuery)) {
// assert(pQuery->pos >= 0 && pQuery->slot >= 0);
}
// }
// the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value
pQuery->lastKey = pQuery->window.skey;

18
src/query/tests/astTest.cpp
View File

@ -550,11 +550,12 @@ tExprNode* createExpr2() {
void exprSerializeTest1() {
tExprNode* p1 = createExpr1();
SBuffer buf = exprTreeToBinary(p1);
SBufferWriter bw = tbufInitWriter(NULL, false);
exprTreeToBinary(&bw, p1);
size_t size = tbufTell(&buf);
size_t size = tbufTell(&bw);
ASSERT_TRUE(size > 0);
char* b = tbufGetData(&buf, false);
char* b = tbufGetData(&bw, false);
tExprNode* p2 = exprTreeFromBinary(b, size);
ASSERT_EQ(p1->nodeType, p2->nodeType);
@ -581,16 +582,17 @@ void exprSerializeTest1() {
tExprTreeDestroy(&p1, nullptr);
tExprTreeDestroy(&p2, nullptr);
tbufClose(&buf, false);
tbufClose(&bw);
}
void exprSerializeTest2() {
tExprNode* p1 = createExpr2();
SBuffer buf = exprTreeToBinary(p1);
SBufferWriter bw = tbufInitWriter(NULL, false);
exprTreeToBinary(&bw, p1);
size_t size = tbufTell(&buf);
size_t size = tbufTell(&bw);
ASSERT_TRUE(size > 0);
char* b = tbufGetData(&buf, false);
char* b = tbufGetData(&bw, false);
tExprNode* p2 = exprTreeFromBinary(b, size);
ASSERT_EQ(p1->nodeType, p2->nodeType);
@ -625,7 +627,7 @@ void exprSerializeTest2() {
tExprTreeDestroy(&p1, nullptr);
tExprTreeDestroy(&p2, nullptr);
tbufClose(&buf, false);
tbufClose(&bw);
}
} // namespace
TEST(testCase, astTest) {

4
src/rpc/inc/rpcCache.h
View File

@ -22,8 +22,8 @@ extern "C" {
void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer);
void rpcCloseConnCache(void *handle);
void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, int8_t connType);
void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connType);
void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType);
void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType);
#ifdef __cplusplus
}

6
src/rpc/inc/rpcTcp.h
View File

@ -20,12 +20,12 @@
extern "C" {
#endif
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosCleanUpTcpServer(void *param);
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void taosCleanUpTcpClient(void *chandle);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
void taosCloseTcpConnection(void *chandle);
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);

4
src/rpc/inc/rpcUdp.h
View File

@ -22,10 +22,10 @@ extern "C" {
#include "taosdef.h"
void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int, void *fp, void *shandle);
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle);
void taosCleanUpUdpConnection(void *handle);
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port);
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
void taosFreeMsgHdr(void *hdr);
int taosMsgHdrSize(void *hdr);

32
src/rpc/src/rpcCache.c
View File

@ -23,7 +23,7 @@
#include "rpcCache.h"
typedef struct SConnHash {
uint32_t ip;
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
char connType;
struct SConnHash *prev;
@ -46,7 +46,7 @@ typedef struct {
int64_t *lockedBy;
} SConnCache;
static int rpcHashConn(void *handle, uint32_t ip, uint16_t port, int8_t connType);
static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType);
static void rpcLockCache(int64_t *lockedBy);
static void rpcUnlockCache(int64_t *lockedBy);
static void rpcCleanConnCache(void *handle, void *tmrId);
@ -114,7 +114,7 @@ void rpcCloseConnCache(void *handle) {
free(pCache);
}
void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, int8_t connType) {
void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) {
int hash;
SConnHash * pNode;
SConnCache *pCache;
@ -125,9 +125,9 @@ void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, i
assert(pCache);
assert(data);
hash = rpcHashConn(pCache, ip, port, connType);
hash = rpcHashConn(pCache, fqdn, port, connType);
pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool);
pNode->ip = ip;
strcpy(pNode->fqdn, fqdn);
pNode->port = port;
pNode->connType = connType;
pNode->data = data;
@ -147,12 +147,12 @@ void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, i
pCache->total++;
tTrace("%p ip:0x%x:%hu:%d:%d:%p added into cache, connections:%d", data, ip, port, connType, hash, pNode, pCache->count[hash]);
tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]);
return;
}
void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connType) {
void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType) {
int hash;
SConnHash * pNode;
SConnCache *pCache;
@ -163,7 +163,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT
uint64_t time = taosGetTimestampMs();
hash = rpcHashConn(pCache, ip, port, connType);
hash = rpcHashConn(pCache, fqdn, port, connType);
rpcLockCache(pCache->lockedBy+hash);
pNode = pCache->connHashList[hash];
@ -174,7 +174,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT
break;
}
if (pNode->ip == ip && pNode->port == port && pNode->connType == connType) break;
if (strcmp(pNode->fqdn, fqdn) == 0 && pNode->port == port && pNode->connType == connType) break;
pNode = pNode->next;
}
@ -201,7 +201,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT
rpcUnlockCache(pCache->lockedBy+hash);
if (pData) {
tTrace("%p ip:0x%x:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, ip, port, connType, hash, pNode, pCache->count[hash]);
tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]);
}
return pData;
@ -239,7 +239,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
pNext = pNode->next;
pCache->total--;
pCache->count[hash]--;
tTrace("%p ip:0x%x:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->ip, pNode->port, pNode->connType, hash, pNode,
tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode,
pCache->count[hash]);
taosMemPoolFree(pCache->connHashMemPool, (char *)pNode);
pNode = pNext;
@ -251,12 +251,16 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
pCache->connHashList[hash] = NULL;
}
static int rpcHashConn(void *handle, uint32_t ip, uint16_t port, int8_t connType) {
static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) {
SConnCache *pCache = (SConnCache *)handle;
int hash = 0;
char *temp = fqdn;
while (*temp) {
hash += *temp;
++temp;
}
hash = ip >> 16;
hash += (unsigned short)(ip & 0xFFFF);
hash += port;
hash += connType;

61
src/rpc/src/rpcMain.c
View File

@ -44,7 +44,6 @@ typedef struct {
int sessions; // number of sessions allowed
int numOfThreads; // number of threads to process incoming messages
int idleTime; // milliseconds;
char localIp[TSDB_IPv4ADDR_LEN];
uint16_t localPort;
int8_t connType;
int index; // for UDP server only, round robin for multiple threads
@ -101,9 +100,8 @@ typedef struct SRpcConn {
uint16_t localPort; // for UDP only
uint32_t linkUid; // connection unique ID assigned by client
uint32_t peerIp; // peer IP
uint32_t destIp; // server destination IP to handle NAT
uint16_t peerPort; // peer port
char peerIpstr[TSDB_IPv4ADDR_LEN]; // peer IP string
char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string
uint16_t tranId; // outgoing transcation ID, for build message
uint16_t outTranId; // outgoing transcation ID
uint16_t inTranId; // transcation ID for incoming msg
@ -140,7 +138,7 @@ int tsRpcOverhead;
#define RPC_CONN_TCPC 3
#define RPC_CONN_TCP 2
void *(*taosInitConn[])(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
taosInitUdpConnection,
taosInitUdpConnection,
taosInitTcpServer,
@ -161,7 +159,7 @@ int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *cha
taosSendTcpData
};
void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = {
void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = {
taosOpenUdpConnection,
taosOpenUdpConnection,
NULL,
@ -175,7 +173,7 @@ void (*taosCloseConn[])(void *chandle) = {
taosCloseTcpConnection
};
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType);
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType);
static void rpcCloseConn(void *thandle);
static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext);
static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc);
@ -217,7 +215,6 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime;
pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
if (pInit->localIp) strcpy(pRpc->localIp, pInit->localIp);
pRpc->localPort = pInit->localPort;
pRpc->afp = pInit->afp;
pRpc->sessions = pInit->sessions;
@ -229,13 +226,13 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp;
pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(pRpc->localIp, pRpc->localPort, pRpc->label,
pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label,
pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
pRpc->udphandle = (*taosInitConn[pRpc->connType])(pRpc->localIp, pRpc->localPort, pRpc->label,
pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label,
pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) {
tError("%s failed to init network, %s:%d", pRpc->label, pRpc->localIp, pRpc->localPort);
tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort);
rpcClose(pRpc);
return NULL;
}
@ -457,7 +454,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
pInfo->clientIp = pConn->peerIp;
pInfo->clientPort = pConn->peerPort;
pInfo->serverIp = pConn->destIp;
// pInfo->serverIp = pConn->destIp;
strcpy(pInfo->user, pConn->user);
return 0;
@ -490,27 +487,32 @@ static void rpcFreeMsg(void *msg) {
}
}
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType) {
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) {
SRpcConn *pConn;
uint32_t peerIp = taosGetIpFromFqdn(peerFqdn);
if (peerIp == -1) {
tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn);
return NULL;
}
pConn = rpcAllocateClientConn(pRpc);
if (pConn) {
strcpy(pConn->peerIpstr, peerIpStr);
pConn->peerIp = inet_addr(peerIpStr);
strcpy(pConn->peerFqdn, peerFqdn);
pConn->peerIp = peerIp;
pConn->peerPort = peerPort;
strcpy(pConn->user, pRpc->user);
pConn->connType = connType;
if (taosOpenConn[connType]) {
void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle;
pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIpstr, pConn->peerPort);
pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort);
if (pConn->chandle) {
tTrace("%s %p, rpc connection is set up, sid:%d id:%s ip:%s:%hu connType:%d", pRpc->label,
pConn, pConn->sid, pRpc->user, pConn->peerIpstr, pConn->peerPort, pConn->connType);
tTrace("%s %p, rpc connection is set up, sid:%d id:%s %s:%hu connType:%d", pRpc->label,
pConn, pConn->sid, pRpc->user, peerFqdn, pConn->peerPort, pConn->connType);
} else {
tError("%s %p, failed to set up connection to ip:%s:%hu", pRpc->label, pConn,
pConn->peerIpstr, pConn->peerPort);
tError("%s %p, failed to set up connection to %s:%hu", pRpc->label, pConn, peerFqdn, pConn->peerPort);
terrno = TSDB_CODE_NETWORK_UNAVAIL;
rpcCloseConn(pConn);
pConn = NULL;
@ -661,12 +663,9 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
SRpcInfo *pRpc = pContext->pRpc;
SRpcIpSet *pIpSet = &pContext->ipSet;
pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->ip[pIpSet->inUse], pIpSet->port, pContext->connType);
pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType);
if ( pConn == NULL || pConn->user[0] == 0) {
char ipstr[20] = {0};
tinet_ntoa(ipstr, pIpSet->ip[pIpSet->inUse]);
pConn = rpcOpenConn(pRpc, ipstr, pIpSet->port, pContext->connType);
if (pConn) pConn->destIp = pIpSet->ip[pIpSet->inUse];
pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType);
} else {
tTrace("%s %p, connection is retrieved from cache", pRpc->label, pConn);
}
@ -789,7 +788,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
pConn->peerIp = pRecv->ip;
char ipstr[20] = {0};
tinet_ntoa(ipstr, pRecv->ip);
strcpy(pConn->peerIpstr, ipstr);
strcpy(pConn->peerFqdn, ipstr);
}
if (pRecv->port) pConn->peerPort = pRecv->port;
@ -922,7 +921,6 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
if ( rpcIsReq(pHead->msgType) ) {
rpcMsg.handle = pConn;
pConn->destIp = pHead->destIp;
taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer);
(*(pRpc->cfp))(&rpcMsg);
} else {
@ -932,7 +930,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
pConn->pContext = NULL;
// for UDP, port may be changed by server, the port in ipSet shall be used for cache
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerIp, pContext->ipSet.port, pConn->connType);
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType);
if (pHead->code == TSDB_CODE_REDIRECT) {
pContext->redirect = 1;
@ -1053,7 +1051,6 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
pHead->tranId = pConn->tranId;
pHead->sourceId = pConn->ownId;
pHead->destId = pConn->peerId;
pHead->destIp = pConn->destIp;
pHead->port = 0;
pHead->linkUid = pConn->linkUid;
if (!pConn->secured) memcpy(pHead->user, pConn->user, tListLen(pHead->user));
@ -1081,12 +1078,12 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
if ( rpcIsReq(pHead->msgType)) {
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
tTrace("%s %p, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr,
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn,
pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
} else {
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, pConn->peerPort,
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort,
htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
}
@ -1141,13 +1138,13 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
if (pConn->retry < 4) {
tTrace("%s %p, re-send msg:%s to %s:%hud", pRpc->label, pConn,
taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort);
taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen);
taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer);
} else {
// close the connection
tTrace("%s %p, failed to send msg:%s to %s:%hu", pRpc->label, pConn,
taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort);
taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
reportDisc = 1;
}
} else {

26
src/rpc/src/rpcTcp.c
View File

@ -40,7 +40,7 @@ typedef struct SThreadObj {
SFdObj * pHead;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
char ipstr[TSDB_IPv4ADDR_LEN];
uint32_t ip;
int pollFd;
int numOfFds;
int threadId;
@ -50,7 +50,7 @@ typedef struct SThreadObj {
} SThreadObj;
typedef struct {
char ip[TSDB_IPv4ADDR_LEN];
uint32_t ip;
uint16_t port;
char label[12];
int numOfThreads;
@ -65,12 +65,12 @@ static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
SServerObj *pServerObj;
SThreadObj *pThreadObj;
pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
strcpy(pServerObj->ip, ip);
pServerObj->ip = ip;
pServerObj->port = port;
strcpy(pServerObj->label, label);
pServerObj->numOfThreads = numOfThreads;
@ -138,7 +138,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads,
free(pServerObj);
pServerObj = NULL;
} else {
tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
tTrace("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
}
return (void *)pServerObj;
@ -222,14 +222,14 @@ static void taosAcceptTcpConnection(void *arg) {
}
}
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
SThreadObj *pThreadObj;
pthread_attr_t thattr;
pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj));
memset(pThreadObj, 0, sizeof(SThreadObj));
strcpy(pThreadObj->label, label);
strcpy(pThreadObj->ipstr, ip);
pThreadObj->ip = ip;
pThreadObj->shandle = shandle;
if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) {
@ -284,21 +284,19 @@ void taosCleanUpTcpClient(void *chandle) {
tfree(pThreadObj);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
SThreadObj * pThreadObj = shandle;
struct in_addr destIp;
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr);
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
if (fd <= 0) return NULL;
inet_aton(ip, &destIp);
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = destIp.s_addr;
tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d",
pFdObj->ip = ip;
tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
} else {
close(fd);
@ -403,7 +401,7 @@ static void *taosProcessTcpData(void *param) {
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen);
// tTrace("%s TCP data is received, ip:0x%x:%u len:%d", pThreadObj->label, pFdObj->ip, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;

14
src/rpc/src/rpcUdp.c
View File

@ -51,7 +51,7 @@ typedef struct {
typedef struct {
int index;
int server;
char ip[16]; // local IP
uint32_t ip; // local IP
uint16_t port; // local Port
void *shandle; // handle passed by upper layer during server initialization
int threads;
@ -77,7 +77,7 @@ static void *taosRecvUdpData(void *param);
static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port);
static void taosProcessUdpBufTimer(void *param, void *tmrId);
void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
SUdpConn *pConn;
SUdpConnSet *pSet;
@ -89,7 +89,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
}
memset(pSet, 0, (size_t)size);
strcpy(pSet->ip, ip);
pSet->ip = ip;
pSet->port = port;
pSet->shandle = shandle;
pSet->fp = fp;
@ -111,7 +111,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
ownPort = (port ? port + i : 0);
pConn->fd = taosOpenUdpSocket(ip, ownPort);
if (pConn->fd < 0) {
tError("%s failed to open UDP socket %s:%hu", label, ip, port);
tError("%s failed to open UDP socket %x:%hu", label, ip, port);
taosCleanUpUdpConnection(pSet);
return NULL;
}
@ -157,7 +157,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
++pSet->threads;
}
tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads);
tTrace("%s UDP connection is initialized, ip:%x port:%hu threads:%d", label, ip, port, threads);
return pSet;
}
@ -190,7 +190,7 @@ void taosCleanUpUdpConnection(void *handle) {
tfree(pSet);
}
void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
SUdpConnSet *pSet = (SUdpConnSet *)shandle;
pSet->index = (pSet->index + 1) % pSet->threads;
@ -198,7 +198,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por
SUdpConn *pConn = pSet->udpConn + pSet->index;
pConn->port = port;
tTrace("%s UDP connection is setup, ip: %s:%hu, local: %s:%d", pConn->label, ip, port, pSet->ip,
tTrace("%s UDP connection is setup, ip:%x:%hu, local:%x:%d", pConn->label, ip, port, pSet->ip,
ntohs((uint16_t)pConn->localPort));
return pConn;

17
src/rpc/test/rclient.c
View File

@ -88,13 +88,13 @@ int main(int argc, char *argv[]) {
// server info
ipSet.numOfIps = 1;
ipSet.inUse = 0;
ipSet.port = 7000;
ipSet.ip[0] = inet_addr(serverIp);
ipSet.ip[1] = inet_addr("192.168.0.1");
ipSet.port[0] = 7000;
ipSet.port[1] = 7000;
strcpy(ipSet.fqdn[0], serverIp);
strcpy(ipSet.fqdn[1], "192.168.0.1");
// client info
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = "0.0.0.0";
rpcInit.localPort = 0;
rpcInit.label = "APP";
rpcInit.numOfThreads = 1;
@ -110,11 +110,9 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port = atoi(argv[++i]);
ipSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
ipSet.ip[0] = inet_addr(argv[++i]);
} else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
strcpy(rpcInit.localIp, argv[++i]);
strcpy(ipSet.fqdn[0], argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -138,10 +136,9 @@ int main(int argc, char *argv[]) {
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-l localIp]: local IP address, default is:%s\n", rpcInit.localIp);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-a threads]: number of app threads, default is:%d\n", appThreads);
printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);

18
src/rpc/test/rsclient.c
View File

@ -89,13 +89,14 @@ int main(int argc, char *argv[]) {
// server info
ipSet.numOfIps = 1;
ipSet.inUse = 0;
ipSet.port = 7000;
ipSet.ip[0] = inet_addr(serverIp);
ipSet.ip[1] = inet_addr("192.168.0.1");
ipSet.port[0] = 7000;
ipSet.port[1] = 7000;
strcpy(ipSet.fqdn[0], serverIp);
strcpy(ipSet.fqdn[1], "192.168.0.1");
// client info
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = "0.0.0.0";
//rpcInit.localIp = "0.0.0.0";
rpcInit.localPort = 0;
rpcInit.label = "APP";
rpcInit.numOfThreads = 1;
@ -111,11 +112,9 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port = atoi(argv[++i]);
ipSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
ipSet.ip[0] = inet_addr(argv[++i]);
} else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
strcpy(rpcInit.localIp, argv[++i]);
strcpy(ipSet.fqdn[0], argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -139,10 +138,9 @@ int main(int argc, char *argv[]) {
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-l localIp]: local IP address, default is:%s\n", rpcInit.localIp);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-a threads]: number of app threads, default is:%d\n", appThreads);
printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);

5
src/rpc/test/rserver.c
View File

@ -126,10 +126,8 @@ void processRequestMsg(SRpcMsg *pMsg) {
int main(int argc, char *argv[]) {
SRpcInit rpcInit;
char dataName[20] = "server.data";
char localIp[40] = "0.0.0.0";
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = localIp;
rpcInit.localPort = 7000;
rpcInit.label = "SER";
rpcInit.numOfThreads = 1;
@ -141,8 +139,6 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
rpcInit.localPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i")==0 && i < argc-1) {
strcpy(rpcInit.localIp, argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -159,7 +155,6 @@ int main(int argc, char *argv[]) {
uDebugFlag = rpcDebugFlag;
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: server IP address, default is:%s\n", rpcInit.localIp);
printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);

6
src/tsdb/inc/tsdbMain.h
View File

@ -74,9 +74,12 @@ typedef struct STable {
void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index
void * eventHandler; // TODO
void * streamHandler; // TODO
TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure
struct STable *next; // TODO: remove the next
} STable;
#define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey)
void * tsdbEncodeTable(STable *pTable, int *contLen);
STable *tsdbDecodeTable(void *cont, int contLen);
void tsdbFreeEncode(void *cont);
@ -129,9 +132,6 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid);
char * getTupleKey(const void *data);
// ------------------------------ TSDB CACHE INTERFACES ------------------------------
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */
typedef struct {
int blockId;
int offset;

3
src/tsdb/src/tsdbCache.c
View File

@ -26,6 +26,9 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) {
if (pCache == NULL) return NULL;
if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
cacheBlockSize *= (1024 * 1024);
if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS;
pCache->maxBytes = maxBytes;
pCache->cacheBlockSize = cacheBlockSize;

70
src/tsdb/src/tsdbMain.c
View File

@ -1,4 +1,5 @@
#include "os.h"
#include "taosdef.h"
#include "tulog.h"
#include "talgo.h"
#include "tsdb.h"
@ -11,24 +12,6 @@
#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
#define TSDB_MIN_ID 0
#define TSDB_MAX_ID INT_MAX
#define TSDB_MIN_TABLES 4
#define TSDB_MAX_TABLES 100000
#define TSDB_DEFAULT_TABLES 1000
#define TSDB_DEFAULT_DAYS_PER_FILE 10
#define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 60
#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100
#define TSDB_MIN_MIN_ROW_FBLOCK 10
#define TSDB_MAX_MIN_ROW_FBLOCK 1000
#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096
#define TSDB_MIN_MAX_ROW_FBLOCK 200
#define TSDB_MAX_MAX_ROW_FBLOCK 10000
#define TSDB_DEFAULT_KEEP 3650
#define TSDB_MIN_KEEP 1
#define TSDB_MAX_KEEP INT_MAX
#define TSDB_DEFAULT_CACHE_SIZE (16 * 1024 * 1024) // 16M
#define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024) // 4M
#define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024) // 1G
#define TSDB_CFG_FILE_NAME "CONFIG"
#define TSDB_DATA_DIR_NAME "data"
@ -70,7 +53,6 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) {
pCfg->minRowsPerFileBlock = -1;
pCfg->maxRowsPerFileBlock = -1;
pCfg->keep = -1;
pCfg->maxCacheSize = -1;
pCfg->compression = TWO_STAGE_COMP;
}
@ -163,6 +145,34 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) {
return 0;
}
static int tsdbRestoreInfo(STsdbRepo *pRepo) {
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
SFileGroup *pFGroup = NULL;
SFileGroupIter iter;
SRWHelper rhelper = {0};
if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err;
tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
for (int i = 0; i < pRepo->config.maxTables; i++) {
STable * pTable = pMeta->tables[i];
SCompIdx *pIdx = &rhelper.pCompIdx[i];
if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey;
}
}
tsdbDestroyHelper(&rhelper);
return 0;
_err:
tsdbDestroyHelper(&rhelper);
return -1;
}
/**
* Open an existing TSDB storage repository
* @param tsdbDir the existing TSDB root directory
@ -192,7 +202,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
return NULL;
}
pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo);
pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo);
if (pRepo->tsdbCache == NULL) {
tsdbFreeMeta(pRepo->tsdbMeta);
free(pRepo->rootDir);
@ -210,6 +220,16 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
return NULL;
}
// Restore key from file
if (tsdbRestoreInfo(pRepo) < 0) {
tsdbFreeCache(pRepo->tsdbCache);
tsdbFreeMeta(pRepo->tsdbMeta);
tsdbCloseFileH(pRepo->tsdbFileH);
free(pRepo->rootDir);
free(pRepo);
return NULL;
}
pRepo->state = TSDB_REPO_STATE_ACTIVE;
return (TsdbRepoT *)pRepo;
@ -612,13 +632,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) return -1;
}
// Check maxCacheSize
if (pCfg->maxCacheSize == -1) {
pCfg->maxCacheSize = TSDB_DEFAULT_CACHE_SIZE;
} else {
if (pCfg->maxCacheSize < TSDB_MIN_CACHE_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_SIZE) return -1;
}
return 0;
}
@ -755,6 +768,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
tSkipListPut(pTable->mem->pData, pNode);
if (key > pTable->mem->keyLast) pTable->mem->keyLast = key;
if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key;
if (key > pTable->lastKey) pTable->lastKey = key;
pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData);
@ -901,7 +915,7 @@ _exit:
}
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) {
char dataDir[128] = {0};
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
STsdbCfg * pCfg = &pRepo->config;

1
src/tsdb/src/tsdbMeta.c
View File

@ -315,6 +315,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
table->tableId = pCfg->tableId;
table->name = strdup(pCfg->name);
table->lastKey = 0;
if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE
table->type = TSDB_CHILD_TABLE;
table->superUid = pCfg->superUid;

119
src/tsdb/src/tsdbRead.c
View File

@ -18,6 +18,7 @@
#include "talgo.h"
#include "tutil.h"
#include "tcompare.h"
#include "exception.h"
#include "../../../query/inc/qast.h" // todo move to common module
#include "../../../query/inc/tlosertree.h" // todo move to util module
@ -25,7 +26,6 @@
#include "tsdbMain.h"
#define EXTRA_BYTES 2
#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)
#define ASCENDING_ORDER_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
@ -34,6 +34,11 @@ enum {
QUERY_RANGE_GREATER_EQUAL = 1,
};
enum {
TSDB_QUERY_TYPE_ALL_ROWS = 1,
TSDB_QUERY_TYPE_LAST_ROW = 2,
};
typedef struct SField {
// todo need the definition
} SField;
@ -66,8 +71,8 @@ typedef struct STableCheckInfo {
SCompInfo* pCompInfo;
int32_t compSize;
int32_t numOfBlocks; // number of qualified data blocks not the original blocks
SDataCols* pDataCols;
SDataCols* pDataCols;
SSkipListIterator* iter;
} STableCheckInfo;
@ -105,11 +110,11 @@ typedef struct STsdbQueryHandle {
SArray* pColumns; // column list, SColumnInfoData array list
bool locateStart;
int32_t realNumOfRows;
SArray* pTableCheckInfo;
SArray* pTableCheckInfo; //SArray<STableCheckInfo>
int32_t activeIndex;
bool checkFiles; // check file stage
void* qinfo; // query info handle, for debug purpose
int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
STableBlockInfo* pDataBlockInfo;
SFileGroup* pFileGroup;
@ -117,6 +122,8 @@ typedef struct STsdbQueryHandle {
SRWHelper rhelper;
} STsdbQueryHandle;
static void changeQueryHandleForQuery(TsdbQueryHandleT pqHandle);
static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
pBlockLoadInfo->slot = -1;
pBlockLoadInfo->sid = -1;
@ -191,7 +198,17 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
return (TsdbQueryHandleT)pQueryHandle;
return (TsdbQueryHandleT) pQueryHandle;
}
TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList);
pQueryHandle->type = TSDB_QUERY_TYPE_LAST_ROW;
pQueryHandle->order = TSDB_ORDER_DESC;
changeQueryHandleForQuery(pQueryHandle);
return pQueryHandle;
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
@ -932,6 +949,54 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pqHandle) {
}
}
void changeQueryHandleForQuery(TsdbQueryHandleT pqHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle;
assert(!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order));
// starts from the buffer in case of descending timestamp order check data blocks
// todo consider the query time window, current last_row does not apply the query time window
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
TSKEY key = INT64_MIN;
int32_t index = -1;
for(int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
if (pCheckInfo->pTableObj->lastKey > key) {
key = pCheckInfo->pTableObj->lastKey;
index = i;
}
}
// erase all other elements in array list, todo refactor
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) {
if (i == index) {
continue;
}
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
tSkipListDestroyIter(pTableCheckInfo->iter);
if (pTableCheckInfo->pDataCols != NULL) {
tfree(pTableCheckInfo->pDataCols->buf);
}
tfree(pTableCheckInfo->pDataCols);
tfree(pTableCheckInfo->pCompInfo);
}
STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index);
taosArrayDestroy(pQueryHandle->pTableCheckInfo);
pQueryHandle->pTableCheckInfo = taosArrayInit(1, sizeof(STableCheckInfo));
taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
// update the query time window according to the chosen last timestamp
pQueryHandle->window = (STimeWindow) {key, key};
}
static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey,
STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0;
@ -1101,10 +1166,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
}
}
int32_t tsdbResetQuery(TsdbQueryHandleT* pQueryHandle, STimeWindow* window, TsdbPosT position, int16_t order) {
return 0;
}
SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; }
TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) {
@ -1412,7 +1473,6 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
return true;
}
static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) {
// query according to the binary expression
STSchema* pSchema = pSTable->tagSchema;
@ -1440,7 +1500,6 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
return TSDB_CODE_SUCCESS;
}
int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, int16_t tagNameRelType,
const char* tbnameCond, STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols) {
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@ -1464,21 +1523,35 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC
}
int32_t ret = TSDB_CODE_SUCCESS;
tExprNode* expr = NULL;
tExprNode* expr = exprTreeFromTableName(tbnameCond);
tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len);
if (tagExpr != NULL) {
TRY(32) {
expr = exprTreeFromTableName(tbnameCond);
if (expr == NULL) {
expr = tagExpr;
expr = exprTreeFromBinary(pTagCond, len);
} else {
tExprNode* tbnameExpr = expr;
expr = calloc(1, sizeof(tExprNode));
expr->nodeType = TSQL_NODE_EXPR;
expr->_node.optr = tagNameRelType;
expr->_node.pLeft = tagExpr;
expr->_node.pRight = tbnameExpr;
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL);
tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len);
if (tagExpr != NULL) {
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, tagExpr, NULL);
tExprNode* tbnameExpr = expr;
expr = calloc(1, sizeof(tExprNode));
if (expr == NULL) {
THROW( TSDB_CODE_SERV_OUT_OF_MEMORY );
}
expr->nodeType = TSQL_NODE_EXPR;
expr->_node.optr = tagNameRelType;
expr->_node.pLeft = tagExpr;
expr->_node.pRight = tbnameExpr;
}
}
}
CLEANUP_EXECUTE();
} CATCH( code ) {
CLEANUP_EXECUTE();
ret = code;
// TODO: more error handling
} END_TRY
doQueryTableList(pTable, res, expr);
pGroupInfo->numOfTables = taosArrayGetSize(res);

124
src/util/inc/exception.h Normal file
View File

@ -0,0 +1,124 @@
/*
* Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_EXCEPTION_H
#define TDENGINE_EXCEPTION_H
#include <setjmp.h>
#include <stdint.h>
#include <stdbool.h>
#include <assert.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* cleanup actions
*/
typedef struct SCleanupAction {
bool failOnly;
uint8_t wrapper;
uint16_t reserved;
void* func;
union {
void* Ptr;
bool Bool;
char Char;
int8_t Int8;
uint8_t Uint8;
int16_t Int16;
uint16_t Uint16;
int Int;
unsigned int Uint;
int32_t Int32;
uint32_t Uint32;
int64_t Int64;
uint64_t Uint64;
float Float;
double Double;
} arg1, arg2;
} SCleanupAction;
/*
* exception hander registration
*/
typedef struct SExceptionNode {
struct SExceptionNode* prev;
jmp_buf jb;
int32_t code;
int32_t maxCleanupAction;
int32_t numCleanupAction;
SCleanupAction* cleanupActions;
} SExceptionNode;
////////////////////////////////////////////////////////////////////////////////
// functions & macros for auto-cleanup
void cleanupPush_void_ptr_ptr ( bool failOnly, void* func, void* arg1, void* arg2 );
void cleanupPush_void_ptr_bool ( bool failOnly, void* func, void* arg1, bool arg2 );
void cleanupPush_void_ptr ( bool failOnly, void* func, void* arg );
void cleanupPush_int_int ( bool failOnly, void* func, int arg );
void cleanupPush_void ( bool failOnly, void* func );
int32_t cleanupGetActionCount();
void cleanupExecuteTo( int32_t anchor, bool failed );
void cleanupExecute( SExceptionNode* node, bool failed );
#define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) )
#define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) )
#define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) )
#define CLEANUP_PUSH_INT_INT( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) )
#define CLEANUP_PUSH_VOID( failOnly, func ) cleanupPush_void( (failOnly), (void*)(func) )
#define CLEANUP_PUSH_FREE( failOnly, arg ) cleanupPush_void_ptr( (failOnly), free, (void*)(arg) )
#define CLEANUP_PUSH_CLOSE( failOnly, arg ) cleanupPush_int_int( (failOnly), close, (int)(arg) )
#define CLEANUP_GET_ANCHOR() cleanupGetActionCount()
#define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) )
////////////////////////////////////////////////////////////////////////////////
// functions & macros for exception handling
void exceptionPushNode( SExceptionNode* node );
int32_t exceptionPopNode();
void exceptionThrow( int code );
#define TRY(maxCleanupActions) do { \
SExceptionNode exceptionNode = { 0 }; \
SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \
exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \
exceptionNode.cleanupActions = cleanupActions; \
exceptionPushNode( &exceptionNode ); \
int caughtException = setjmp( exceptionNode.jb ); \
if( caughtException == 0 )
#define CATCH( code ) int code = exceptionPopNode(); \
if( caughtException == 1 )
#define FINALLY( code ) int code = exceptionPopNode();
#define END_TRY } while( 0 );
#define THROW( x ) exceptionThrow( (x) )
#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1))
#define CLEANUP_EXECUTE() cleanupExecute( &exceptionNode, CAUGHT_EXCEPTION() )
#ifdef __cplusplus
}
#endif
#endif

227
src/util/inc/tbuffer.h
View File

@ -16,122 +16,163 @@
#ifndef TDENGINE_TBUFFER_H
#define TDENGINE_TBUFFER_H
#include "setjmp.h"
#include "os.h"
#include <stdint.h>
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
////////////////////////////////////////////////////////////////////////////////
// usage example
/*
SBuffer can be used to read or write a buffer, but cannot be used for both
read & write at a same time. Below is an example:
#include <stdio.h>
#include "exception.h"
int main(int argc, char** argv) {
//--------------------- write ------------------------
SBuffer wbuf;
int32_t code = tbufBeginWrite(&wbuf);
if (code != 0) {
// handle errors
return 0;
}
int main( int argc, char** argv ) {
SBufferWriter bw = tbufInitWriter( NULL, false );
// reserve 1024 bytes for the buffer to improve performance
tbufEnsureCapacity(&wbuf, 1024);
TRY( 1 ) {
//--------------------- write ------------------------
// reserve 1024 bytes for the buffer to improve performance
tbufEnsureCapacity( &bw, 1024 );
// write 5 integers to the buffer
for (int i = 0; i < 5; i++) {
tbufWriteInt32(&wbuf, i);
}
// reserve space for the interger count
size_t pos = tbufReserve( &bw, sizeof(int32_t) );
// write 5 integers to the buffer
for( int i = 0; i < 5; i++) {
tbufWriteInt32( &bw, i );
}
// write the integer count to buffer at reserved position
tbufWriteInt32At( &bw, pos, 5 );
// write a string to the buffer
tbufWriteString(&wbuf, "this is a string.\n");
// write a string to the buffer
tbufWriteString( &bw, "this is a string.\n" );
// acquire the result and close the write buffer
size_t size = tbufTell( &bw );
char* data = tbufGetData( &bw, false );
// acquire the result and close the write buffer
size_t size = tbufTell(&wbuf);
char* data = tbufGetData(&wbuf, true);
tbufClose(&wbuf, true);
//------------------------ read -----------------------
SBuffer rbuf;
code = tbufBeginRead(&rbuf, data, size);
if (code != 0) {
printf("you will see this message after print out 5 integers and a string.\n");
tbufClose(&rbuf, false);
return 0;
}
// read & print out 5 integers
for (int i = 0; i < 5; i++) {
printf("%d\n", tbufReadInt32(&rbuf));
}
// read & print out a string
printf(tbufReadString(&rbuf, NULL));
// try read another integer, this result in an error as there no this integer
tbufReadInt32(&rbuf);
printf("you should not see this message.\n");
tbufClose(&rbuf, false);
//------------------------ read -----------------------
SBufferReader br = tbufInitReader( data, size, false );
// read & print out all integers
int32_t count = tbufReadInt32( &br );
for( int i = 0; i < count; i++ ) {
printf( "%d\n", tbufReadInt32(&br) );
}
// read & print out a string
puts( tbufReadString(&br, NULL) );
// try read another integer, this result in an error as there no this integer
tbufReadInt32( &br );
printf( "you should not see this message.\n" );
} CATCH( code ) {
printf( "exception code is: %d, you will see this message after print out 5 integers and a string.\n", code );
} END_TRY
tbufCloseWriter( &bw );
return 0;
}
*/
typedef struct {
jmp_buf jb;
char* data;
size_t pos;
size_t size;
} SBuffer;
bool endian;
const char* data;
size_t pos;
size_t size;
} SBufferReader;
// common functions can be used in both read & write
#define tbufThrowError(buf, code) longjmp((buf)->jb, (code))
size_t tbufTell(SBuffer* buf);
size_t tbufSeekTo(SBuffer* buf, size_t pos);
size_t tbufSkip(SBuffer* buf, size_t size);
void tbufClose(SBuffer* buf, bool keepData);
typedef struct {
bool endian;
char* data;
size_t pos;
size_t size;
void* (*allocator)( void*, size_t );
} SBufferWriter;
// basic read functions
#define tbufBeginRead(buf, _data, len) ((buf)->data = (char*)(_data), ((buf)->pos = 0), ((buf)->size = ((_data) == NULL) ? 0 : (len)), setjmp((buf)->jb))
char* tbufRead(SBuffer* buf, size_t size);
void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size);
const char* tbufReadString(SBuffer* buf, size_t* len);
size_t tbufReadToString(SBuffer* buf, char* dst, size_t size);
////////////////////////////////////////////////////////////////////////////////
// common functions & macros for both reader & writer
// basic write functions
#define tbufBeginWrite(buf) ((buf)->data = NULL, ((buf)->pos = 0), ((buf)->size = 0), setjmp((buf)->jb))
void tbufEnsureCapacity(SBuffer* buf, size_t size);
char* tbufGetData(SBuffer* buf, bool takeOver);
void tbufWrite(SBuffer* buf, const void* data, size_t size);
void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size);
void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len);
void tbufWriteString(SBuffer* buf, const char* str);
#define tbufTell( buf ) ((buf)->pos)
// read & write function for primitive types
#ifndef TBUFFER_DEFINE_FUNCTION
#define TBUFFER_DEFINE_FUNCTION(type, name) \
type tbufRead##name(SBuffer* buf); \
void tbufWrite##name(SBuffer* buf, type data); \
void tbufWrite##name##At(SBuffer* buf, size_t pos, type data);
#endif
TBUFFER_DEFINE_FUNCTION(bool, Bool)
TBUFFER_DEFINE_FUNCTION(char, Char)
TBUFFER_DEFINE_FUNCTION(int8_t, Int8)
TBUFFER_DEFINE_FUNCTION(uint8_t, Uint8)
TBUFFER_DEFINE_FUNCTION(int16_t, Int16)
TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16)
TBUFFER_DEFINE_FUNCTION(int32_t, Int32)
TBUFFER_DEFINE_FUNCTION(uint32_t, Uint32)
TBUFFER_DEFINE_FUNCTION(int64_t, Int64)
TBUFFER_DEFINE_FUNCTION(uint64_t, Uint64)
TBUFFER_DEFINE_FUNCTION(float, Float)
TBUFFER_DEFINE_FUNCTION(double, Double)
////////////////////////////////////////////////////////////////////////////////
// reader functions & macros
// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically
#define tbufInitReader( Data, Size, Endian ) {.endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 :(Size))}
size_t tbufSkip( SBufferReader* buf, size_t size );
const char* tbufRead( SBufferReader* buf, size_t size );
void tbufReadToBuffer( SBufferReader* buf, void* dst, size_t size );
const char* tbufReadString( SBufferReader* buf, size_t* len );
size_t tbufReadToString( SBufferReader* buf, char* dst, size_t size );
const char* tbufReadBinary( SBufferReader* buf, size_t *len );
size_t tbufReadToBinary( SBufferReader* buf, void* dst, size_t size );
bool tbufReadBool( SBufferReader* buf );
char tbufReadChar( SBufferReader* buf );
int8_t tbufReadInt8( SBufferReader* buf );
uint8_t tbufReadUint8( SBufferReader* buf );
int16_t tbufReadInt16( SBufferReader* buf );
uint16_t tbufReadUint16( SBufferReader* buf );
int32_t tbufReadInt32( SBufferReader* buf );
uint32_t tbufReadUint32( SBufferReader* buf );
int64_t tbufReadInt64( SBufferReader* buf );
uint64_t tbufReadUint64( SBufferReader* buf );
float tbufReadFloat( SBufferReader* buf );
double tbufReadDouble( SBufferReader* buf );
////////////////////////////////////////////////////////////////////////////////
// writer functions & macros
// *Allocator*, function to allocate memory, will use 'realloc' if NULL
// *Endian*, if true, writer functions of primitive types will do 'hton' automatically
#define tbufInitWriter( Allocator, Endian ) {.endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator))}
void tbufCloseWriter( SBufferWriter* buf );
void tbufEnsureCapacity( SBufferWriter* buf, size_t size );
size_t tbufReserve( SBufferWriter* buf, size_t size );
char* tbufGetData( SBufferWriter* buf, bool takeOver );
void tbufWrite( SBufferWriter* buf, const void* data, size_t size );
void tbufWriteAt( SBufferWriter* buf, size_t pos, const void* data, size_t size );
void tbufWriteStringLen( SBufferWriter* buf, const char* str, size_t len );
void tbufWriteString( SBufferWriter* buf, const char* str );
// the prototype of tbufWriteBinary and tbufWrite are identical
// the difference is: tbufWriteBinary writes the length of the data to the buffer
// first, then the actual data, which means the reader don't need to know data
// size before read. Write only write the data itself, which means the reader
// need to know data size before read.
void tbufWriteBinary( SBufferWriter* buf, const void* data, size_t len );
void tbufWriteBool( SBufferWriter* buf, bool data );
void tbufWriteBoolAt( SBufferWriter* buf, size_t pos, bool data );
void tbufWriteChar( SBufferWriter* buf, char data );
void tbufWriteCharAt( SBufferWriter* buf, size_t pos, char data );
void tbufWriteInt8( SBufferWriter* buf, int8_t data );
void tbufWriteInt8At( SBufferWriter* buf, size_t pos, int8_t data );
void tbufWriteUint8( SBufferWriter* buf, uint8_t data );
void tbufWriteUint8At( SBufferWriter* buf, size_t pos, uint8_t data );
void tbufWriteInt16( SBufferWriter* buf, int16_t data );
void tbufWriteInt16At( SBufferWriter* buf, size_t pos, int16_t data );
void tbufWriteUint16( SBufferWriter* buf, uint16_t data );
void tbufWriteUint16At( SBufferWriter* buf, size_t pos, uint16_t data );
void tbufWriteInt32( SBufferWriter* buf, int32_t data );
void tbufWriteInt32At( SBufferWriter* buf, size_t pos, int32_t data );
void tbufWriteUint32( SBufferWriter* buf, uint32_t data );
void tbufWriteUint32At( SBufferWriter* buf, size_t pos, uint32_t data );
void tbufWriteInt64( SBufferWriter* buf, int64_t data );
void tbufWriteInt64At( SBufferWriter* buf, size_t pos, int64_t data );
void tbufWriteUint64( SBufferWriter* buf, uint64_t data );
void tbufWriteUint64At( SBufferWriter* buf, size_t pos, uint64_t data );
void tbufWriteFloat( SBufferWriter* buf, float data );
void tbufWriteFloatAt( SBufferWriter* buf, size_t pos, float data );
void tbufWriteDouble( SBufferWriter* buf, double data );
void tbufWriteDoubleAt( SBufferWriter* buf, size_t pos, double data );
#ifdef __cplusplus
}
#endif
#endif
#endif

45
src/util/inc/tsocket.h
View File

@ -20,44 +20,27 @@
extern "C" {
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
int taosNonblockwrite(int fd, char *ptr, int nbytes);
int taosReadn(int sock, char *buffer, int len);
int taosWriteMsg(int fd, void *ptr, int nbytes);
int taosReadMsg(int fd, void *ptr, int nbytes);
int taosNonblockwrite(int fd, char *ptr, int nbytes);
int taosCopyFds(int sfd, int dfd, int64_t len);
int taosSetNonblocking(int sock, int on);
int taosOpenUdpSocket(char *ip, uint16_t port);
int taosOpenTcpClientSocket(char *ip, uint16_t port, char *localIp);
int taosOpenTcpServerSocket(char *ip, uint16_t port);
int taosKeepTcpAlive(int sockFd);
int taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
int taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
int taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int taosKeepTcpAlive(int sockFd);
void taosCloseTcpSocket(int sockFd);
int taosOpenUDServerSocket(char *ip, uint16_t port);
int taosOpenUDServerSocket(uint32_t ip, uint16_t port);
int taosOpenUDClientSocket(uint32_t ip, uint16_t port);
int taosOpenRawSocket(uint32_t ip);
int taosOpenUDClientSocket(char *ip, uint16_t port);
int taosOpenRawSocket(char *ip);
int taosCopyFds(int sfd, int dfd, int64_t len);
int taosGetPublicIp(char *const ip);
int taosGetPrivateIp(char *const ip);
void tinet_ntoa(char *ipstr, unsigned int ip);
int taosSetNonblocking(int sock, int on);
int taosGetFqdn(char *);
uint32_t taosGetIpFromFqdn(const char *);
void tinet_ntoa(char *ipstr, unsigned int ip);
uint32_t ip2uint(const char *const ip_addr);
#ifdef __cplusplus
}

132
src/util/src/exception.c Normal file
View File

@ -0,0 +1,132 @@
#include "exception.h"
static _Thread_local SExceptionNode* expList;
void exceptionPushNode( SExceptionNode* node ) {
node->prev = expList;
expList = node;
}
int32_t exceptionPopNode() {
SExceptionNode* node = expList;
expList = node->prev;
return node->code;
}
void exceptionThrow( int code ) {
expList->code = code;
longjmp( expList->jb, 1 );
}
static void cleanupWrapper_void_ptr_ptr( SCleanupAction* ca ) {
void (*func)( void*, void* ) = ca->func;
func( ca->arg1.Ptr, ca->arg2.Ptr );
}
static void cleanupWrapper_void_ptr_bool( SCleanupAction* ca ) {
void (*func)( void*, bool ) = ca->func;
func( ca->arg1.Ptr, ca->arg2.Bool );
}
static void cleanupWrapper_void_ptr( SCleanupAction* ca ) {
void (*func)( void* ) = ca->func;
func( ca->arg1.Ptr );
}
static void cleanupWrapper_int_int( SCleanupAction* ca ) {
int (*func)( int ) = ca->func;
func( (int)(intptr_t)(ca->arg1.Int) );
}
static void cleanupWrapper_void_void( SCleanupAction* ca ) {
void (*func)() = ca->func;
func();
}
typedef void (*wrapper)(SCleanupAction*);
static wrapper wrappers[] = {
cleanupWrapper_void_ptr_ptr,
cleanupWrapper_void_ptr_bool,
cleanupWrapper_void_ptr,
cleanupWrapper_int_int,
cleanupWrapper_void_void,
};
void cleanupPush_void_ptr_ptr( bool failOnly, void* func, void* arg1, void* arg2 ) {
assert( expList->numCleanupAction < expList->maxCleanupAction );
SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 0;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg1;
ca->arg2.Ptr = arg2;
}
void cleanupPush_void_ptr_bool( bool failOnly, void* func, void* arg1, bool arg2 ) {
assert( expList->numCleanupAction < expList->maxCleanupAction );
SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 1;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg1;
ca->arg2.Bool = arg2;
}
void cleanupPush_void_ptr( bool failOnly, void* func, void* arg ) {
assert( expList->numCleanupAction < expList->maxCleanupAction );
SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 2;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg;
}
void cleanupPush_int_int( bool failOnly, void* func, int arg ) {
assert( expList->numCleanupAction < expList->maxCleanupAction );
SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 3;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Int = arg;
}
void cleanupPush_void( bool failOnly, void* func ) {
assert( expList->numCleanupAction < expList->maxCleanupAction );
SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 4;
ca->failOnly = failOnly;
ca->func = func;
}
int32_t cleanupGetActionCount() {
return expList->numCleanupAction;
}
static void doExecuteCleanup( SExceptionNode* node, int32_t anchor, bool failed ) {
while( node->numCleanupAction > anchor ) {
--node->numCleanupAction;
SCleanupAction *ca = node->cleanupActions + node->numCleanupAction;
if( failed || !(ca->failOnly) )
wrappers[ca->wrapper]( ca );
}
}
void cleanupExecuteTo( int32_t anchor, bool failed ) {
doExecuteCleanup( expList, anchor, failed );
}
void cleanupExecute( SExceptionNode* node, bool failed ) {
doExecuteCleanup( node, 0, failed );
}

5
src/util/src/hash.c
View File

@ -520,7 +520,8 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
static SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
assert(pIter != NULL);
pIter->entryIndex += 1;
while (pIter->entryIndex < pIter->pHashObj->capacity) {
SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
if (pEntry->next == NULL) {
@ -540,7 +541,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) {
}
size_t size = taosHashGetSize(pIter->pHashObj);
if (size == 0 || pIter->num >= size) {
if (size == 0) {
return false;
}

1
src/util/src/tarray.c
View File

@ -176,7 +176,6 @@ SArray* taosArrayClone(const SArray* pSrc) {
return dst;
}
void taosArrayDestroy(SArray* pArray) {
if (pArray == NULL) {
return;

412
src/util/src/tbuffer.c
View File

@ -16,150 +16,384 @@
#include <stdlib.h>
#include <memory.h>
#include <assert.h>
#define TBUFFER_DEFINE_FUNCTION(type, name) \
type tbufRead##name(SBuffer* buf) { \
type ret; \
tbufReadToBuffer(buf, &ret, sizeof(type)); \
return ret; \
}\
void tbufWrite##name(SBuffer* buf, type data) {\
tbufWrite(buf, &data, sizeof(data));\
}\
void tbufWrite##name##At(SBuffer* buf, size_t pos, type data) {\
tbufWriteAt(buf, pos, &data, sizeof(data));\
}
#include <arpa/inet.h>
#include "tbuffer.h"
#include "exception.h"
#include <taoserror.h>
////////////////////////////////////////////////////////////////////////////////
// common functions
// reader functions
size_t tbufTell(SBuffer* buf) {
return buf->pos;
}
size_t tbufSeekTo(SBuffer* buf, size_t pos) {
if (pos > buf->size) {
// TODO: update error code, other tbufThrowError need to be changed too
tbufThrowError(buf, 1);
size_t tbufSkip(SBufferReader* buf, size_t size) {
if( (buf->pos + size) > buf->size ) {
THROW( TSDB_CODE_MEMORY_CORRUPTED );
}
size_t old = buf->pos;
buf->pos = pos;
buf->pos += size;
return old;
}
size_t tbufSkip(SBuffer* buf, size_t size) {
return tbufSeekTo(buf, buf->pos + size);
}
void tbufClose(SBuffer* buf, bool keepData) {
if (!keepData) {
free(buf->data);
}
buf->data = NULL;
buf->pos = 0;
buf->size = 0;
}
////////////////////////////////////////////////////////////////////////////////
// read functions
char* tbufRead(SBuffer* buf, size_t size) {
char* ret = buf->data + buf->pos;
tbufSkip(buf, size);
const char* tbufRead( SBufferReader* buf, size_t size ) {
const char* ret = buf->data + buf->pos;
tbufSkip( buf, size );
return ret;
}
void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size) {
assert(dst != NULL);
void tbufReadToBuffer( SBufferReader* buf, void* dst, size_t size ) {
assert( dst != NULL );
// always using memcpy, leave optimization to compiler
memcpy(dst, tbufRead(buf, size), size);
memcpy( dst, tbufRead(buf, size), size );
}
const char* tbufReadString(SBuffer* buf, size_t* len) {
uint16_t l = tbufReadUint16(buf);
char* ret = buf->data + buf->pos;
tbufSkip(buf, l + 1);
ret[l] = 0; // ensure the string end with '\0'
if (len != NULL) {
static size_t tbufReadLength( SBufferReader* buf ) {
// maximum length is 65535, if larger length is required
// this function and the corresponding write function need to be
// revised.
uint16_t l = tbufReadUint16( buf );
return l;
}
const char* tbufReadString( SBufferReader* buf, size_t* len ) {
size_t l = tbufReadLength( buf );
const char* ret = buf->data + buf->pos;
tbufSkip( buf, l + 1 );
if( ret[l] != 0 ) {
THROW( TSDB_CODE_MEMORY_CORRUPTED );
}
if( len != NULL ) {
*len = l;
}
return ret;
}
size_t tbufReadToString(SBuffer* buf, char* dst, size_t size) {
assert(dst != NULL);
size_t len;
const char* str = tbufReadString(buf, &len);
size_t tbufReadToString( SBufferReader* buf, char* dst, size_t size ) {
assert( dst != NULL );
size_t len;
const char* str = tbufReadString( buf, &len );
if (len >= size) {
len = size - 1;
}
memcpy(dst, str, len);
memcpy( dst, str, len );
dst[len] = 0;
return len;
}
const char* tbufReadBinary( SBufferReader* buf, size_t *len ) {
size_t l = tbufReadLength( buf );
const char* ret = buf->data + buf->pos;
tbufSkip( buf, l );
if( len != NULL ) {
*len = l;
}
return ret;
}
size_t tbufReadToBinary( SBufferReader* buf, void* dst, size_t size ) {
assert( dst != NULL );
size_t len;
const char* data = tbufReadBinary( buf, &len );
if( len >= size ) {
len = size;
}
memcpy( dst, data, len );
return len;
}
bool tbufReadBool( SBufferReader* buf ) {
bool ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
return ret;
}
char tbufReadChar( SBufferReader* buf ) {
char ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
return ret;
}
int8_t tbufReadInt8( SBufferReader* buf ) {
int8_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
return ret;
}
uint8_t tbufReadUint8( SBufferReader* buf ) {
uint8_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
return ret;
}
int16_t tbufReadInt16( SBufferReader* buf ) {
int16_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
if( buf->endian ) {
return (int16_t)ntohs( ret );
}
return ret;
}
uint16_t tbufReadUint16( SBufferReader* buf ) {
uint16_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
if( buf->endian ) {
return ntohs( ret );
}
return ret;
}
int32_t tbufReadInt32( SBufferReader* buf ) {
int32_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
if( buf->endian ) {
return (int32_t)ntohl( ret );
}
return ret;
}
uint32_t tbufReadUint32( SBufferReader* buf ) {
uint32_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
if( buf->endian ) {
return ntohl( ret );
}
return ret;
}
int64_t tbufReadInt64( SBufferReader* buf ) {
int64_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
if( buf->endian ) {
return (int64_t)htobe64( ret ); // TODO: ntohll
}
return ret;
}
uint64_t tbufReadUint64( SBufferReader* buf ) {
uint64_t ret;
tbufReadToBuffer( buf, &ret, sizeof(ret) );
if( buf->endian ) {
return htobe64( ret ); // TODO: ntohll
}
return ret;
}
float tbufReadFloat( SBufferReader* buf ) {
uint32_t ret = tbufReadUint32( buf );
return *(float*)( &ret );
}
double tbufReadDouble(SBufferReader* buf) {
uint64_t ret = tbufReadUint64( buf );
return *(double*)( &ret );
}
////////////////////////////////////////////////////////////////////////////////
// write functions
// writer functions
void tbufEnsureCapacity(SBuffer* buf, size_t size) {
void tbufCloseWriter( SBufferWriter* buf ) {
(*buf->allocator)( buf->data, 0 );
buf->data = NULL;
buf->pos = 0;
buf->size = 0;
}
void tbufEnsureCapacity( SBufferWriter* buf, size_t size ) {
size += buf->pos;
if (size > buf->size) {
if( size > buf->size ) {
size_t nsize = size + buf->size;
char* data = realloc(buf->data, nsize);
if (data == NULL) {
tbufThrowError(buf, 2);
char* data = (*buf->allocator)( buf->data, nsize );
// TODO: the exception should be thrown by the allocator function
if( data == NULL ) {
THROW( TSDB_CODE_SERV_OUT_OF_MEMORY );
}
buf->data = data;
buf->size = nsize;
}
}
char* tbufGetData(SBuffer* buf, bool takeOver) {
size_t tbufReserve( SBufferWriter* buf, size_t size ) {
tbufEnsureCapacity( buf, size );
size_t old = buf->pos;
buf->pos += size;
return old;
}
char* tbufGetData( SBufferWriter* buf, bool takeOver ) {
char* ret = buf->data;
if (takeOver) {
if( takeOver ) {
buf->pos = 0;
buf->size = 0;
buf->data = NULL;
}
return ret;
}
void tbufEndWrite(SBuffer* buf) {
free(buf->data);
buf->data = NULL;
buf->pos = 0;
buf->size = 0;
}
void tbufWrite(SBuffer* buf, const void* data, size_t size) {
assert(data != NULL);
tbufEnsureCapacity(buf, size);
memcpy(buf->data + buf->pos, data, size);
void tbufWrite( SBufferWriter* buf, const void* data, size_t size ) {
assert( data != NULL );
tbufEnsureCapacity( buf, size );
memcpy( buf->data + buf->pos, data, size );
buf->pos += size;
}
void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size) {
assert(data != NULL);
void tbufWriteAt( SBufferWriter* buf, size_t pos, const void* data, size_t size ) {
assert( data != NULL );
// this function can only be called to fill the gap on previous writes,
// so 'pos + size <= buf->pos' must be true
assert(pos + size <= buf->pos);
memcpy(buf->data + pos, data, size);
assert( pos + size <= buf->pos );
memcpy( buf->data + pos, data, size );
}
void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len) {
// maximum string length is 65535, if longer string is required
static void tbufWriteLength( SBufferWriter* buf, size_t len ) {
// maximum length is 65535, if larger length is required
// this function and the corresponding read function need to be
// revised.
assert(len <= 0xffff);
tbufWriteUint16(buf, (uint16_t)len);
tbufWrite(buf, str, len + 1);
assert( len <= 0xffff );
tbufWriteUint16( buf, (uint16_t)len );
}
void tbufWriteString(SBuffer* buf, const char* str) {
tbufWriteStringLen(buf, str, strlen(str));
void tbufWriteStringLen( SBufferWriter* buf, const char* str, size_t len ) {
tbufWriteLength( buf, len );
tbufWrite( buf, str, len );
tbufWriteChar( buf, '\0' );
}
void tbufWriteString( SBufferWriter* buf, const char* str ) {
tbufWriteStringLen( buf, str, strlen(str) );
}
void tbufWriteBinary( SBufferWriter* buf, const void* data, size_t len ) {
tbufWriteLength( buf, len );
tbufWrite( buf, data, len );
}
void tbufWriteBool( SBufferWriter* buf, bool data ) {
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteBoolAt( SBufferWriter* buf, size_t pos, bool data ) {
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteChar( SBufferWriter* buf, char data ) {
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteCharAt( SBufferWriter* buf, size_t pos, char data ) {
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteInt8( SBufferWriter* buf, int8_t data ) {
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteInt8At( SBufferWriter* buf, size_t pos, int8_t data ) {
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteUint8( SBufferWriter* buf, uint8_t data ) {
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteUint8At( SBufferWriter* buf, size_t pos, uint8_t data ) {
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteInt16( SBufferWriter* buf, int16_t data ) {
if( buf->endian ) {
data = (int16_t)htons( data );
}
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteInt16At( SBufferWriter* buf, size_t pos, int16_t data ) {
if( buf->endian ) {
data = (int16_t)htons( data );
}
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteUint16( SBufferWriter* buf, uint16_t data ) {
if( buf->endian ) {
data = htons( data );
}
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteUint16At( SBufferWriter* buf, size_t pos, uint16_t data ) {
if( buf->endian ) {
data = htons( data );
}
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteInt32( SBufferWriter* buf, int32_t data ) {
if( buf->endian ) {
data = (int32_t)htonl( data );
}
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteInt32At( SBufferWriter* buf, size_t pos, int32_t data ) {
if( buf->endian ) {
data = (int32_t)htonl( data );
}
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteUint32( SBufferWriter* buf, uint32_t data ) {
if( buf->endian ) {
data = htonl( data );
}
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteUint32At( SBufferWriter* buf, size_t pos, uint32_t data ) {
if( buf->endian ) {
data = htonl( data );
}
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteInt64( SBufferWriter* buf, int64_t data ) {
if( buf->endian ) {
data = (int64_t)htobe64( data );
}
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteInt64At( SBufferWriter* buf, size_t pos, int64_t data ) {
if( buf->endian ) {
data = (int64_t)htobe64( data );
}
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteUint64( SBufferWriter* buf, uint64_t data ) {
if( buf->endian ) {
data = htobe64( data );
}
tbufWrite( buf, &data, sizeof(data) );
}
void tbufWriteUint64At( SBufferWriter* buf, size_t pos, uint64_t data ) {
if( buf->endian ) {
data = htobe64( data );
}
tbufWriteAt( buf, pos, &data, sizeof(data) );
}
void tbufWriteFloat( SBufferWriter* buf, float data ) {
tbufWriteUint32( buf, *(uint32_t*)(&data) );
}
void tbufWriteFloatAt( SBufferWriter* buf, size_t pos, float data ) {
tbufWriteUint32At( buf, pos, *(uint32_t*)(&data) );
}
void tbufWriteDouble( SBufferWriter* buf, double data ) {
tbufWriteUint64( buf, *(uint64_t*)(&data) );
}
void tbufWriteDoubleAt( SBufferWriter* buf, size_t pos, double data ) {
tbufWriteUint64At( buf, pos, *(uint64_t*)(&data) );
}

2
src/util/src/tconfig.c
View File

@ -249,7 +249,7 @@ void taosReadGlobalLogCfg() {
}
wordfree(&full_path);
taosReadLogOption("logDir", logDir);
taosReadLogOption("tsLogDir", tsLogDir);
sprintf(fileName, "%s/taos.cfg", configDir);
fp = fopen(fileName, "r");

8
src/util/src/tlog.c
View File

@ -66,7 +66,7 @@ int32_t tsAsyncLog = 1;
float tsTotalLogDirGB = 0;
float tsAvailLogDirGB = 0;
float tsMinimalLogDirGB = 0.1;
char logDir[TSDB_FILENAME_LEN] = "/var/log/taos";
char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos";
static SLogObj tsLogObj = { .fileNum = 1 };
static void * taosAsyncOutputLog(void *param);
@ -298,7 +298,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
void taosPrintLog(const char *const flags, int32_t dflag, const char *const format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB);
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
return;
}
@ -356,7 +356,7 @@ void taosPrintLog(const char *const flags, int32_t dflag, const char *const form
void taosDumpData(unsigned char *msg, int32_t len) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB);
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
return;
}
@ -385,7 +385,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
void taosPrintLongString(const char *const flags, int32_t dflag, const char *const format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB);
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
return;
}

4
src/util/src/tnote.c
View File

@ -27,10 +27,10 @@ void taosInitNote(int numOfNoteLines, int maxNotes, char* lable)
if (strcasecmp(lable, "http_note") == 0) {
pNote = &m_HttpNote;
sprintf(temp, "%s/httpnote", logDir);
sprintf(temp, "%s/httpnote", tsLogDir);
} else if (strcasecmp(lable, "tsc_note") == 0) {
pNote = &m_TscNote;
sprintf(temp, "%s/tscnote-%d", logDir, getpid());
sprintf(temp, "%s/tscnote-%d", tsLogDir, getpid());
} else {
return;
}

126
src/util/src/tsocket.c
View File

@ -19,89 +19,23 @@
#include "tsocket.h"
#include "tutil.h"
/*
* Function to get the public ip address of current machine. If get IP
* successfully, return 0, else, return -1. The return values is ip.
*
* Use:
* if (taosGetPublicIp(ip) != 0) {
* perror("Fail to get public IP address\n");
* exit(EXIT_FAILURE);
* }
*/
int taosGetPublicIp(char *const ip) {
/* bool flag; */
int flag;
int sock;
char ** pptr = NULL;
struct sockaddr_in destAddr;
struct hostent * ptr = NULL;
char destIP[128];
char szBuffer[] = {
"GET / HTTP/1.1\nHost: ident.me\nUser-Agent: curl/7.47.0\nAccept: "
"*/*\n\n"};
char res[1024];
// Create socket
sock = (int)socket(AF_INET, SOCK_STREAM, 0);
if (sock == -1) {
return -1;
}
bzero((void *)&destAddr, sizeof(destAddr));
destAddr.sin_family = AF_INET;
destAddr.sin_port = htons(80);
ptr = gethostbyname("ident.me");
if (ptr == NULL) {
return -1;
}
// Loop to find a valid IP address
for (flag = 0, pptr = ptr->h_addr_list; NULL != *pptr; ++pptr) {
inet_ntop(ptr->h_addrtype, *pptr, destIP, sizeof(destIP));
destAddr.sin_addr.s_addr = inet_addr(destIP);
if (connect(sock, (struct sockaddr *)&destAddr, sizeof(struct sockaddr)) != -1) {
flag = 1;
break;
}
}
// Check if the host is available.
if (flag == 0) {
return -1;
}
// Check send.
if (strlen(szBuffer) != taosWriteSocket(sock, szBuffer, (size_t)strlen(szBuffer))) {
return -1;
}
// Receive response.
if (taosReadSocket(sock, res, 1024) == -1) {
return -1;
}
// Extract the IP address from the response.
int c_start = 0, c_end = 0;
for (; c_start < (int)strlen(res); c_start = c_end + 1) {
for (c_end = c_start; c_end < (int)strlen(res) && res[c_end] != '\n'; c_end++) {
}
if (c_end >= (int)strlen(res)) {
return -1;
}
if (res[c_start] >= '0' && res[c_start] <= '9') {
strncpy(ip, res + c_start, (size_t)(c_end - c_start));
ip[c_end - c_start] = '\0';
break;
}
}
int taosGetFqdn(char *fqdn) {
char hostname[1024];
hostname[1023] = '\0';
gethostname(hostname, 1023);
struct hostent* h;
h = gethostbyname(hostname);
strcpy(fqdn, h->h_name);
return 0;
}
uint32_t taosGetIpFromFqdn(const char *fqdn) {
struct hostent * record = gethostbyname(fqdn);
if(record == NULL) return -1;
return ((struct in_addr *)record->h_addr)->s_addr;
}
// Function converting an IP address string to an unsigned int.
uint32_t ip2uint(const char *const ip_addr) {
char ip_addr_cpy[20];
@ -259,7 +193,7 @@ int taosReadn(int fd, char *ptr, int nbytes) {
return (nbytes - nleft);
}
int taosOpenUdpSocket(char *ip, uint16_t port) {
int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in localAddr;
int sockFd;
int ttl = 128;
@ -270,7 +204,7 @@ int taosOpenUdpSocket(char *ip, uint16_t port) {
memset((char *)&localAddr, 0, sizeof(localAddr));
localAddr.sin_family = AF_INET;
localAddr.sin_addr.s_addr = inet_addr(ip);
localAddr.sin_addr.s_addr = ip;
localAddr.sin_port = (uint16_t)htons(port);
if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
@ -325,13 +259,11 @@ int taosOpenUdpSocket(char *ip, uint16_t port) {
return sockFd;
}
int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) {
int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) {
int sockFd = 0;
struct sockaddr_in serverAddr, clientAddr;
int ret;
// uTrace("open tcp client socket:%s:%d, local Ip:%s", destIp, destPort, clientIp);
sockFd = (int)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sockFd < 0) {
@ -339,16 +271,16 @@ int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) {
return -1;
}
if (clientIp && clientIp[0] && clientIp[0] != '0') {
if ( clientIp != 0) {
memset((char *)&clientAddr, 0, sizeof(clientAddr));
clientAddr.sin_family = AF_INET;
clientAddr.sin_addr.s_addr = inet_addr(clientIp);
clientAddr.sin_addr.s_addr = clientIp;
clientAddr.sin_port = 0;
/* bind socket to client address */
if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) {
uError("bind tcp client socket failed, client(%s:0), dest(%s:%d), reason:%d(%s)",
clientIp, destIp, destPort, errno, strerror(errno));
uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)",
clientIp, destIp, destPort, strerror(errno));
close(sockFd);
return -1;
}
@ -356,13 +288,13 @@ int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) {
memset((char *)&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_addr.s_addr = inet_addr(destIp);
serverAddr.sin_addr.s_addr = destIp;
serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort);
ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
if (ret != 0) {
//uError("failed to connect socket, ip:%s, port:%hu, reason: %s", destIp, destPort, strerror(errno));
//uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
taosCloseSocket(sockFd);
sockFd = -1;
}
@ -420,7 +352,7 @@ int taosKeepTcpAlive(int sockFd) {
return 0;
}
int taosOpenTcpServerSocket(char *ip, uint16_t port) {
int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in serverAdd;
int sockFd;
int reuse;
@ -429,7 +361,7 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) {
bzero((char *)&serverAdd, sizeof(serverAdd));
serverAdd.sin_family = AF_INET;
serverAdd.sin_addr.s_addr = inet_addr(ip);
serverAdd.sin_addr.s_addr = ip;
serverAdd.sin_port = (uint16_t)htons(port);
if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
@ -447,7 +379,7 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) {
/* bind socket to server address */
if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
uError("bind tcp server socket failed, %s:%hu, reason:%d(%s)", ip, port, errno, strerror(errno));
uError("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
close(sockFd);
return -1;
}
@ -455,14 +387,14 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) {
if (taosKeepTcpAlive(sockFd) < 0) return -1;
if (listen(sockFd, 10) < 0) {
uError("listen tcp server socket failed, %s:%hu, reason:%d(%s)", ip, port, errno, strerror(errno));
uError("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
return -1;
}
return sockFd;
}
int taosOpenRawSocket(char *ip) {
int taosOpenRawSocket(uint32_t ip) {
int fd, hold;
struct sockaddr_in rawAdd;
@ -483,10 +415,10 @@ int taosOpenRawSocket(char *ip) {
bzero((char *)&rawAdd, sizeof(rawAdd));
rawAdd.sin_family = AF_INET;
rawAdd.sin_addr.s_addr = inet_addr(ip);
rawAdd.sin_addr.s_addr = ip;
if (bind(fd, (struct sockaddr *)&rawAdd, sizeof(rawAdd)) < 0) {
uError("failed to bind RAW socket: %d (%s)", errno, strerror(errno));
uError("failed to bind RAW socket:(%s)", strerror(errno));
close(fd);
return -1;
}

1
src/vnode/inc/vnodeInt.h
View File

@ -36,6 +36,7 @@ typedef struct {
void *sync;
void *events;
void *cq; // continuous query
int32_t cfgVersion;
STsdbCfg tsdbCfg;
SSyncCfg syncCfg;
SWalCfg walCfg;

173
src/vnode/src/vnodeMain.c
View File

@ -104,8 +104,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep;
tsdbCfg.maxCacheSize = pVnodeCfg->cfg.maxCacheSize;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL);
@ -198,7 +197,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
syncInfo.vgId = pVnode->vgId;
syncInfo.version = pVnode->version;
syncInfo.syncCfg = pVnode->syncCfg;
sprintf(syncInfo.path, "%s/tsdb/", rootDir);
sprintf(syncInfo.path, "%s", rootDir);
syncInfo.ahandle = pVnode;
syncInfo.getWalInfo = vnodeGetWalInfo;
syncInfo.getFileInfo = vnodeGetFileInfo;
@ -287,7 +286,7 @@ void *vnodeGetVnode(int32_t vgId) {
SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
if (ppVnode == NULL || *ppVnode == NULL) {
terrno = TSDB_CODE_INVALID_VGROUP_ID;
dError("vgId:%d not exist", vgId);
dPrint("vgId:%d not exist", vgId);
return NULL;
}
@ -312,7 +311,7 @@ void *vnodeGetWqueue(int32_t vgId) {
SVnodeObj *pVnode = vnodeAccquireVnode(vgId);
if (pVnode == NULL) return NULL;
return pVnode->wqueue;
}
}
void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
@ -332,6 +331,7 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) {
SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
pLoad->vgId = htonl(pVnode->vgId);
pLoad->cfgVersion = htonl(pVnode->cfgVersion);
pLoad->totalStorage = htobe64(pLoad->totalStorage);
pLoad->compStorage = htobe64(pLoad->compStorage);
pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
@ -341,10 +341,13 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) {
}
static void vnodeCleanUp(SVnodeObj *pVnode) {
taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
//syncStop(pVnode->sync);
if (pVnode->sync) {
syncStop(pVnode->sync);
pVnode->sync = NULL;
}
tsdbCloseRepo(pVnode->tsdb);
walClose(pVnode->wal);
vnodeSaveVersion(pVnode);
@ -379,46 +382,39 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId);
FILE *fp = fopen(cfgFile, "w");
if (!fp) {
dError("vgId:%d, failed to open vnode cfg file for write, error:%s", pVnodeCfg->cfg.vgId, strerror(errno));
dError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile,
strerror(errno));
return errno;
}
char ipStr[20];
int32_t len = 0;
int32_t maxLen = 1000;
char * content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision);
len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression);
len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion);
len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize);
len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks);
len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables);
len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile);
len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep);
len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1);
len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2);
len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock);
len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock);
len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep);
len += snprintf(content + len, maxLen - len, " \"maxCacheSize\": %" PRId64 ",\n", pVnodeCfg->cfg.maxCacheSize);
len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime);
len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision);
len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression);
len += snprintf(content + len, maxLen - len, " \"commitLog\": %d,\n", pVnodeCfg->cfg.commitLog);
len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals);
uint32_t ipInt = pVnodeCfg->cfg.arbitratorIp;
sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24));
len += snprintf(content + len, maxLen - len, " \"arbitratorIp\": \"%s\",\n", ipStr);
len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum);
len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications);
len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals);
len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum);
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) {
len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId);
uint32_t ipInt = pVnodeCfg->nodes[i].nodeIp;
sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24));
len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", ipStr);
len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", pVnodeCfg->nodes[i].nodeName);
len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp);
if (i < pVnodeCfg->cfg.replications - 1) {
len += snprintf(content + len, maxLen - len, " },{\n");
@ -442,7 +438,8 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
FILE *fp = fopen(cfgFile, "r");
if (!fp) {
dError("pVnode:%p vgId:%d, failed to open vnode cfg file for read, error:%s", pVnode, pVnode->vgId, strerror(errno));
dError("pVnode:%p vgId:%d, failed to open vnode cfg file for read, file:%s, error:%s", pVnode, pVnode->vgId,
cfgFile, strerror(errno));
return errno;
}
@ -463,19 +460,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
goto PARSE_OVER;
}
cJSON *precision = cJSON_GetObjectItem(root, "precision");
if (!precision || precision->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId);
cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
if (!cfgVersion || cfgVersion->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.precision = (int8_t)precision->valueint;
pVnode->cfgVersion = cfgVersion->valueint;
cJSON *compression = cJSON_GetObjectItem(root, "compression");
if (!compression || compression->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId);
cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.compression = (int8_t)compression->valueint;
pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint;
cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks");
if (!totalBlocks || totalBlocks->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint;
cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables");
if (!maxTables || maxTables->type != cJSON_Number) {
@ -484,13 +488,34 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
pVnode->tsdbCfg.maxTables = maxTables->valueint;
cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile");
cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile");
if (!daysPerFile || daysPerFile->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint;
cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep");
if (!daysToKeep || daysToKeep->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.keep = daysToKeep->valueint;
cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1");
if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.keep1 = daysToKeep1->valueint;
cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2");
if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.keep2 = daysToKeep2->valueint;
cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock");
if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode, pVnode->vgId);
@ -505,19 +530,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint;
cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep");
if (!daysToKeep || daysToKeep->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId);
cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime");
if (!commitTime || commitTime->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, commitTime not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.keep = daysToKeep->valueint;
pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint;
cJSON *maxCacheSize = cJSON_GetObjectItem(root, "maxCacheSize");
if (!maxCacheSize || maxCacheSize->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, maxCacheSize not found", pVnode, pVnode->vgId);
cJSON *precision = cJSON_GetObjectItem(root, "precision");
if (!precision || precision->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.maxCacheSize = maxCacheSize->valueint;
pVnode->tsdbCfg.precision = (int8_t)precision->valueint;
cJSON *compression = cJSON_GetObjectItem(root, "compression");
if (!compression || compression->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->tsdbCfg.compression = (int8_t)compression->valueint;
cJSON *commitLog = cJSON_GetObjectItem(root, "commitLog");
if (!commitLog || commitLog->type != cJSON_Number) {
@ -534,12 +566,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
pVnode->walCfg.wals = (int8_t)wals->valueint;
pVnode->walCfg.keep = 0;
cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp");
if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId);
cJSON *replica = cJSON_GetObjectItem(root, "replica");
if (!replica || replica->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring);
pVnode->syncCfg.replica = (int8_t)replica->valueint;
cJSON *quorum = cJSON_GetObjectItem(root, "quorum");
if (!quorum || quorum->type != cJSON_Number) {
@ -548,13 +580,6 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
pVnode->syncCfg.quorum = (int8_t)quorum->valueint;
cJSON *replica = cJSON_GetObjectItem(root, "replica");
if (!replica || replica->type != cJSON_Number) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->syncCfg.replica = (int8_t)replica->valueint;
cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode, pVnode->vgId);
@ -578,27 +603,22 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint;
cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp");
if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeIp not found", pVnode, pVnode->vgId);
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->syncCfg.nodeInfo[i].nodeIp = inet_addr(nodeIp->valuestring);
cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName");
if (!nodeName || nodeName->type != cJSON_String || nodeName->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeName not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
strncpy(pVnode->syncCfg.nodeInfo[i].name, nodeName->valuestring, TSDB_NODE_NAME_LEN);
taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort);
pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
}
ret = 0;
dPrint("pVnode:%p vgId:%d, read vnode cfg successed, replcia:%d", pVnode, pVnode->vgId, pVnode->syncCfg.replica);
for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
dPrint("pVnode:%p vgId:%d, dnode:%d, ip:%s name:%s", pVnode, pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
taosIpStr(pVnode->syncCfg.nodeInfo[i].nodeIp), pVnode->syncCfg.nodeInfo[i].name);
dPrint("pVnode:%p vgId:%d, dnode:%d, %s:%d", pVnode, pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort);
}
PARSE_OVER:
@ -608,13 +628,13 @@ PARSE_OVER:
return ret;
}
static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
char versionFile[TSDB_FILENAME_LEN + 30] = {0};
sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
FILE *fp = fopen(versionFile, "w");
if (!fp) {
dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno));
dError("pVnode:%p vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode, pVnode->vgId,
versionFile, strerror(errno));
return errno;
}
@ -630,7 +650,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
fclose(fp);
free(content);
dPrint("pVnode:%p vgId:%d, save vnode version successed", pVnode, pVnode->vgId);
dPrint("pVnode:%p vgId:%d, save vnode version:%" PRId64 " successed", pVnode, pVnode->vgId, pVnode->version);
return 0;
}
@ -638,9 +658,10 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
static bool vnodeReadVersion(SVnodeObj *pVnode) {
char versionFile[TSDB_FILENAME_LEN + 30] = {0};
sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
FILE *fp = fopen(versionFile, "w");
FILE *fp = fopen(versionFile, "r");
if (!fp) {
dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno));
dTrace("pVnode:%p vgId:%d, failed to open version file:%s error:%s", pVnode, pVnode->vgId,
versionFile, strerror(errno));
return false;
}
@ -677,4 +698,4 @@ PARSE_OVER:
cJSON_Delete(root);
fclose(fp);
return ret;
}
}

19
src/vnode/src/vnodeWrite.c
View File

@ -58,7 +58,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
return TSDB_CODE_NOT_ACTIVE_VNODE;
if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER)
return TSDB_CODE_NO_MASTER;
return TSDB_CODE_NOT_READY;
// assign version
pVnode->version++;
@ -74,15 +74,15 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
// write into WAL
code = walWrite(pVnode->wal, pHead);
if ( code < 0) return code;
if (code < 0) return code;
int32_t syncCode = syncForwardToPeer(pVnode->sync, pHead, item);
if (syncCode < 0) return syncCode;
code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
if (code < 0) return code;
if (pVnode->syncCfg.replica > 1)
code = syncForwardToPeer(pVnode->sync, pHead, item);
return code;
return syncCode;
}
static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
@ -225,11 +225,10 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
int32_t code = 0;
dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId);
// int64_t uid = htobe64(pTable->uid);
// TODO: drop stable in vvnode
//int64_t uid = htobe64(pTable->uid);
//void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode);
//rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid);
//rpcRsp.code = tsdbDropTable(pTsdb, pTable->uid);
code = TSDB_CODE_SUCCESS;
dTrace("pVnode:%p vgId:%d, stable:%s, drop stable result:%x", pVnode, pTable->tableId, code);

2
src/wal/src/walMain.c
View File

@ -269,7 +269,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
if (*index < first && *index > pWal->id) {
code = -1; // index out of range
} else {
sprintf(name, "%s/%s%d", pWal->path, walPrefix, *index);
sprintf(name, "wal/%s%d", walPrefix, *index);
code = (*index == pWal->id) ? 0:1;
}

4
tests/examples/c/demo.c
View File

@ -75,7 +75,9 @@ int main(int argc, char *argv[]) {
doQuery(taos, "create database if not exists test");
doQuery(taos, "use test");
doQuery(taos, "select count(*),k,sum(k) from m1 group by k");
// doQuery(taos, "select a from m1");
// doQuery(taos, "select m2.u from m1, m2 where m1.ts=m2.ts and m1.a=m2.b;");
doQuery(taos, "select last_row(ts) from tm99");
// doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);");
// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);");
// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);");

77
tests/pytest/fulltest.sh Executable file
View File

@ -0,0 +1,77 @@
#!/bin/bash
python3 ./test.py $1 -f insert/basic.py
python3 ./test.py $1 -f insert/int.py
python3 ./test.py $1 -f insert/float.py
python3 ./test.py $1 -f insert/bigint.py
python3 ./test.py $1 -f insert/bool.py
python3 ./test.py $1 -f insert/double.py
python3 ./test.py $1 -f insert/smallint.py
python3 ./test.py $1 -f insert/tinyint.py
python3 ./test.py $1 -f insert/date.py
python3 ./test.py $1 -f insert/binary.py
python3 ./test.py $1 -f import_merge/importBlock1HO.py
python3 ./test.py $1 -f import_merge/importBlock1HPO.py
python3 ./test.py $1 -f import_merge/importBlock1H.py
python3 ./test.py $1 -f import_merge/importBlock1S.py
python3 ./test.py $1 -f import_merge/importBlock1Sub.py
python3 ./test.py $1 -f import_merge/importBlock1TO.py
python3 ./test.py $1 -f import_merge/importBlock1TPO.py
python3 ./test.py $1 -f import_merge/importBlock1T.py
python3 ./test.py $1 -f import_merge/importBlock2HO.py
python3 ./test.py $1 -f import_merge/importBlock2HPO.py
python3 ./test.py $1 -f import_merge/importBlock2H.py
python3 ./test.py $1 -f import_merge/importBlock2S.py
python3 ./test.py $1 -f import_merge/importBlock2Sub.py
python3 ./test.py $1 -f import_merge/importBlock2TO.py
python3 ./test.py $1 -f import_merge/importBlock2TPO.py
python3 ./test.py $1 -f import_merge/importBlock2T.py
python3 ./test.py $1 -f import_merge/importBlockbetween.py
python3 ./test.py $1 -f import_merge/importCacheFileHO.py
python3 ./test.py $1 -f import_merge/importCacheFileHPO.py
python3 ./test.py $1 -f import_merge/importCacheFileH.py
python3 ./test.py $1 -f import_merge/importCacheFileS.py
python3 ./test.py $1 -f import_merge/importCacheFileSub.py
python3 ./test.py $1 -f import_merge/importCacheFileTO.py
python3 ./test.py $1 -f import_merge/importCacheFileTPO.py
python3 ./test.py $1 -f import_merge/importCacheFileT.py
python3 ./test.py $1 -f import_merge/importDataH2.py
python3 ./test.py $1 -f import_merge/importDataHO2.py
python3 ./test.py $1 -f import_merge/importDataHO.py
python3 ./test.py $1 -f import_merge/importDataHPO.py
python3 ./test.py $1 -f import_merge/importDataLastHO.py
python3 ./test.py $1 -f import_merge/importDataLastHPO.py
python3 ./test.py $1 -f import_merge/importDataLastH.py
python3 ./test.py $1 -f import_merge/importDataLastS.py
python3 ./test.py $1 -f import_merge/importDataLastSub.py
python3 ./test.py $1 -f import_merge/importDataLastTO.py
python3 ./test.py $1 -f import_merge/importDataLastTPO.py
python3 ./test.py $1 -f import_merge/importDataLastT.py
python3 ./test.py $1 -f import_merge/importDataS.py
python3 ./test.py $1 -f import_merge/importDataSub.py
python3 ./test.py $1 -f import_merge/importDataTO.py
python3 ./test.py $1 -f import_merge/importDataTPO.py
python3 ./test.py $1 -f import_merge/importDataT.py
python3 ./test.py $1 -f import_merge/importHeadOverlap.py
python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py
python3 ./test.py $1 -f import_merge/importHead.py
python3 ./test.py $1 -f import_merge/importHORestart.py
python3 ./test.py $1 -f import_merge/importHPORestart.py
python3 ./test.py $1 -f import_merge/importHRestart.py
python3 ./test.py $1 -f import_merge/importLastHO.py
python3 ./test.py $1 -f import_merge/importLastHPO.py
python3 ./test.py $1 -f import_merge/importLastH.py
python3 ./test.py $1 -f import_merge/importLastS.py
python3 ./test.py $1 -f import_merge/importLastSub.py
python3 ./test.py $1 -f import_merge/importLastTO.py
python3 ./test.py $1 -f import_merge/importLastTPO.py
python3 ./test.py $1 -f import_merge/importLastT.py
python3 ./test.py $1 -f import_merge/importSpan.py
python3 ./test.py $1 -f import_merge/importSRestart.py
python3 ./test.py $1 -f import_merge/importSubRestart.py
python3 ./test.py $1 -f import_merge/importTailOverlap.py
python3 ./test.py $1 -f import_merge/importTailPartOverlap.py
python3 ./test.py $1 -f import_merge/importTail.py
python3 ./test.py $1 -f import_merge/importToCommit.py
python3 ./test.py $1 -f import_merge/importTORestart.py
python3 ./test.py $1 -f import_merge/importTPORestart.py
python3 ./test.py $1 -f import_merge/importTRestart.py

0
tests/pytest/import_merge/__init__.py Normal file
View File

72
tests/pytest/import_merge/importBlock1H.py Normal file
View File

@ -0,0 +1,72 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 39):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 1 data before")
startTime = self.startTime - 1
tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(39)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

75
tests/pytest/import_merge/importBlock1HO.py Normal file
View File

@ -0,0 +1,75 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 39):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 10 data before with overlap")
startTime = self.startTime - 5
sqlcmd = ['import into tb1 values']
for rid in range(1, 11):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(43)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

77
tests/pytest/import_merge/importBlock1HPO.py Normal file
View File

@ -0,0 +1,77 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 11):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
for rid in range(15, 43):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 20 data before with partly overlap")
startTime = self.startTime - 5
sqlcmd = ['import into tb1 values']
for rid in range(1, 21):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(47)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

75
tests/pytest/import_merge/importBlock1S.py Normal file
View File

@ -0,0 +1,75 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 39):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 50 data covering existing data")
startTime = self.startTime - 5
sqlcmd = ['import into tb1 values']
for rid in range(1, 51):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(50)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

75
tests/pytest/import_merge/importBlock1Sub.py Normal file
View File

@ -0,0 +1,75 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 39):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 10 data totally repetitive")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 11):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

72
tests/pytest/import_merge/importBlock1T.py Normal file
View File

@ -0,0 +1,72 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 39):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 1 data after")
startTime = self.startTime + 38
tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, rid))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(39)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

75
tests/pytest/import_merge/importBlock1TO.py Normal file
View File

@ -0,0 +1,75 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 39):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 10 data later with overlap")
startTime = self.startTime + 30
sqlcmd = ['import into tb1 values']
for rid in range(1, 11):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(40)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

77
tests/pytest/import_merge/importBlock1TPO.py Normal file
View File

@ -0,0 +1,77 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 38 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 31):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
for rid in range(35, 43):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(38)
tdLog.info("================= step4")
tdLog.info("import 30 data later with partly overlap")
startTime = self.startTime + 25
sqlcmd = ['import into tb1 values']
for rid in range(1, 31):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(55)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

72
tests/pytest/import_merge/importBlock2H.py Normal file
View File

@ -0,0 +1,72 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 76 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 77):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(76)
tdLog.info("================= step4")
tdLog.info("import 1 data before")
startTime = self.startTime - 1
tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(77)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

75
tests/pytest/import_merge/importBlock2HO.py Normal file
View File

@ -0,0 +1,75 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
self.ntables = 1
self.startTime = 1520000010000
tdDnodes.stop(1)
tdDnodes.deploy(1)
tdDnodes.start(1)
tdSql.execute('reset query cache')
tdSql.execute('drop database if exists db')
tdSql.execute('create database db cache 512')
tdSql.execute('use db')
tdLog.info("================= step1")
tdLog.info("create 1 table")
tdSql.execute('create table tb1 (ts timestamp, speed int)')
tdLog.info("one block can import 38 records")
tdLog.info("================= step2")
tdLog.info("import 76 sequential data")
startTime = self.startTime
sqlcmd = ['import into tb1 values']
for rid in range(1, 77):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step3")
tdSql.query('select * from tb1')
tdSql.checkRows(76)
tdLog.info("================= step4")
tdLog.info("import 10 data before with overlap")
startTime = self.startTime - 5
sqlcmd = ['import into tb1 values']
for rid in range(1, 11):
sqlcmd.append('(%ld, %d)' % (startTime + rid, rid))
tdSql.execute(" ".join(sqlcmd))
tdLog.info("================= step5")
tdSql.query('select * from tb1')
tdSql.checkRows(81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

Some files were not shown because too many files have changed in this diff Show More