xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/python-test-no-sudo

This commit is contained in:
Steven Li 2020-04-30 15:12:13 -07:00
parent 742005e860 59a397927f
commit 1d1a027d04
441 changed files with 4619 additions and 7317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.travis.yml
View File

@ -32,15 +32,15 @@ matrix:
- cd debug
script:
- cmake ..
- make
- cmake .. > /dev/null
- make > /dev/null
after_success:
- |-
case $TRAVIS_OS_NAME in
linux)
cd ${TRAVIS_BUILD_DIR}/debug
make install || travis_terminate $?
make install > /dev/null || travis_terminate $?
pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
@ -98,11 +98,11 @@ matrix:
# Commands to prepare for build_command
# ** likely specific to your build **
build_command_prepend: cmake .
build_command_prepend: cmake . > /dev/null
# The command that will be added as an argument to "cov-build" to compile your project for analysis,
# ** likely specific to your build **
build_command: make
build_command: make > /dev/null
# Pattern to match selecting branches that will run analysis. We recommend leaving this set to 'coverity_scan'.
# Take care in resource usage, and consider the build frequency allowances per
@ -135,15 +135,15 @@ matrix:
- cd debug
script:
- cmake -DCOVER=true ..
- make
- cmake -DCOVER=true .. > /dev/null
- make > /dev/null
after_success:
- |-
case $TRAVIS_OS_NAME in
linux)
cd ${TRAVIS_BUILD_DIR}/debug
make install || travis_terminate $?
make install > /dev/null || travis_terminate $?
pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
@ -208,8 +208,8 @@ matrix:
- cd debug
script:
- cmake ..
- make
- cmake .. > /dev/null
- make > /dev/null
# - os: osx
# language: c
@ -225,5 +225,5 @@ matrix:
# - cd ${TRAVIS_BUILD_DIR}
# - mkdir debug
# - cd debug
# - cmake ..
# - make
# - cmake .. > /dev/null
# - make > /dev/null

9
src/client/src/tscSQLParser.c
View File

@ -304,10 +304,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0];
if (!validateIpAddress(pIpAddr->z, pIpAddr->n)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
// if (!validateIpAddress(pIpAddr->z, pIpAddr->n)) {
// return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
// }
pIpAddr->n = strdequote(pIpAddr->z);
break;
}
@ -385,7 +386,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
pMsg += sizeof(SMgmtHead);
SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg;
strncpy(pCfg->ip, pDCL->a[0].z, pDCL->a[0].n);
strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n);

41
src/client/src/tscServer.c
View File

@ -49,11 +49,11 @@ static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) {
SRpcIpSet* pIpList = &pSql->ipList;
pIpList->numOfIps = pTableMeta->vgroupInfo.numOfIps;
pIpList->port = tsDnodeShellPort;
pIpList->inUse = 0;
for(int32_t i = 0; i < pTableMeta->vgroupInfo.numOfIps; ++i) {
pIpList->ip[i] = pTableMeta->vgroupInfo.ipAddr[i].ip;
strcpy(pIpList->fqdn[i], pTableMeta->vgroupInfo.ipAddr[i].fqdn);
pIpList->port[i] = pTableMeta->vgroupInfo.ipAddr[i].port;
}
}
@ -62,7 +62,7 @@ void tscPrintMgmtIp() {
tscError("invalid mgmt IP list:%d", tscMgmtIpSet.numOfIps);
} else {
for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
tscTrace("mgmt index:%d ip:%d", i, tscMgmtIpSet.ip[i]);
tscTrace("mgmt index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
}
}
}
@ -70,9 +70,8 @@ void tscPrintMgmtIp() {
void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) {
tscMgmtIpSet.numOfIps = pIpList->numOfIps;
tscMgmtIpSet.inUse = pIpList->inUse;
tscMgmtIpSet.port = htons(pIpList->port);
for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
tscMgmtIpSet.ip[i] = htonl(pIpList->ip[i]);
tscMgmtIpSet.port[i] = htons(pIpList->port[i]);
}
}
@ -80,8 +79,7 @@ void tscSetMgmtIpListFromEdge() {
if (tscMgmtIpSet.numOfIps != 1) {
tscMgmtIpSet.numOfIps = 1;
tscMgmtIpSet.inUse = 0;
tscMgmtIpSet.port = tsMnodeShellPort;
tscMgmtIpSet.ip[0] = inet_addr(tsMasterIp);
taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
tscTrace("edge mgmt IP list:");
tscPrintMgmtIp();
}
@ -213,9 +211,6 @@ int tscSendMsgToServer(SSqlObj *pSql) {
rpcSendRequest(pVnodeConn, &pSql->ipList, &rpcMsg);
} else {
pSql->ipList = tscMgmtIpSet;
pSql->ipList.port = tsMnodeShellPort;
tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port);
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = {
.msgType = pSql->cmd.msgType,
@ -224,6 +219,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.handle = pSql,
.code = 0
};
tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]);
rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg);
}
@ -294,11 +290,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
}
}
}
if (pRes->code == TSDB_CODE_SUCCESS) {
tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry);
pSql->retry = 0;
}
pRes->rspLen = 0;
@ -308,6 +299,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code));
}
if (pRes->code == TSDB_CODE_SUCCESS) {
tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry);
pSql->retry = 0;
}
if (pRes->code != TSDB_CODE_QUERY_CANCELLED) {
assert(rpcMsg->msgType == pCmd->msgType + 1);
pRes->code = rpcMsg->code;
@ -569,7 +565,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
tscSetDnodeIpList(pSql, pTableMeta);
tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes));
tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d numberOfIP:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes), pSql->ipList.numOfIps);
return TSDB_CODE_SUCCESS;
}
@ -664,11 +660,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index];
pSql->ipList.numOfIps = pVgroupInfo->numOfIps; // todo fix me
pSql->ipList.port = tsDnodeShellPort;
pSql->ipList.inUse = 0;
for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) {
pSql->ipList.ip[i] = pVgroupInfo->ipAddr[i].ip;
strcpy(pSql->ipList.fqdn[i], pVgroupInfo->ipAddr[i].fqdn);
pSql->ipList.port[i] = pVgroupInfo->ipAddr[i].port;
}
tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index);
@ -935,7 +931,8 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload;
strncpy(pCreate->ip, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n);
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE;
return TSDB_CODE_SUCCESS;
@ -1078,7 +1075,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
strcpy(pDrop->ip, pTableMetaInfo->name);
strcpy(pDrop->ep, pTableMetaInfo->name);
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE;
return TSDB_CODE_SUCCESS;
@ -1857,10 +1854,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
}
for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) {
pMetaMsg->vgroup.ipAddr[i].ip = htonl(pMetaMsg->vgroup.ipAddr[i].ip);
pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port);
assert(pMetaMsg->vgroup.ipAddr[i].ip != 0);
}
SSchema* pSchema = pMetaMsg->schema;
@ -2144,7 +2138,6 @@ _error_clean:
assert(pVgroups->numOfIps >= 1);
for(int32_t j = 0; j < pVgroups->numOfIps; ++j) {
pVgroups->ipAddr[j].ip = htonl(pVgroups->ipAddr[j].ip);
pVgroups->ipAddr[j].port = htons(pVgroups->ipAddr[j].port);
}
}

28
src/client/src/tscSql.c
View File

@ -72,24 +72,24 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
return NULL;
}
tscMgmtIpSet.numOfIps = 0;
if (ip && ip[0]) {
tscMgmtIpSet.inUse = 0;
tscMgmtIpSet.port = tsMnodeShellPort;
tscMgmtIpSet.numOfIps = 1;
tscMgmtIpSet.ip[0] = inet_addr(ip);
if (tsMasterIp[0] && strcmp(ip, tsMasterIp) != 0) {
tscMgmtIpSet.numOfIps = 2;
tscMgmtIpSet.ip[1] = inet_addr(tsMasterIp);
strcpy(tscMgmtIpSet.fqdn[0], ip);
tscMgmtIpSet.port[0] = port? port: tsMnodeShellPort;
} else {
if (tsFirst[0] != 0) {
taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
tscMgmtIpSet.numOfIps++;
}
if (tsSecondIp[0] && strcmp(tsSecondIp, tsMasterIp) != 0) {
tscMgmtIpSet.numOfIps = 3;
tscMgmtIpSet.ip[2] = inet_addr(tsSecondIp);
if (tsSecond[0] != 0) {
taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
tscMgmtIpSet.numOfIps++;
}
}
tscMgmtIpSet.port = port ? port : tsMnodeShellPort;
STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
if (NULL == pObj) {
@ -167,10 +167,6 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) {
}
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
if (ip == NULL || (ip != NULL && (strcmp("127.0.0.1", ip) == 0 || strcasecmp("localhost", ip) == 0))) {
ip = tsMasterIp;
}
tscTrace("try to create a connection to %s", ip);
STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL);
@ -939,7 +935,7 @@ char *taos_errstr(TAOS *taos) {
void taos_config(int debug, char *log_path) {
uDebugFlag = debug;
strcpy(logDir, log_path);
strcpy(tsLogDir, log_path);
}
char *taos_get_server_info(TAOS *taos) {

8
src/client/src/tscSub.c
View File

@ -220,7 +220,7 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
static int tscLoadSubscriptionProgress(SSub* pSub) {
char buf[TSDB_MAX_SQL_LEN];
sprintf(buf, "%s/subscribe/%s", dataDir, pSub->topic);
sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(buf, "r");
if (fp == NULL) {
@ -281,12 +281,12 @@ void tscSaveSubscriptionProgress(void* sub) {
SSub* pSub = (SSub*)sub;
char path[256];
sprintf(path, "%s/subscribe", dataDir);
sprintf(path, "%s/subscribe", tsDataDir);
if (access(path, 0) != 0) {
mkdir(path, 0777);
}
sprintf(path, "%s/subscribe/%s", dataDir, pSub->topic);
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(path, "w+");
if (fp == NULL) {
tscError("failed to create progress file for subscription: %s", pSub->topic);
@ -416,7 +416,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
tscSaveSubscriptionProgress(pSub);
} else {
char path[256];
sprintf(path, "%s/subscribe/%s", dataDir, pSub->topic);
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
remove(path);
}

6
src/client/src/tscSubquery.c
View File

@ -1245,7 +1245,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
// data in from current vnode is stored in cache and disk
uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].ip, pTableMetaInfo->vgroupList->vgroups[0].vgId,
pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId,
numOfRowsFromSubquery, idx);
tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
@ -1473,12 +1473,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
if (pState->code != TSDB_CODE_SUCCESS) { // at least one peer subquery failed, abort current query
tscTrace("%p sub:%p query failed,ip:%u,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql,
pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex, pState->code);
pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex, pState->code);
tscHandleSubqueryError(param, tres, pState->code);
} else { // success, proceed to retrieve data from dnode
tscTrace("%p sub:%p query complete, ip:%u, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSqlObj, pSql,
pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex);
pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex);
if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode
tscRetrieveFromDnodeCallBack(param, pSql, 0);

23
src/client/src/tscSystem.c
View File

@ -55,7 +55,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) {
if (pVnodeConn == NULL) {
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsLocalIp;
rpcInit.localPort = 0;
rpcInit.label = "TSC-vnode";
rpcInit.numOfThreads = tscNumOfThreads;
@ -76,7 +75,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) {
if (*pMgmtConn == NULL) {
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsLocalIp;
rpcInit.localPort = 0;
rpcInit.label = "TSC-mgmt";
rpcInit.numOfThreads = 1;
@ -109,23 +107,17 @@ void taos_init_imp() {
deltaToUtcInitOnce();
if (tscEmbedded == 0) {
/*
* set localIp = 0
* means unset tsLocalIp in client
* except read from config file
*/
strcpy(tsLocalIp, "0.0.0.0");
// Read global configuration.
taosInitGlobalCfg();
taosReadGlobalLogCfg();
// For log directory
if (stat(logDir, &dirstat) < 0) mkdir(logDir, 0755);
if (stat(tsLogDir, &dirstat) < 0) mkdir(tsLogDir, 0755);
sprintf(temp, "%s/taoslog", logDir);
sprintf(temp, "%s/taoslog", tsLogDir);
if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) {
printf("failed to open log file in directory:%s\n", logDir);
printf("failed to open log file in directory:%s\n", tsLogDir);
}
taosReadGlobalCfg();
@ -133,7 +125,7 @@ void taos_init_imp() {
taosPrintGlobalCfg();
tscTrace("starting to initialize TAOS client ...");
tscTrace("Local IP address is:%s", tsLocalIp);
tscTrace("Local End Point is:%s", tsLocalEp);
}
taosSetCoreDump();
@ -143,13 +135,12 @@ void taos_init_imp() {
}
tscMgmtIpSet.inUse = 0;
tscMgmtIpSet.port = tsMnodeShellPort;
tscMgmtIpSet.numOfIps = 1;
tscMgmtIpSet.ip[0] = inet_addr(tsMasterIp);
taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
if (tsSecondIp[0] && strcmp(tsSecondIp, tsMasterIp) != 0) {
if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) {
tscMgmtIpSet.numOfIps = 2;
tscMgmtIpSet.ip[1] = inet_addr(tsSecondIp);
taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]);
}
tscInitMsgsFp();

69
src/client/src/tscUtil.c
View File

@ -631,47 +631,44 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
return TSDB_CODE_SUCCESS;
}
static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
int32_t firstPartLen = 0;
STableMeta* pTableMeta = pTableDataBlock->pTableMeta;
static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
// TODO: optimize this function
int len = 0;
STableMeta* pTableMeta = pTableDataBlock->pTableMeta;
STableComInfo tinfo = tscGetTableInfo(pTableMeta);
SSchema* pSchema = tscGetTableSchema(pTableMeta);
SSchema* pSchema = tscGetTableSchema(pTableMeta);
SSubmitBlk* pBlock = pDataBlock;
memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
pDataBlock += sizeof(SSubmitBlk);
int32_t total = sizeof(int32_t)*2;
for(int32_t i = 0; i < tinfo.numOfColumns; ++i) {
switch (pSchema[i].type) {
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_BINARY: {
assert(0); // not support binary yet
firstPartLen += sizeof(int32_t);break;
}
default:
firstPartLen += tDataTypeDesc[pSchema[i].type].nSize;
total += tDataTypeDesc[pSchema[i].type].nSize;
}
int32_t flen = 0;
for (int32_t i = 0; i < tinfo.numOfColumns; ++i) {
flen += TYPE_BYTES[pSchema[i].type];
}
char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
SSubmitBlk* pBlock = (SSubmitBlk*) pTableDataBlock->pData;
int32_t rows = htons(pBlock->numOfRows);
for(int32_t i = 0; i < rows; ++i) {
*(int32_t*) pDataBlock = total;
pDataBlock += sizeof(int32_t);
*(int32_t*) pDataBlock = firstPartLen;
pDataBlock += sizeof(int32_t);
memcpy(pDataBlock, p, pTableDataBlock->rowSize);
p += pTableDataBlock->rowSize;
pDataBlock += pTableDataBlock->rowSize;
pBlock->len = 0;
for (int32_t i = 0; i < htons(pBlock->numOfRows); ++i) {
SDataRow trow = (SDataRow)pDataBlock;
dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen);
int toffset = 0;
for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset);
toffset += TYPE_BYTES[pSchema[j].type];
p += pSchema[j].bytes;
}
// p += pTableDataBlock->rowSize;
pDataBlock += dataRowLen(trow);
pBlock->len += dataRowLen(trow);
}
len = pBlock->len;
pBlock->len = htonl(pBlock->len);
return len;
}
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) {
@ -734,7 +731,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
pBlocks->len = htonl(len);
// erase the empty space reserved for binary data
trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
dataBuf->size += (len + sizeof(SSubmitBlk));
dataBuf->numOfTables += 1;
}

121
src/common/inc/tdataformat.h
View File

@ -20,6 +20,7 @@
#include <string.h>
#include "taosdef.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -30,7 +31,7 @@ typedef struct {
int8_t type; // Column type
int16_t colId; // column ID
int32_t bytes; // column bytes
int32_t offset; // point offset in a row data
int32_t offset; // point offset in SDataRow after the header part
} STColumn;
#define colType(col) ((col)->type)
@ -43,26 +44,25 @@ typedef struct {
#define colSetBytes(col, b) (colBytes(col) = (b))
#define colSetOffset(col, o) (colOffset(col) = (o))
STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes);
void tdFreeCol(STColumn *pCol);
void tdColCpy(STColumn *dst, STColumn *src);
void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes);
// ----------------- TSDB SCHEMA DEFINITION
typedef struct {
int totalCols; // Total columns allocated
int numOfCols; // Number of columns appended
int padding; // Total columns allocated
int tlen; // maximum length of a SDataRow without the header part
int flen; // First part length in a SDataRow after the header part
STColumn columns[];
} STSchema;
#define schemaNCols(s) ((s)->numOfCols)
#define schemaTotalCols(s) ((s)->totalCols)
#define schemaTLen(s) ((s)->tlen)
#define schemaFLen(s) ((s)->flen)
#define schemaColAt(s, i) ((s)->columns + i)
STSchema *tdNewSchema(int32_t nCols);
int tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
#define tdFreeSchema(s) tfree((s))
int tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
STSchema *tdDupSchema(STSchema *pSchema);
void tdFreeSchema(STSchema *pSchema);
void tdUpdateSchema(STSchema *pSchema);
int tdGetSchemaEncodeSize(STSchema *pSchema);
void * tdEncodeSchema(void *dst, STSchema *pSchema);
STSchema *tdDecodeSchema(void **psrc);
@ -70,53 +70,100 @@ STSchema *tdDecodeSchema(void **psrc);
// ----------------- Data row structure
/* A data row, the format is like below:
* +----------+---------+---------------------------------+---------------------------------+
* | int32_t | int32_t | | |
* +----------+---------+---------------------------------+---------------------------------+
* | len | flen | First part | Second part |
* +----------+---------+---------------------------------+---------------------------------+
* plen: first part length
* len: the length including sizeof(row) + sizeof(len)
* row: actual row data encoding
* |<------------------------------------- len ---------------------------------->|
* |<--Head ->|<--------- flen -------------->| |
* +----------+---------------------------------+---------------------------------+
* | int32_t | | |
* +----------+---------------------------------+---------------------------------+
* | len | First part | Second part |
* +----------+---------------------------------+---------------------------------+
*/
typedef void *SDataRow;
#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
#define dataRowLen(r) (*(int32_t *)(r))
#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowTuple(r) POINTER_DRIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
#define dataRowIdx(r, i) ((char *)(r) + i)
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowAt(r, idx) ((char *)(r) + (idx))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
void tdInitDataRow(SDataRow row, STSchema *pSchema);
int tdMaxRowBytesFromSchema(STSchema *pSchema);
SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema);
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
int tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
void tdDataRowReset(SDataRow row, STSchema *pSchema);
void tdInitDataRow(SDataRow row, STSchema *pSchema);
int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
SDataRow tdDataRowDup(SDataRow row);
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return POINTER_DRIFT(row, *(VarDataOffsetT *)POINTER_DRIFT(row, offset));
break;
default:
return POINTER_DRIFT(row, offset);
break;
}
}
// ----------------- Data column structure
typedef struct SDataCol {
int8_t type;
int16_t colId;
int bytes;
int len;
int offset;
void * pData; // Original data
int8_t type; // column type
int16_t colId; // column ID
int bytes; // column data bytes defined
int offset; // data offset in a SDataRow (including the header size)
int spaceSize; // Total space size for this column
int len; // column data length
VarDataOffsetT *dataOff; // For binary and nchar data, the offset in the data column
void * pData; // Actual data pointer
} SDataCol;
static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints);
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints);
void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle);
void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return POINTER_DRIFT(pCol->pData, pCol->dataOff[row]);
break;
default:
return POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
break;
}
}
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
ASSERT(rows > 0);
switch (pDataCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
break;
default:
return TYPE_BYTES[pDataCol->type] * rows;
}
}
typedef struct {
int maxRowSize;
int maxCols; // max number of columns
int maxPoints; // max number of points
int bufSize;
int numOfPoints;
int numOfCols; // Total number of cols
int sversion; // TODO: set sversion
@ -125,7 +172,7 @@ typedef struct {
} SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
#define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1)

17
src/common/inc/tglobal.h
View File

@ -24,10 +24,10 @@ extern char configDir[];
extern char tsVnodeDir[];
extern char tsDnodeDir[];
extern char tsMnodeDir[];
extern char dataDir[];
extern char logDir[];
extern char scriptDir[];
extern char osName[];
extern char tsDataDir[];
extern char tsLogDir[];
extern char tsScriptDir[];
extern char tsOsName[];
// system info
extern int64_t tsPageSize;
@ -51,8 +51,10 @@ extern int32_t tsVersion;
extern int32_t tscEmbedded;
extern int64_t tsMsPerDay[2];
extern char tsMasterIp[];
extern char tsSecondIp[];
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern uint16_t tsMnodeDnodePort;
extern uint16_t tsMnodeShellPort;
extern uint16_t tsDnodeShellPort;
@ -178,7 +180,8 @@ void taosInitGlobalCfg();
bool taosCheckGlobalCfg();
void taosSetAllDebugFlag();
bool taosCfgDynamicOptions(char *msg);
int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port);
#ifdef __cplusplus
}
#endif

450
src/common/src/tdataformat.c
View File

@ -13,72 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdataformat.h"
#include "tutil.h"
static int tdFLenFromSchema(STSchema *pSchema);
/**
* Create a new STColumn object
* ASSUMPTIONS: VALID PARAMETERS
*
* @param type column type
* @param colId column ID
* @param bytes maximum bytes the col taken
*
* @return a STColumn object on success
* NULL for failure
*/
STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes) {
if (!isValidDataType(type, 0)) return NULL;
STColumn *pCol = (STColumn *)calloc(1, sizeof(STColumn));
if (pCol == NULL) return NULL;
colSetType(pCol, type);
colSetColId(pCol, colId);
colSetOffset(pCol, -1);
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
colSetBytes(pCol, bytes);
break;
default:
colSetBytes(pCol, TYPE_BYTES[type]);
break;
}
return pCol;
}
/**
* Free a STColumn object CREATED with tdNewCol
*/
void tdFreeCol(STColumn *pCol) {
if (pCol) free(pCol);
}
/**
* Copy from source to destinition
*/
void tdColCpy(STColumn *dst, STColumn *src) { memcpy((void *)dst, (void *)src, sizeof(STColumn)); }
/**
* Set the column
*/
void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) {
colSetType(pCol, type);
colSetColId(pCol, colId);
switch (type)
{
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
colSetBytes(pCol, bytes);
break;
default:
colSetBytes(pCol, TYPE_BYTES[type]);
break;
}
}
#include "wchar.h"
/**
* Create a SSchema object with nCols columns
@ -90,11 +25,15 @@ void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) {
* NULL for failure
*/
STSchema *tdNewSchema(int32_t nCols) {
int32_t size = sizeof(STSchema) + sizeof(STColumn) * nCols;
int32_t size = sizeof(STSchema) + sizeof(STColumn) * nCols;
STSchema *pSchema = (STSchema *)calloc(1, size);
if (pSchema == NULL) return NULL;
pSchema->numOfCols = 0;
pSchema->totalCols = nCols;
pSchema->flen = 0;
pSchema->tlen = 0;
return pSchema;
}
@ -102,25 +41,34 @@ STSchema *tdNewSchema(int32_t nCols) {
/**
* Append a column to the schema
*/
int tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes) {
// if (pSchema->numOfCols >= pSchema->totalCols) return -1;
if (!isValidDataType(type, 0)) return -1;
int tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes) {
if (!isValidDataType(type, 0) || pSchema->numOfCols >= pSchema->totalCols) return -1;
STColumn *pCol = schemaColAt(pSchema, schemaNCols(pSchema));
colSetType(pCol, type);
colSetColId(pCol, colId);
colSetOffset(pCol, -1);
if (schemaNCols(pSchema) == 0) {
colSetOffset(pCol, 0);
} else {
STColumn *pTCol = schemaColAt(pSchema, schemaNCols(pSchema)-1);
colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
}
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
colSetBytes(pCol, bytes);
colSetBytes(pCol, bytes); // Set as maximum bytes
pSchema->tlen += (TYPE_BYTES[type] + sizeof(VarDataLenT) + bytes);
break;
default:
colSetBytes(pCol, TYPE_BYTES[type]);
pSchema->tlen += TYPE_BYTES[type];
break;
}
pSchema->numOfCols++;
pSchema->flen += TYPE_BYTES[type];
ASSERT(pCol->offset < pSchema->flen);
return 0;
}
@ -138,40 +86,22 @@ STSchema *tdDupSchema(STSchema *pSchema) {
return tSchema;
}
/**
* Free the SSchema object created by tdNewSchema or tdDupSchema
*/
void tdFreeSchema(STSchema *pSchema) {
if (pSchema != NULL) free(pSchema);
}
/**
* Function to update each columns's offset field in the schema.
* ASSUMPTIONS: VALID PARAMETERS
*/
void tdUpdateSchema(STSchema *pSchema) {
STColumn *pCol = NULL;
int32_t offset = TD_DATA_ROW_HEAD_SIZE;
for (int i = 0; i < schemaNCols(pSchema); i++) {
pCol = schemaColAt(pSchema, i);
colSetOffset(pCol, offset);
offset += TYPE_BYTES[pCol->type];
}
}
/**
* Return the size of encoded schema
*/
int tdGetSchemaEncodeSize(STSchema *pSchema) {
return sizeof(STSchema) + schemaNCols(pSchema) * (T_MEMBER_SIZE(STColumn, type) + T_MEMBER_SIZE(STColumn, colId) +
T_MEMBER_SIZE(STColumn, bytes));
return T_MEMBER_SIZE(STSchema, totalCols) +
schemaNCols(pSchema) *
(T_MEMBER_SIZE(STColumn, type) + T_MEMBER_SIZE(STColumn, colId) + T_MEMBER_SIZE(STColumn, bytes));
}
/**
* Encode a schema to dst, and return the next pointer
*/
void *tdEncodeSchema(void *dst, STSchema *pSchema) {
T_APPEND_MEMBER(dst, pSchema, STSchema, numOfCols);
ASSERT(pSchema->numOfCols == pSchema->totalCols);
T_APPEND_MEMBER(dst, pSchema, STSchema, totalCols);
for (int i = 0; i < schemaNCols(pSchema); i++) {
STColumn *pCol = schemaColAt(pSchema, i);
T_APPEND_MEMBER(dst, pCol, STColumn, type);
@ -186,13 +116,13 @@ void *tdEncodeSchema(void *dst, STSchema *pSchema) {
* Decode a schema from a binary.
*/
STSchema *tdDecodeSchema(void **psrc) {
int numOfCols = 0;
int totalCols = 0;
T_READ_MEMBER(*psrc, int, numOfCols);
T_READ_MEMBER(*psrc, int, totalCols);
STSchema *pSchema = tdNewSchema(numOfCols);
STSchema *pSchema = tdNewSchema(totalCols);
if (pSchema == NULL) return NULL;
for (int i = 0; i < numOfCols; i++) {
for (int i = 0; i < totalCols; i++) {
int8_t type = 0;
int16_t colId = 0;
int32_t bytes = 0;
@ -200,7 +130,7 @@ STSchema *tdDecodeSchema(void **psrc) {
T_READ_MEMBER(*psrc, int16_t, colId);
T_READ_MEMBER(*psrc, int32_t, bytes);
tdSchemaAppendCol(pSchema, type, colId, bytes);
tdSchemaAddCol(pSchema, type, colId, bytes);
}
return pSchema;
@ -209,53 +139,18 @@ STSchema *tdDecodeSchema(void **psrc) {
/**
* Initialize a data row
*/
void tdInitDataRow(SDataRow row, STSchema *pSchema) {
dataRowSetFLen(row, TD_DATA_ROW_HEAD_SIZE);
dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + tdFLenFromSchema(pSchema));
}
void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); }
/**
* Create a data row with maximum row length bytes.
*
* NOTE: THE AAPLICATION SHOULD MAKE SURE BYTES IS LARGE ENOUGH TO
* HOLD THE WHOE ROW.
*
* @param bytes max bytes a row can take
* @return SDataRow object for success
* NULL for failure
*/
SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema) {
int32_t size = sizeof(int32_t) + bytes;
SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
int32_t size = dataRowMaxBytesFromSchema(pSchema);
SDataRow row = malloc(size);
if (row == NULL) return NULL;
tdInitDataRow(row, pSchema);
return row;
}
/**
* Get maximum bytes a data row from a schema
* ASSUMPTIONS: VALID PARAMETER
*/
int tdMaxRowBytesFromSchema(STSchema *pSchema) {
// TODO
int bytes = TD_DATA_ROW_HEAD_SIZE;
for (int i = 0; i < schemaNCols(pSchema); i++) {
STColumn *pCol = schemaColAt(pSchema, i);
bytes += TYPE_BYTES[pCol->type];
if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
bytes += pCol->bytes;
}
}
return bytes;
}
SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { return tdNewDataRow(tdMaxRowBytesFromSchema(pSchema), pSchema); }
/**
* Free the SDataRow object
*/
@ -265,28 +160,49 @@ void tdFreeDataRow(SDataRow row) {
/**
* Append a column value to the data row
* @param type: column type
* @param bytes: column bytes
* @param offset: offset in the data row tuple, not including the data row header
*/
int tdAppendColVal(SDataRow row, void *value, STColumn *pCol) {
switch (colType(pCol))
{
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
*(int32_t *)dataRowAt(row, dataRowFLen(row)) = dataRowLen(row);
dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
memcpy((void *)dataRowAt(row, dataRowLen(row)), value, strlen(value));
dataRowLen(row) += strlen(value);
break;
default:
memcpy(dataRowAt(row, dataRowFLen(row)), value, TYPE_BYTES[colType(pCol)]);
dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
break;
int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
ASSERT(value != NULL);
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = POINTER_DRIFT(row, dataRowLen(row));
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
// set offset
*(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row);
// set length
VarDataLenT slen = 0;
if (isNull(value, type)) {
slen = (type == TSDB_DATA_TYPE_BINARY) ? sizeof(int8_t) : sizeof(int32_t);
} else {
if (type == TSDB_DATA_TYPE_BINARY) {
slen = strnlen((char *)value, bytes);
} else {
slen = wcsnlen((wchar_t *)value, (bytes) / TSDB_NCHAR_SIZE) * TSDB_NCHAR_SIZE;
}
}
ASSERT(slen <= bytes);
*(VarDataLenT *)ptr = slen;
ptr = POINTER_DRIFT(ptr, sizeof(VarDataLenT));
memcpy((void *)ptr, value, slen);
dataRowLen(row) += (sizeof(int16_t) + slen);
break;
default:
memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]);
break;
}
return 0;
}
void tdDataRowReset(SDataRow row, STSchema *pSchema) { tdInitDataRow(row, pSchema); }
SDataRow tdDataRowDup(SDataRow row) {
SDataRow trow = malloc(dataRowLen(row));
if (trow == NULL) return NULL;
@ -295,6 +211,119 @@ SDataRow tdDataRowDup(SDataRow row) {
return trow;
}
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) {
pDataCol->type = colType(pCol);
pDataCol->colId = colColId(pCol);
pDataCol->bytes = colBytes(pCol);
pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
pDataCol->len = 0;
if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
pDataCol->spaceSize = (sizeof(VarDataLenT) + pDataCol->bytes) * maxPoints;
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_DRIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints);
*pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints);
} else {
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
pDataCol->dataOff = NULL;
pDataCol->pData = *pBuf;
*pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize);
}
}
void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
// set offset
pCol->dataOff[numOfPoints] = pCol->len;
// Copy data
memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, varDataTLen(value));
// Update the length
pCol->len += varDataTLen(value);
break;
default:
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes;
break;
}
}
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
int pointsLeft = numOfPoints - pointsToPop;
ASSERT(pointsLeft > 0);
if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
ASSERT(pCol->len > 0);
VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
pCol->len = pCol->len - toffset;
ASSERT(pCol->len > 0);
memmove(pCol->pData, POINTER_DRIFT(pCol->pData, toffset), pCol->len);
dataColSetOffset(pCol, pointsLeft);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
memmove(pCol->pData, POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
}
}
bool isNEleNull(SDataCol *pCol, int nEle) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
for (int i = 0; i < nEle; i++) {
if (!isNull(varDataVal(tdGetColDataOfRow(pCol, i)), pCol->type)) return false;
}
return true;
default:
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
}
}
void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
char *ptr = NULL;
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
pCol->len = 0;
for (int i = 0; i < nEle; i++) {
pCol->dataOff[i] = pCol->len;
ptr = (char *)pCol->pData + pCol->len;
varDataLen(ptr) = (pCol->type == TSDB_DATA_TYPE_BINARY) ? sizeof(char) : TSDB_NCHAR_SIZE;
setNull(ptr + sizeof(VarDataLenT), pCol->type, pCol->bytes);
pCol->len += varDataTLen(ptr);
}
break;
default:
setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
pCol->len = TYPE_BYTES[pCol->type] * nEle;
break;
}
}
void dataColSetOffset(SDataCol *pCol, int nEle) {
ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
void * tptr = pCol->pData;
// char *tptr = (char *)(pCol->pData);
VarDataOffsetT offset = 0;
for (int i = 0; i < nEle; i++) {
pCol->dataOff[i] = offset;
offset += varDataTLen(tptr);
tptr = POINTER_DRIFT(tptr, varDataTLen(tptr));
}
}
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols);
if (pCols == NULL) return NULL;
@ -302,8 +331,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
pCols->maxRowSize = maxRowSize;
pCols->maxCols = maxCols;
pCols->maxPoints = maxRows;
pCols->bufSize = maxRowSize * maxRows;
pCols->buf = malloc(maxRowSize * maxRows);
pCols->buf = malloc(pCols->bufSize);
if (pCols->buf == NULL) {
free(pCols);
return NULL;
@ -317,24 +347,16 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
tdResetDataCols(pCols);
pCols->numOfCols = schemaNCols(pSchema);
pCols->cols[0].pData = pCols->buf;
int offset = TD_DATA_ROW_HEAD_SIZE;
void *ptr = pCols->buf;
for (int i = 0; i < schemaNCols(pSchema); i++) {
if (i > 0) {
pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints;
}
pCols->cols[i].type = colType(schemaColAt(pSchema, i));
pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i));
pCols->cols[i].offset = offset;
pCols->cols[i].colId = colColId(schemaColAt(pSchema, i));
offset += TYPE_BYTES[pCols->cols[i].type];
dataColInit(pCols->cols + i, schemaColAt(pSchema, i), &ptr, pCols->maxPoints);
ASSERT((char *)ptr - (char *)(pCols->buf) <= pCols->bufSize);
}
}
void tdFreeDataCols(SDataCols *pCols) {
if (pCols) {
if (pCols->buf) free(pCols->buf);
tfree(pCols->buf);
free(pCols);
}
}
@ -351,11 +373,24 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].type = pDataCols->cols[i].type;
pRet->cols[i].colId = pDataCols->cols[i].colId;
pRet->cols[i].bytes = pDataCols->cols[i].bytes;
pRet->cols[i].len = pDataCols->cols[i].len;
pRet->cols[i].offset = pDataCols->cols[i].offset;
pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].bytes * pDataCols->numOfPoints);
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
ASSERT(pDataCols->cols[i].dataOff != NULL);
pRet->cols[i].dataOff =
(int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
}
if (keepData) {
pRet->cols[i].len = pDataCols->cols[i].len;
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
}
}
}
return pRet;
@ -364,57 +399,60 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
void tdResetDataCols(SDataCols *pCols) {
pCols->numOfPoints = 0;
for (int i = 0; i < pCols->maxCols; i++) {
pCols->cols[i].len = 0;
dataColReset(pCols->cols + i);
}
}
void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) {
ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
for (int i = 0; i < pCols->numOfCols; i++) {
SDataCol *pCol = pCols->cols + i;
memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes);
pCol->len += pCol->bytes;
void * value = tdGetRowDataOfCol(row, pCol->type, pCol->offset);
dataColAppendVal(pCol, value, pCols->numOfPoints, pCols->maxPoints);
}
pCols->numOfPoints++;
}
// Pop pointsToPop points from the SDataCols
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
int pointsLeft = pCols->numOfPoints - pointsToPop;
if (pointsLeft <= 0) {
tdResetDataCols(pCols);
return;
}
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SDataCol *p_col = pCols->cols + iCol;
if (p_col->len > 0) {
p_col->len = TYPE_BYTES[p_col->type] * pointsLeft;
if (pointsLeft > 0) {
memmove((void *)(p_col->pData), (void *)((char *)(p_col->pData) + TYPE_BYTES[p_col->type] * pointsToPop), p_col->len);
}
}
SDataCol *pCol = pCols->cols + iCol;
dataColPopPoints(pCol, pointsToPop, pCols->numOfPoints);
}
pCols->numOfPoints = pointsLeft;
}
/**
* Return the first part length of a data row for a schema
*/
static int tdFLenFromSchema(STSchema *pSchema) {
int ret = 0;
for (int i = 0; i < schemaNCols(pSchema); i++) {
STColumn *pCol = schemaColAt(pSchema, i);
ret += TYPE_BYTES[pCol->type];
}
return ret;
}
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints);
ASSERT(target->numOfPoints + rowsToMerge <= target->maxPoints);
ASSERT(target->numOfCols == source->numOfCols);
SDataCols *pTarget = tdDupDataCols(target, true);
if (pTarget == NULL) goto _err;
// tdResetDataCols(target);
SDataCols *pTarget = NULL;
int iter1 = 0;
int iter2 = 0;
tdMergeTwoDataCols(target,pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap
for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) {
dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfPoints,
target->maxPoints);
}
}
target->numOfPoints++;
} else {
pTarget = tdDupDataCols(target, true);
if (pTarget == NULL) goto _err;
int iter1 = 0;
int iter2 = 0;
tdMergeTwoDataCols(target, pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
}
tdFreeDataCols(pTarget);
return 0;
@ -425,6 +463,7 @@ _err:
}
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) {
// TODO: add resolve duplicate key here
tdResetDataCols(target);
while (target->numOfPoints < tRows) {
@ -436,10 +475,8 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
(void *)((char *)(src1->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * (*iter1)),
TYPE_BYTES[target->cols[i].type]);
target->cols[i].len += TYPE_BYTES[target->cols[i].type];
dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
target->maxPoints);
}
target->numOfPoints++;
@ -447,15 +484,14 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
} else if (key1 > key2) {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
(void *)((char *)(src2->cols[i].pData) + TYPE_BYTES[src2->cols[i].type] * (*iter2)),
TYPE_BYTES[target->cols[i].type]);
target->cols[i].len += TYPE_BYTES[target->cols[i].type];
dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
target->maxPoints);
}
target->numOfPoints++;
(*iter2)++;
} else {
// TODO: deal with duplicate keys
ASSERT(false);
}
}

223
src/common/src/tglobal.c
View File

@ -25,14 +25,15 @@
#include "tutil.h"
#include "tlocale.h"
#include "ttimezone.h"
#include "tsync.h"
char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
char dataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
char scriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
char osName[10] = "Linux";
char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos";
char tsOsName[10] = "Linux";
// system info, not configurable
int64_t tsPageSize;
@ -60,8 +61,11 @@ int32_t tscEmbedded = 0;
*/
int64_t tsMsPerDay[] = {86400000L, 86400000000L};
char tsMasterIp[TSDB_IPv4ADDR_LEN] = {0};
char tsSecondIp[TSDB_IPv4ADDR_LEN] = {0};
char tsFirst[TSDB_FQDN_LEN] = {0};
char tsSecond[TSDB_FQDN_LEN] = {0};
char tsArbitrator[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_FQDN_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030;
uint16_t tsMnodeShellPort = 6030; // udp[6030-6034] tcp[6030]
uint16_t tsDnodeShellPort = 6035; // udp[6035-6039] tcp[6035]
uint16_t tsMnodeDnodePort = 6040; // udp/tcp
@ -70,8 +74,6 @@ uint16_t tsSyncPort = 6050;
int32_t tsStatusInterval = 1; // second
int32_t tsShellActivityTimer = 3; // second
int32_t tsVnodePeerHBTimer = 1; // second
int32_t tsMgmtPeerHBTimer = 1; // second
int32_t tsMeterMetaKeepTimer = 7200; // second
int32_t tsMetricMetaKeepTimer = 600; // second
int32_t tsRpcTimer = 300;
@ -79,8 +81,6 @@ int32_t tsRpcMaxTime = 600; // seconds;
float tsNumOfThreadsPerCore = 1.0;
float tsRatioOfQueryThreads = 0.5;
char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0};
char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0};
int16_t tsNumOfVnodesPerCore = 8;
int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
@ -112,7 +112,6 @@ int32_t tsNumOfMPeers = 3;
int32_t tsMaxShellConns = 2000;
int32_t tsMaxTables = 100000;
char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0};
char tsDefaultDB[TSDB_DB_NAME_LEN] = {0};
char tsDefaultUser[64] = "root";
char tsDefaultPass[64] = "taosdata";
@ -144,6 +143,7 @@ int32_t qdebugFlag = 131;
int32_t rpcDebugFlag = 131;
int32_t uDebugFlag = 131;
int32_t debugFlag = 131;
int32_t sDebugFlag = 131;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
@ -183,7 +183,6 @@ float tsStreamComputDelayRatio = 0.1;
int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once
int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance
char tsHttpIp[TSDB_IPv4ADDR_LEN] = "0.0.0.0";
uint16_t tsHttpPort = 6020; // only tcp, range tcp[6020]
// uint16_t tsNginxPort = 6060; //only tcp, range tcp[6060]
int32_t tsHttpCacheSessions = 100;
@ -194,7 +193,6 @@ int32_t tsHttpEnableRecordSql = 0;
int32_t tsTelegrafUseFieldNum = 0;
int32_t tsTscEnableRecordSql = 0;
int32_t tsAnyIp = 1;
uint32_t tsPublicIpInt = 0;
char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
@ -274,69 +272,29 @@ static void doInitGlobalConfig() {
SGlobalCfg cfg = {0};
// ip address
cfg.option = "masterIp";
cfg.ptr = tsMasterIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.option = "first";
cfg.ptr = tsFirst;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "secondIp";
cfg.ptr = tsSecondIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.option = "second";
cfg.ptr = tsSecond;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "publicIp";
cfg.ptr = tsPublicIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "privateIp";
cfg.ptr = tsPrivateIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "localIp";
cfg.ptr = tsLocalIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "httpIp";
cfg.ptr = tsHttpIp;
cfg.valType = TAOS_CFG_VTYPE_IPSTR;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_IPv4ADDR_LEN;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// port
cfg.option = "mnodeShellPort";
cfg.ptr = &tsMnodeShellPort;
cfg.option = "serverPort";
cfg.ptr = &tsServerPort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
@ -345,56 +303,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "dnodeShellPort";
cfg.ptr = &tsDnodeShellPort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mnodeDnodePort";
cfg.ptr = &tsMnodeDnodePort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "dnodeMnodePort";
cfg.ptr = &tsDnodeMnodePort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// cfg.option = "syncPort";
// cfg.ptr = &syncPort;
// cfg.valType = TAOS_CFG_VTYPE_INT16;
// cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
// cfg.minValue = 1;
// cfg.maxValue = 65535;
// cfg.ptrLength = 0;
// cfg.unitType = TAOS_CFG_UTYPE_NONE;
// taosInitConfigOption(cfg);
cfg.option = "httpPort";
cfg.ptr = &tsHttpPort;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 65535;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// directory
cfg.option = "configDir";
cfg.ptr = configDir;
@ -407,7 +315,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
cfg.option = "logDir";
cfg.ptr = logDir;
cfg.ptr = tsLogDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
@ -417,7 +325,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
cfg.option = "scriptDir";
cfg.ptr = scriptDir;
cfg.ptr = tsScriptDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
@ -427,7 +335,7 @@ static void doInitGlobalConfig() {
taosInitConfigOption(cfg);
cfg.option = "dataDir";
cfg.ptr = dataDir;
cfg.ptr = tsDataDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
@ -436,6 +344,16 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "arbitrator";
cfg.ptr = tsArbitrator;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// dnode configs
cfg.option = "numOfThreadsPerCore";
cfg.ptr = &tsNumOfThreadsPerCore;
@ -1107,6 +1025,16 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "sDebugFlag";
cfg.ptr = &sDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "sdbDebugFlag";
cfg.ptr = &sdbDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1227,16 +1155,6 @@ static void doInitGlobalConfig() {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "anyIp";
cfg.ptr = &tsAnyIp;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// version info
cfg.option = "gitinfo";
cfg.ptr = gitinfo;
@ -1284,25 +1202,16 @@ void taosInitGlobalCfg() {
}
bool taosCheckGlobalCfg() {
if (tsPrivateIp[0] == 0) {
taosGetPrivateIp(tsPrivateIp);
taosGetFqdn(tsLocalEp);
sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
uPrint("localEp is %s", tsLocalEp);
if (tsFirst[0] == 0) {
strcpy(tsFirst, tsLocalEp);
}
if (tsPublicIp[0] == 0) {
strcpy(tsPublicIp, tsPrivateIp);
}
tsPublicIpInt = inet_addr(tsPublicIp);
if (tsLocalIp[0] == 0) {
strcpy(tsLocalIp, tsPrivateIp);
}
if (tsMasterIp[0] == 0) {
strcpy(tsMasterIp, tsPrivateIp);
}
if (tsSecondIp[0] == 0) {
strcpy(tsSecondIp, tsMasterIp);
if (tsSecond[0] == 0) {
strcpy(tsSecond, tsLocalEp);
}
taosGetSystemInfo();
@ -1324,15 +1233,6 @@ bool taosCheckGlobalCfg() {
tsNumOfTotalVnodes = tsNumOfTotalVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : tsNumOfTotalVnodes;
}
if (strlen(tsPrivateIp) == 0) {
uError("privateIp is null");
return false;
}
if (tscEmbedded) {
strcpy(tsLocalIp, tsPrivateIp);
}
// todo refactor
tsVersion = 0;
for (int i = 0; i < 10; i++) {
@ -1345,5 +1245,26 @@ bool taosCheckGlobalCfg() {
tsVersion = 10 * tsVersion;
tsMnodeShellPort = tsServerPort + TSDB_PORT_MNODESHELL; // udp[6030-6034] tcp[6030]
tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035]
tsMnodeDnodePort = tsServerPort + TSDB_PORT_MNODEDNODE; // udp/tcp
tsDnodeMnodePort = tsServerPort + TSDB_PORT_DNODEMNODE; // udp/tcp
tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
return true;
}
int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) {
*port = 0;
strcpy(fqdn, ep);
char *temp = strchr(fqdn, ':');
if (temp) {
*temp = 0;
*port = atoi(temp+1);
}
if (*port == 0) *port = tsServerPort;
return 0;
}

45
src/common/src/ttypes.c
View File

@ -16,33 +16,34 @@
#include "taosdef.h"
#include "ttokendef.h"
#include "tscompression.h"
const int32_t TYPE_BYTES[11] = {
-1, // TSDB_DATA_TYPE_NULL
sizeof(int8_t), // TSDB_DATA_TYPE_BOOL
sizeof(int8_t), // TSDB_DATA_TYPE_TINYINT
sizeof(int16_t), // TSDB_DATA_TYPE_SMALLINT
sizeof(int32_t), // TSDB_DATA_TYPE_INT
sizeof(int64_t), // TSDB_DATA_TYPE_BIGINT
sizeof(float), // TSDB_DATA_TYPE_FLOAT
sizeof(double), // TSDB_DATA_TYPE_DOUBLE
sizeof(int32_t), // TSDB_DATA_TYPE_BINARY
sizeof(TSKEY), // TSDB_DATA_TYPE_TIMESTAMP
sizeof(int32_t) // TSDB_DATA_TYPE_NCHAR
-1, // TSDB_DATA_TYPE_NULL
sizeof(int8_t), // TSDB_DATA_TYPE_BOOL
sizeof(int8_t), // TSDB_DATA_TYPE_TINYINT
sizeof(int16_t), // TSDB_DATA_TYPE_SMALLINT
sizeof(int32_t), // TSDB_DATA_TYPE_INT
sizeof(int64_t), // TSDB_DATA_TYPE_BIGINT
sizeof(float), // TSDB_DATA_TYPE_FLOAT
sizeof(double), // TSDB_DATA_TYPE_DOUBLE
sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_BINARY
sizeof(TSKEY), // TSDB_DATA_TYPE_TIMESTAMP
sizeof(VarDataOffsetT) // TSDB_DATA_TYPE_NCHAR
};
tDataTypeDescriptor tDataTypeDesc[11] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE"},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL"},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT"},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT"},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT"},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT"},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT"},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE"},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY"},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP"},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR"},
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble},
{TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp},
{TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString},
};
char tTokenTypeSwitcher[13] = {

81
src/dnode/src/dnodeMClient.c
View File

@ -59,16 +59,13 @@ void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
tsMnodeIpSet = *pIpSet;
}
void dnodeGetMnodeIpSet(void *ipSetRaw, bool usePublicIp) {
void dnodeGetMnodeDnodeIpSet(void *ipSetRaw) {
SRpcIpSet *ipSet = ipSetRaw;
ipSet->numOfIps = tsMnodeInfos.nodeNum;
ipSet->inUse = tsMnodeInfos.inUse;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) {
if (usePublicIp) {
ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
} else {
ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
}
taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, ipSet->fqdn[i], &ipSet->port[i]);
ipSet->port[i] += TSDB_PORT_MNODEDNODE;
}
}
@ -85,26 +82,25 @@ int32_t dnodeInitMClient() {
if (!dnodeReadMnodeInfos()) {
memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
tsMnodeIpSet.port = tsMnodeDnodePort;
tsMnodeIpSet.numOfIps = 1;
tsMnodeIpSet.ip[0] = inet_addr(tsMasterIp);
if (strcmp(tsSecondIp, tsMasterIp) != 0) {
taosGetFqdnPortFromEp(tsFirst, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
tsMnodeIpSet.port[0] += TSDB_PORT_MNODEDNODE;
if (strcmp(tsSecond, tsFirst) != 0) {
tsMnodeIpSet.numOfIps = 2;
tsMnodeIpSet.ip[1] = inet_addr(tsSecondIp);
taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
tsMnodeIpSet.port[1] += TSDB_PORT_MNODEDNODE;
}
} else {
tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
tsMnodeIpSet.port = tsMnodeInfos.nodeInfos[0].nodePort;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
tsMnodeIpSet.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
}
}
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = 0;
rpcInit.label = "DND-MC";
rpcInit.numOfThreads = 1;
rpcInit.cfp = dnodeProcessRspFromMnode;
@ -182,9 +178,6 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId);
pMnodeInfo->nodeIp = htonl(pMnodeInfo->nodeIp);
pMnodeInfo->nodePort = htons(pMnodeInfo->nodePort);
pMnodeInfo->syncPort = htons(pMnodeInfo->syncPort);
}
SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess;
@ -207,15 +200,14 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
tsMnodeIpSet.port = tsMnodeInfos.nodeInfos[0].nodePort;
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
tsMnodeIpSet.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp;
taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
}
dPrint("mnodes is changed, nodeNum:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp),
tsMnodeInfos.nodeInfos[i].nodePort, tsMnodeInfos.nodeInfos[i].nodeName);
dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
}
dnodeSaveMnodeInfos();
@ -291,42 +283,19 @@ static bool dnodeReadMnodeInfos() {
}
tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp");
if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeIp not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring);
cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort");
if (!nodePort || nodePort->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, nodePort not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint;
cJSON *syncPort = cJSON_GetObjectItem(nodeInfo, "syncPort");
if (!syncPort || syncPort->type != cJSON_Number) {
dError("failed to read mnode mgmtIpList.json, syncPort not found");
goto PARSE_OVER;
}
tsMnodeInfos.nodeInfos[i].syncPort = (uint16_t)syncPort->valueint;
cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName");
if (!nodeName || nodeName->type != cJSON_String || nodeName->valuestring == NULL) {
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
dError("failed to read mnode mgmtIpList.json, nodeName not found");
goto PARSE_OVER;
}
strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN);
}
strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN);
}
ret = true;
dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId,
taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), tsMnodeInfos.nodeInfos[i].nodePort,
tsMnodeInfos.nodeInfos[i].nodeName);
dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
}
PARSE_OVER:
@ -352,10 +321,7 @@ static void dnodeSaveMnodeInfos() {
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp));
len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort);
len += snprintf(content + len, maxLen - len, " \"syncPort\": %u,\n", tsMnodeInfos.nodeInfos[i].syncPort);
len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName);
len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeEp);
if (i < tsMnodeInfos.nodeNum -1) {
len += snprintf(content + len, maxLen - len, " },{\n");
} else {
@ -371,8 +337,8 @@ static void dnodeSaveMnodeInfos() {
dPrint("save mnode iplist successed");
}
uint32_t dnodeGetMnodeMasteIp() {
return tsMnodeIpSet.ip[tsMnodeIpSet.inUse];
char *dnodeGetMnodeMasterEp() {
return tsMnodeInfos.nodeInfos[tsMnodeIpSet.inUse].nodeEp;
}
void* dnodeGetMnodeInfos() {
@ -402,8 +368,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
//strcpy(pStatus->dnodeName, tsDnodeName);
pStatus->version = htonl(tsVersion);
pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId);
pStatus->privateIp = htonl(inet_addr(tsPrivateIp));
pStatus->publicIp = htonl(inet_addr(tsPublicIp));
strcpy(pStatus->dnodeEp, tsLocalEp);
pStatus->lastReboot = htonl(tsRebootTime);
pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes);
pStatus->numOfCores = htons((uint16_t) tsNumOfCores);
@ -500,4 +465,4 @@ void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
int32_t dnodeGetDnodeId() {
return tsDnodeCfg.dnodeId;
}
}

25
src/dnode/src/dnodeMain.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taos.h"
#include "tglobal.h"
#include "trpc.h"
#include "tutil.h"
@ -144,12 +145,12 @@ static int32_t dnodeInitSystem() {
signal(SIGPIPE, SIG_IGN);
struct stat dirstat;
if (stat(logDir, &dirstat) < 0) {
mkdir(logDir, 0755);
if (stat(tsLogDir, &dirstat) < 0) {
mkdir(tsLogDir, 0755);
}
char temp[TSDB_FILENAME_LEN];
sprintf(temp, "%s/taosdlog", logDir);
sprintf(temp, "%s/taosdlog", tsLogDir);
if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) {
printf("failed to init log file\n");
}
@ -161,8 +162,7 @@ static int32_t dnodeInitSystem() {
}
taosPrintGlobalCfg();
dPrint("Server IP address is:%s", tsPrivateIp);
dPrint("starting to initialize TDengine ...");
dPrint("start to initialize TDengine on %s", tsLocalEp);
if (dnodeInitStorage() != 0) return -1;
if (dnodeInitRead() != 0) return -1;
@ -191,6 +191,7 @@ static void dnodeCleanUpSystem() {
dnodeCleanupWrite();
dnodeCleanupRead();
dnodeCleanUpModules();
taos_cleanup();
dnodeCleanupStorage();
taosCloseLog();
}
@ -218,13 +219,13 @@ static void dnodeCheckDataDirOpenned(char *dir) {
static int32_t dnodeInitStorage() {
struct stat dirstat;
if (stat(dataDir, &dirstat) < 0) {
mkdir(dataDir, 0755);
if (stat(tsDataDir, &dirstat) < 0) {
mkdir(tsDataDir, 0755);
}
sprintf(tsMnodeDir, "%s/mnode", dataDir);
sprintf(tsVnodeDir, "%s/vnode", dataDir);
sprintf(tsDnodeDir, "%s/dnode", dataDir);
sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
mkdir(tsVnodeDir, 0755);
mkdir(tsDnodeDir, 0755);
@ -237,5 +238,5 @@ static int32_t dnodeInitStorage() {
static void dnodeCleanupStorage() {}
bool dnodeIsFirstDeploy() {
return strcmp(tsMasterIp, tsPrivateIp) == 0;
}
return strcmp(tsFirst, tsLocalEp) == 0;
}

2
src/dnode/src/dnodeMgmt.c
View File

@ -139,11 +139,9 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime);
pCreate->cfg.arbitratorIp = htonl(pCreate->cfg.arbitratorIp);
for (int32_t j = 0; j < pCreate->cfg.replications; ++j) {
pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
pCreate->nodes[j].nodeIp = htonl(pCreate->nodes[j].nodeIp);
}
void *pVnode = vnodeAccquireVnode(pCreate->cfg.vgId);

1
src/dnode/src/dnodeMnode.c
View File

@ -38,7 +38,6 @@ int32_t dnodeInitMnode() {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = tsDnodeMnodePort;
rpcInit.label = "DND-MS";
rpcInit.numOfThreads = 1;

1
src/dnode/src/dnodeShell.c
View File

@ -47,7 +47,6 @@ int32_t dnodeInitShell() {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = tsDnodeShellPort;
rpcInit.label = "DND-shell";
rpcInit.numOfThreads = numOfThreads;

4
src/inc/dnode.h
View File

@ -42,8 +42,8 @@ void dnodeFreeRqueue(void *rqueue);
void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code);
bool dnodeIsFirstDeploy();
uint32_t dnodeGetMnodeMasteIp();
void dnodeGetMnodeIpSet(void *ipSet, bool usePublicIp);
char *dnodeGetMnodeMasterEp();
void dnodeGetMnodeDnodeIpSet(void *ipSet);
void * dnodeGetMnodeInfos();
int32_t dnodeGetDnodeId();

23
src/inc/taosdef.h
View File

@ -32,6 +32,13 @@ extern "C" {
#define TSKEY int64_t
#endif
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
typedef int32_t VarDataOffsetT;
typedef int16_t VarDataLenT;
#define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
#define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT)))
// this data type is internally used only in 'in' query to hold the values
#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1)
@ -121,6 +128,10 @@ typedef struct tDataTypeDescriptor {
int16_t nameLen;
int32_t nSize;
char * aName;
int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize);
int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize);
} tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypeDesc[11];
@ -187,11 +198,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_CITY_LEN 20
#define TSDB_STATE_LEN 20
#define TSDB_COUNTRY_LEN 20
#define TSDB_VNODES_SUPPORT 6
#define TSDB_MGMT_SUPPORT 4
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 64
#define TSDB_FQDN_LEN 72
#define TSDB_IPv4ADDR_LEN 16
#define TSDB_FILENAME_LEN 128
#define TSDB_METER_VNODE_BITS 20
@ -218,8 +228,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_DNODE_ROLE_MGMT 1
#define TSDB_DNODE_ROLE_VNODE 2
#define TSDB_MAX_MPEERS 5
#define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1)
#define TSDB_MAX_REPLICA 5
#define TSDB_TBNAME_COLUMN_INDEX (-1)
#define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta
@ -320,6 +329,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_NORMAL_TABLES 1000
#define TSDB_MAX_CHILD_TABLES 100000
#define TSDB_PORT_MNODESHELL 0
#define TSDB_PORT_DNODESHELL 5
#define TSDB_PORT_DNODEMNODE 10
#define TSDB_PORT_MNODEDNODE 15
#define TSDB_PORT_SYNC 20
typedef enum {
TSDB_PRECISION_MILLI,
TSDB_PRECISION_MICRO,

24
src/inc/taosmsg.h
View File

@ -187,7 +187,7 @@ extern char *taosMsg[];
#pragma pack(push, 1)
typedef struct {
uint32_t ip;
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
} SIpAddr;
@ -265,6 +265,7 @@ typedef struct {
char tableId[TSDB_TABLE_ID_LEN + 1];
char db[TSDB_DB_NAME_LEN + 1];
int8_t igExists;
int8_t getMeta;
int16_t numOfTags;
int16_t numOfColumns;
int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string
@ -549,24 +550,19 @@ typedef struct {
typedef struct {
int32_t nodeId;
uint32_t nodeIp;
uint16_t nodePort;
uint16_t syncPort;
char nodeName[TSDB_NODE_NAME_LEN + 1];
char nodeEp[TSDB_FQDN_LEN];
} SDMMnodeInfo;
typedef struct {
int8_t inUse;
int8_t nodeNum;
SDMMnodeInfo nodeInfos[TSDB_MAX_MPEERS];
SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA];
} SDMMnodeInfos;
typedef struct {
uint32_t version;
int32_t dnodeId;
char dnodeName[TSDB_NODE_NAME_LEN + 1];
uint32_t privateIp;
uint32_t publicIp;
char dnodeEp[TSDB_FQDN_LEN];
uint32_t moduleStatus;
uint32_t lastReboot; // time stamp for last reboot
uint16_t numOfTotalVnodes; // from config file
@ -603,19 +599,17 @@ typedef struct {
int8_t replications;
int8_t wals;
int8_t quorum;
uint32_t arbitratorIp;
int8_t reserved[16];
} SMDVnodeCfg;
typedef struct {
int32_t nodeId;
uint32_t nodeIp;
char nodeName[TSDB_NODE_NAME_LEN + 1];
char nodeEp[TSDB_FQDN_LEN];
} SMDVnodeDesc;
typedef struct {
SMDVnodeCfg cfg;
SMDVnodeDesc nodes[TSDB_MAX_MPEERS];
SMDVnodeDesc nodes[TSDB_MAX_REPLICA];
} SMDCreateVnodeMsg;
typedef struct {
@ -738,7 +732,7 @@ typedef struct SCMShowRsp {
} SCMShowRsp;
typedef struct {
char ip[32];
char ep[TSDB_FQDN_LEN]; // end point, hostname:port
} SCMCreateDnodeMsg, SCMDropDnodeMsg;
typedef struct {
@ -753,7 +747,7 @@ typedef struct {
} SDMConfigVnodeMsg;
typedef struct {
char ip[32];
char ep[TSDB_FQDN_LEN]; // end point, hostname:port
char config[64];
} SMDCfgDnodeMsg, SCMCfgDnodeMsg;

5
src/inc/trpc.h
View File

@ -31,8 +31,8 @@ extern int tsRpcHeadSize;
typedef struct {
int8_t inUse;
int8_t numOfIps;
uint16_t port;
uint32_t ip[TSDB_MAX_MPEERS];
uint16_t port[TSDB_MAX_REPLICA];
char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN];
} SRpcIpSet;
typedef struct {
@ -51,7 +51,6 @@ typedef struct {
} SRpcMsg;
typedef struct {
char *localIp; // local IP used
uint16_t localPort; // local port
char *label; // for debug purpose
int numOfThreads; // number of threads to handle connections

7
src/inc/tsdb.h
View File

@ -104,9 +104,10 @@ void tsdbClearTableCfg(STableCfg *config);
int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId id, int32_t col, int16_t *type, int16_t *bytes, char **val);
int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name);
int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid);
// the TSDB repository info
typedef struct STsdbRepoInfo {

6
src/inc/tsync.h
View File

@ -38,12 +38,11 @@ typedef enum _TAOS_SYNC_STATUS {
typedef struct {
uint32_t nodeId; // node ID assigned by TDengine
uint32_t nodeIp; // node IP address
char name[TSDB_FILENAME_LEN]; // external node name
uint16_t nodePort; // node sync Port
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN
} SNodeInfo;
typedef struct {
uint32_t arbitratorIp; // arbitrator IP address
int8_t quorum; // number of confirms required, >=1
int8_t replica; // number of replications, >=1
SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA];
@ -107,6 +106,7 @@ extern int tsMaxWatchFiles;
extern int tsSyncTimer;
extern int tsMaxFwdInfo;
extern int sDebugFlag;
extern char tsArbitrator[];
extern uint16_t tsSyncPort;
#ifdef __cplusplus

2
src/kit/shell/src/shellEngine.c
View File

@ -40,7 +40,7 @@ History history;
*/
TAOS *shellInit(struct arguments *args) {
printf("\n");
printf(CLIENT_VERSION, osName, taos_get_client_info());
printf(CLIENT_VERSION, tsOsName, taos_get_client_info());
fflush(stdout);
// set options before initializing

1
src/mnode/inc/mgmtDb.h
View File

@ -32,6 +32,7 @@ int32_t mgmtInitDbs();
void mgmtCleanUpDbs();
SDbObj *mgmtGetDb(char *db);
SDbObj *mgmtGetDbByTableId(char *db);
void * mgmtGetNextDb(void *pNode, SDbObj **pDb);
void mgmtIncDbRef(SDbObj *pDb);
void mgmtDecDbRef(SDbObj *pDb);
bool mgmtCheckIsMonitorDB(char *db, char *monitordb);

14
src/mnode/inc/mgmtDef.h
View File

@ -31,13 +31,9 @@ struct SMnodeObj;
typedef struct SDnodeObj {
int32_t dnodeId;
uint32_t privateIp;
uint32_t publicIp;
uint16_t mnodeShellPort;
uint16_t mnodeDnodePort;
uint16_t dnodeShellPort;
uint16_t dnodeMnodePort;
uint16_t syncPort;
uint16_t dnodePort;
char dnodeFqdn[TSDB_FQDN_LEN];
char dnodeEp[TSDB_FQDN_LEN];
int64_t createdTime;
uint32_t lastAccess;
int32_t openVnodes;
@ -47,7 +43,6 @@ typedef struct SDnodeObj {
int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode
int8_t status; // set in balance function
int8_t isMgmt;
char dnodeName[TSDB_NODE_NAME_LEN + 1];
int8_t reserved[15];
int8_t updateEnd[1];
int32_t refCount;
@ -123,7 +118,7 @@ typedef struct SVgObj {
uint32_t vgId;
char dbName[TSDB_DB_NAME_LEN + 1];
int64_t createdTime;
SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT];
SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
int32_t numOfVnodes;
int32_t lbDnodeId;
int32_t lbTime;
@ -242,7 +237,6 @@ typedef struct {
typedef struct {
uint8_t msgType;
int8_t usePublicIp;
int8_t received;
int8_t successed;
int8_t expected;

2
src/mnode/inc/mgmtDnode.h
View File

@ -38,7 +38,7 @@ void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
void mgmtIncDnodeRef(SDnodeObj *pDnode);
void mgmtDecDnodeRef(SDnodeObj *pDnode);
void * mgmtGetDnode(int32_t dnodeId);
void * mgmtGetDnodeByIp(uint32_t ip);
void * mgmtGetDnodeByIp(char *ep);
void mgmtUpdateDnode(SDnodeObj *pDnode);
int32_t mgmtDropDnode(SDnodeObj *pDnode);

7
src/mnode/inc/mgmtMnode.h
View File

@ -33,16 +33,19 @@ void mgmtCleanupMnodes();
int32_t mgmtAddMnode(int32_t dnodeId);
int32_t mgmtDropMnode(int32_t dnodeId);
void mgmtDropMnodeLocal(int32_t dnodeId);
void * mgmtGetMnode(int32_t mnodeId);
int32_t mgmtGetMnodesNum();
void * mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
void mgmtReleaseMnode(struct SMnodeObj *pMnode);
void mgmtIncMnodeRef(struct SMnodeObj *pMnode);
void mgmtDecMnodeRef(struct SMnodeObj *pMnode);
char * mgmtGetMnodeRoleStr();
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet, bool usePublicIp);
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
void mgmtGetMnodeInfos(void *mnodes);
#ifdef __cplusplus
}
#endif

8
src/mnode/inc/mgmtProfile.h
View File

@ -24,14 +24,6 @@ extern "C" {
int32_t mgmtInitProfile();
void mgmtCleanUpProfile();
bool mgmtCheckQhandle(uint64_t qhandle);
void mgmtSaveQhandle(void *qhandle);
void mgmtFreeQhandle(void *qhandle);
void * mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
void * mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
void mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
#ifdef __cplusplus
}
#endif

1
src/mnode/inc/mgmtSdb.h
View File

@ -72,6 +72,7 @@ void sdbCleanUp();
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
bool sdbIsMaster();
bool sdbIsServing();
void sdbUpdateMnodeRoles();
int32_t sdbInsertRow(SSdbOper *pOper);

8
src/mnode/inc/mgmtShell.h
View File

@ -34,6 +34,14 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg);
void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg);
void mgmtSendSimpleResp(void *thandle, int32_t code);
bool mgmtCheckQhandle(uint64_t qhandle);
void *mgmtSaveQhandle(void *qhandle, int32_t size);
void mgmtFreeQhandle(void *qhandle, bool forceRemove);
void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
void *mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
void mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
#ifdef __cplusplus
}
#endif

16
src/mnode/inc/mgmtTable.h
View File

@ -22,13 +22,15 @@ extern "C" {
#include "mgmtDef.h"
int32_t mgmtInitTables();
void mgmtCleanUpTables();
STableObj* mgmtGetTable(char* tableId);
void mgmtIncTableRef(void *pTable);
void mgmtDecTableRef(void *pTable);
void mgmtDropAllChildTables(SDbObj *pDropDb);
void mgmtDropAllSuperTables(SDbObj *pDropDb);
int32_t mgmtInitTables();
void mgmtCleanUpTables();
void * mgmtGetTable(char *tableId);
void mgmtIncTableRef(void *pTable);
void mgmtDecTableRef(void *pTable);
void * mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable);
void * mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable);
void mgmtDropAllChildTables(SDbObj *pDropDb);
void mgmtDropAllSuperTables(SDbObj *pDropDb);
#ifdef __cplusplus
}

2
src/mnode/inc/mgmtUser.h
View File

@ -27,7 +27,7 @@ SUserObj *mgmtGetUser(char *name);
void * mgmtGetNextUser(void *pNode, SUserObj **pUser);
void mgmtIncUserRef(SUserObj *pUser);
void mgmtDecUserRef(SUserObj *pUser);
SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
SUserObj *mgmtGetUserFromConn(void *pConn);
int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
void mgmtDropAllUsers(SAcctObj *pAcct);

5
src/mnode/inc/mgmtVgroup.h
View File

@ -32,7 +32,8 @@ void mgmtCleanUpVgroups();
SVgObj *mgmtGetVgroup(int32_t vgId);
void mgmtIncVgroupRef(SVgObj *pVgroup);
void mgmtDecVgroupRef(SVgObj *pVgroup);
void mgmtDropAllVgroups(SDbObj *pDropDb);
void mgmtDropAllDbVgroups(SDbObj *pDropDb);
void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
void * mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup);
void mgmtUpdateVgroup(SVgObj *pVgroup);
@ -50,7 +51,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle);
void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup);
SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip);
SRpcIpSet mgmtGetIpSetFromIp(char *ep);
#ifdef __cplusplus
}

4
src/mnode/src/mgmtAcct.c
View File

@ -27,8 +27,8 @@
#include "mgmtUser.h"
void * tsAcctSdb = NULL;
int32_t tsAcctUpdateSize;
static void mgmtCreateRootAcct();
static int32_t tsAcctUpdateSize;
static void mgmtCreateRootAcct();
static int32_t mgmtActionAcctDestroy(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;

2
src/mnode/src/mgmtBalance.c
View File

@ -35,7 +35,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
void * pNode = NULL;
SDnodeObj *pDnode = NULL;
SDnodeObj *pSelDnode = NULL;
float vnodeUsage = 1.0;
float vnodeUsage = 1000.0;
while (1) {
pNode = mgmtGetNextDnode(pNode, &pDnode);

1
src/mnode/src/mgmtDClient.c
View File

@ -38,7 +38,6 @@ static void *tsMgmtDClientRpc = NULL;
int32_t mgmtInitDClient() {
SRpcInit rpcInit = {0};
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = 0;
rpcInit.label = "MND-DC";
rpcInit.numOfThreads = 1;

8
src/mnode/src/mgmtDServer.c
View File

@ -43,7 +43,6 @@ static void *tsMgmtDServerQhandle = NULL;
int32_t mgmtInitDServer() {
SRpcInit rpcInit = {0};
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;;
rpcInit.localPort = tsMnodeDnodePort;
rpcInit.label = "MND-DS";
rpcInit.numOfThreads = 1;
@ -105,14 +104,13 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
if (!sdbIsMaster()) {
SRpcConnInfo connInfo;
rpcGetConnInfo(rpcMsg->handle, &connInfo);
bool usePublicIp = false;
SRpcIpSet ipSet = {0};
ipSet.port = tsMnodeDnodePort;
dnodeGetMnodeIpSet(&ipSet, usePublicIp);
dnodeGetMnodeDnodeIpSet(&ipSet);
mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i]));
mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
}
rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
return;

76
src/mnode/src/mgmtDb.c
View File

@ -36,7 +36,7 @@
#include "mgmtUser.h"
#include "mgmtVgroup.h"
void * tsDbSdb = NULL;
static void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
@ -82,7 +82,7 @@ static int32_t mgmtDbActionDelete(SSdbOper *pOper) {
mgmtDropDbFromAcct(pAcct, pDb);
mgmtDropAllChildTables(pDb);
mgmtDropAllSuperTables(pDb);
mgmtDropAllVgroups(pDb);
mgmtDropAllDbVgroups(pDb);
mgmtDecAcctRef(pAcct);
return TSDB_CODE_SUCCESS;
@ -95,6 +95,7 @@ static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
memcpy(pSaved, pDb, pOper->rowSize);
free(pDb);
}
mgmtDecDbRef(pSaved);
return TSDB_CODE_SUCCESS;
}
@ -154,6 +155,10 @@ int32_t mgmtInitDbs() {
return 0;
}
void *mgmtGetNextDb(void *pNode, SDbObj **pDb) {
return sdbFetchRow(tsDbSdb, pNode, (void **)pDb);
}
SDbObj *mgmtGetDb(char *db) {
return (SDbObj *)sdbGetRow(tsDbSdb, db);
}
@ -174,7 +179,7 @@ SDbObj *mgmtGetDbByTableId(char *tableId) {
memset(db, 0, sizeof(db));
strncpy(db, tableId, pos - tableId);
return (SDbObj *)sdbGetRow(tsDbSdb, db);
return mgmtGetDb(db);
}
static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
@ -346,8 +351,27 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) {
return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb));
}
#if 0
void mgmtPrintVgroups(SDbObj *pDb, char *oper) {
mPrint("db:%s, vgroup link from head, oper:%s", pDb->name, oper);
SVgObj *pVgroup = pDb->pHead;
while (pVgroup != NULL) {
mPrint("vgId:%d", pVgroup->vgId);
pVgroup = pVgroup->next;
}
mPrint("db:%s, vgroup link from tail", pDb->name, pDb->numOfVgroups);
pVgroup = pDb->pTail;
while (pVgroup != NULL) {
mPrint("vgId:%d", pVgroup->vgId);
pVgroup = pVgroup->prev;
}
}
#endif
void mgmtAddVgroupIntoDb(SVgObj *pVgroup) {
SDbObj *pDb = pVgroup->pDb;
pVgroup->next = pDb->pHead;
pVgroup->prev = NULL;
@ -397,7 +421,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
pShow->bytes[cols] = TSDB_DB_NAME_LEN;
@ -545,11 +569,11 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
SDbObj *pDb = NULL;
char * pWrite;
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
while (numOfRows < rows) {
pShow->pNode = sdbFetchRow(tsDbSdb, pShow->pNode, (void **) &pDb);
pShow->pNode = mgmtGetNextDb(pShow->pNode, &pDb);
if (pDb == NULL) break;
cols = 0;
@ -674,8 +698,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
.pObj = pDb
};
int32_t code = sdbUpdateRow(&oper);
@ -770,11 +793,17 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications);
newCfg.replications = replications;
}
if (replications > mgmtGetDnodesNum()) {
mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications);
terrno = TSDB_CODE_NO_ENOUGH_DNODES;
}
if (pDb->cfg.replications - replications >= 2) {
mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications);
terrno = TSDB_CODE_INVALID_OPTION;
}
return newCfg;
}
@ -797,8 +826,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDbSdb,
.pObj = pDb,
.rowSize = tsDbUpdateSize
.pObj = pDb
};
int32_t code = sdbUpdateRow(&oper);
@ -833,21 +861,21 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
return;
}
SDbObj *pDb = pMsg->pDb = mgmtGetDb(pAlter->db);
if (pDb == NULL) {
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pAlter->db);
if (pMsg->pDb == NULL) {
mError("db:%s, failed to alter, invalid db", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB);
return;
}
int32_t code = mgmtAlterDb(pDb, pAlter);
int32_t code = mgmtAlterDb(pMsg->pDb, pAlter);
if (code != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to alter, invalid db option", pAlter->db);
mgmtSendSimpleResp(pMsg->thandle, code);
return;
}
mTrace("db:%s, all vgroups is altered", pDb->name);
mTrace("db:%s, all vgroups is altered", pMsg->pDb->name);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
}
@ -878,8 +906,8 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
return;
}
SDbObj *pDb = pMsg->pDb = mgmtGetDb(pDrop->db);
if (pDb == NULL) {
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pDrop->db);
if (pMsg->pDb == NULL) {
if (pDrop->ignoreNotExists) {
mTrace("db:%s, db is not exist, think drop success", pDrop->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@ -891,30 +919,32 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
}
}
if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
mError("db:%s, can't drop monitor database", pDrop->db);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN);
return;
}
int32_t code = mgmtSetDbDropping(pDb);
int32_t code = mgmtSetDbDropping(pMsg->pDb);
if (code != TSDB_CODE_SUCCESS) {
mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code));
mgmtSendSimpleResp(pMsg->thandle, code);
return;
}
SVgObj *pVgroup = pDb->pHead;
#if 0
SVgObj *pVgroup = pMsg->pDb->pHead;
if (pVgroup != NULL) {
mPrint("vgroup:%d, will be dropped", pVgroup->vgId);
mPrint("vgId:%d, will be dropped", pVgroup->vgId);
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
newMsg->ahandle = pVgroup;
newMsg->expected = pVgroup->numOfVnodes;
mgmtDropVgroup(pVgroup, newMsg);
return;
}
#endif
mTrace("db:%s, all vgroups is dropped", pDb->name);
mTrace("db:%s, all vgroups is dropped", pMsg->pDb->name);
mgmtDropDb(pMsg);
}
@ -926,7 +956,7 @@ void mgmtDropAllDbs(SAcctObj *pAcct) {
mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user);
while (1) {
pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb);
pNode = mgmtGetNextDb(pNode, &pDb);
if (pDb == NULL) break;
if (pDb->pAcct == pAcct) {

172
src/mnode/src/mgmtDnode.c
View File

@ -36,13 +36,13 @@
#include "mgmtUser.h"
#include "mgmtVgroup.h"
void *tsDnodeSdb = NULL;
int32_t tsDnodeUpdateSize = 0;
int32_t tsAccessSquence = 0;
static void *tsDnodeSdb = NULL;
static int32_t tsDnodeUpdateSize = 0;
extern void * tsMnodeSdb;
extern void * tsVgroupSdb;
static int32_t mgmtCreateDnode(uint32_t ip);
static int32_t mgmtCreateDnode(char *ep);
static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg);
static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg);
static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
@ -68,50 +68,17 @@ static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
}
pDnode->mnodeShellPort = tsMnodeShellPort;
pDnode->mnodeDnodePort = tsMnodeDnodePort;
pDnode->dnodeShellPort = tsDnodeShellPort;
pDnode->dnodeMnodePort = tsDnodeMnodePort;
pDnode->syncPort = tsSyncPort;
return TSDB_CODE_SUCCESS;
}
static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj;
void * pNode = NULL;
void * pLastNode = NULL;
SVgObj * pVgroup = NULL;
int32_t numOfVgroups = 0;
while (1) {
pLastNode = pNode;
pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
if (pVgroup == NULL) break;
if (pVgroup->vnodeGid[0].dnodeId == pDnode->dnodeId) {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
};
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfVgroups++;
continue;
}
}
SMnodeObj *pMnode = mgmtGetMnode(pDnode->dnodeId);
if (pMnode != NULL) {
SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
sdbDeleteRow(&oper);
mgmtReleaseMnode(pMnode);
}
mgmtDropAllDnodeVgroups(pDnode);
mgmtDropMnodeLocal(pDnode->dnodeId);
balanceNotify();
mTrace("dnode:%d, all vgroups:%d is dropped from sdb", pDnode->dnodeId, numOfVgroups);
mTrace("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
return TSDB_CODE_SUCCESS;
}
@ -122,6 +89,7 @@ static int32_t mgmtDnodeActionUpdate(SSdbOper *pOper) {
memcpy(pSaved, pDnode, pOper->rowSize);
free(pDnode);
}
mgmtDecDnodeRef(pSaved);
return TSDB_CODE_SUCCESS;
}
@ -144,9 +112,8 @@ static int32_t mgmtDnodeActionDecode(SSdbOper *pOper) {
static int32_t mgmtDnodeActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
uint32_t ip = inet_addr(tsPrivateIp);
mgmtCreateDnode(ip);
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
mgmtCreateDnode(tsLocalEp);
SDnodeObj *pDnode = mgmtGetDnodeByIp(tsLocalEp);
mgmtAddMnode(pDnode->dnodeId);
mgmtDecDnodeRef(pDnode);
}
@ -214,14 +181,14 @@ void *mgmtGetDnode(int32_t dnodeId) {
return sdbGetRow(tsDnodeSdb, &dnodeId);
}
void *mgmtGetDnodeByIp(uint32_t ip) {
void *mgmtGetDnodeByIp(char *ep) {
SDnodeObj *pDnode = NULL;
void * pNode = NULL;
while (1) {
pNode = sdbFetchRow(tsDnodeSdb, pNode, (void**)&pDnode);
pNode = mgmtGetNextDnode(pNode, &pDnode);
if (pDnode == NULL) break;
if (ip == pDnode->privateIp) {
if (strcmp(ep, pDnode->dnodeEp) == 0) {
return pDnode;
}
mgmtDecDnodeRef(pDnode);
@ -242,8 +209,7 @@ void mgmtUpdateDnode(SDnodeObj *pDnode) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsDnodeSdb,
.pObj = pDnode,
.rowSize = tsDnodeUpdateSize
.pObj = pDnode
};
sdbUpdateRow(&oper);
@ -253,19 +219,18 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont;
if (pCmCfgDnode->ip[0] == 0) {
strcpy(pCmCfgDnode->ip, tsPrivateIp);
if (pCmCfgDnode->ep[0] == 0) {
strcpy(pCmCfgDnode->ep, tsLocalEp);
} else {
strcpy(pCmCfgDnode->ip, pCmCfgDnode->ip);
strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep);
}
uint32_t dnodeIp = inet_addr(pCmCfgDnode->ip);
if (strcmp(pMsg->pUser->user, "root") != 0) {
rpcRsp.code = TSDB_CODE_NO_RIGHTS;
} else {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(dnodeIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCmCfgDnode->ep);
SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg));
strcpy(pMdCfgDnode->ip, pCmCfgDnode->ip);
strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep);
strcpy(pMdCfgDnode->config, pCmCfgDnode->config);
SRpcMsg rpcMdCfgDnodeMsg = {
.handle = 0,
@ -279,7 +244,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
}
if (rpcRsp.code == TSDB_CODE_SUCCESS) {
mPrint("dnode:%s, is configured by %s", pCmCfgDnode->ip, pMsg->pUser->user);
mPrint("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user);
}
rpcSendResponse(&rpcRsp);
@ -292,8 +257,6 @@ static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
SDMStatusMsg *pStatus = rpcMsg->pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->privateIp = htonl(pStatus->privateIp);
pStatus->publicIp = htonl(pStatus->publicIp);
pStatus->moduleStatus = htonl(pStatus->moduleStatus);
pStatus->lastReboot = htonl(pStatus->lastReboot);
pStatus->numOfCores = htons(pStatus->numOfCores);
@ -308,23 +271,21 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
SDnodeObj *pDnode = NULL;
if (pStatus->dnodeId == 0) {
pDnode = mgmtGetDnodeByIp(pStatus->privateIp);
pDnode = mgmtGetDnodeByIp(pStatus->dnodeEp);
if (pDnode == NULL) {
mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp));
mTrace("dnode %s not created", pStatus->dnodeEp);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
return;
}
} else {
pDnode = mgmtGetDnode(pStatus->dnodeId);
if (pDnode == NULL) {
mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp));
mError("dnode id:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
return;
}
}
pDnode->privateIp = pStatus->privateIp;
pDnode->publicIp = pStatus->publicIp;
pDnode->lastReboot = pStatus->lastReboot;
pDnode->numOfCores = pStatus->numOfCores;
pDnode->diskAvailable = pStatus->diskAvailable;
@ -334,7 +295,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
pDnode->lastAccess = tsAccessSquence;
if (pStatus->dnodeId == 0) {
mTrace("dnode:%d, first access, privateIp:%s, name:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName);
mTrace("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp);
} else {
//mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
}
@ -347,8 +308,8 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId);
if (pVgroup == NULL) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
mPrint("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
} else {
mgmtUpdateVgroupStatus(pVgroup, pDnode, pVload);
@ -391,25 +352,25 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
rpcSendResponse(&rpcRsp);
}
static int32_t mgmtCreateDnode(uint32_t ip) {
static int32_t mgmtCreateDnode(char *ep) {
int32_t grantCode = grantCheck(TSDB_GRANT_DNODE);
if (grantCode != TSDB_CODE_SUCCESS) {
return grantCode;
}
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
SDnodeObj *pDnode = mgmtGetDnodeByIp(ep);
if (pDnode != NULL) {
mError("dnode:%d is alredy exist, ip:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp));
mgmtDecDnodeRef(pDnode);
mError("dnode:%d is alredy exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort);
return TSDB_CODE_DNODE_ALREADY_EXIST;
}
pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
pDnode->privateIp = ip;
pDnode->publicIp = ip;
pDnode->createdTime = taosGetTimestampMs();
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM;
sprintf(pDnode->dnodeName, "n%d", sdbGetId(tsDnodeSdb) + 1);
strcpy(pDnode->dnodeEp, ep);
taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -446,15 +407,16 @@ int32_t mgmtDropDnode(SDnodeObj *pDnode) {
return code;
}
static int32_t mgmtDropDnodeByIp(uint32_t ip) {
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
static int32_t mgmtDropDnodeByIp(char *ep) {
SDnodeObj *pDnode = mgmtGetDnodeByIp(ep);
if (pDnode == NULL) {
mError("dnode:%s, is not exist", taosIpStr(ip));
mError("dnode:%s, is not exist", ep);
return TSDB_CODE_DNODE_NOT_EXIST;
}
if (pDnode->privateIp == dnodeGetMnodeMasteIp()) {
mError("dnode:%d, can't drop dnode which is master", pDnode->dnodeId);
mgmtDecDnodeRef(pDnode);
if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) {
mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
return TSDB_CODE_NO_REMOVE_MASTER;
}
@ -473,13 +435,13 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) {
if (strcmp(pMsg->pUser->user, "root") != 0) {
rpcRsp.code = TSDB_CODE_NO_RIGHTS;
} else {
uint32_t ip = inet_addr(pCreate->ip);
rpcRsp.code = mgmtCreateDnode(ip);
rpcRsp.code = mgmtCreateDnode(pCreate->ep);
if (rpcRsp.code == TSDB_CODE_SUCCESS) {
SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
mLPrint("dnode:%d, ip:%s is created by %s", pDnode->dnodeId, pCreate->ip, pMsg->pUser->user);
SDnodeObj *pDnode = mgmtGetDnodeByIp(pCreate->ep);
mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user);
mgmtDecDnodeRef(pDnode);
} else {
mError("failed to create dnode:%s, reason:%s", pCreate->ip, tstrerror(rpcRsp.code));
mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(rpcRsp.code));
}
}
rpcSendResponse(&rpcRsp);
@ -490,15 +452,15 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SCMDropDnodeMsg *pDrop = pMsg->pCont;
if (strcmp(pMsg->pUser->user, "root") != 0) {
rpcRsp.code = TSDB_CODE_NO_RIGHTS;
} else {
uint32_t ip = inet_addr(pDrop->ip);
rpcRsp.code = mgmtDropDnodeByIp(ip);
rpcRsp.code = mgmtDropDnodeByIp(pDrop->ep);
if (rpcRsp.code == TSDB_CODE_SUCCESS) {
mLPrint("dnode:%s is dropped by %s", pDrop->ip, pMsg->pUser->user);
mLPrint("dnode:%s is dropped by %s", pDrop->ep, pMsg->pUser->user);
} else {
mError("failed to drop dnode:%s, reason:%s", pDrop->ip, tstrerror(rpcRsp.code));
mError("failed to drop dnode:%s, reason:%s", pDrop->ep, tstrerror(rpcRsp.code));
}
}
@ -506,7 +468,7 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) {
}
static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->pAcct->user, "root") != 0) {
@ -523,15 +485,9 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "private ip");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "public ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -581,7 +537,6 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
int32_t cols = 0;
SDnodeObj *pDnode = NULL;
char *pWrite;
char ipstr[32];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode);
@ -593,14 +548,8 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
*(int16_t *)pWrite = pDnode->dnodeId;
cols++;
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
cols++;
tinet_ntoa(ipstr, pDnode->publicIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -636,7 +585,7 @@ static bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) {
@ -652,9 +601,9 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -702,10 +651,8 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
*(int16_t *)pWrite = pDnode->dnodeId;
cols++;
char ipstr[20];
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -748,7 +695,7 @@ static bool mgmtCheckConfigShow(SGlobalCfg *cfg) {
static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) {
@ -835,7 +782,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo
static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->user, "root") != 0) {
@ -865,8 +812,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
SDnodeObj *pDnode = NULL;
if (pShow->payloadLen > 0 ) {
uint32_t ip = ip2uint(pShow->payload);
pDnode = mgmtGetDnodeByIp(ip);
pDnode = mgmtGetDnodeByIp(pShow->payload);
} else {
mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode);
}

10
src/mnode/src/mgmtMain.c
View File

@ -36,7 +36,7 @@
#include "mgmtTable.h"
#include "mgmtShell.h"
void *tsMgmtTmr = NULL;
extern void *tsMgmtTmr;
static bool tsMgmtIsRunning = false;
int32_t mgmtStartSystem() {
@ -51,12 +51,6 @@ int32_t mgmtStartSystem() {
mkdir(tsMnodeDir, 0755);
}
tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
if (tsMgmtTmr == NULL) {
mError("failed to init timer");
return -1;
}
if (mgmtInitAccts() < 0) {
mError("failed to init accts");
return -1;
@ -130,7 +124,7 @@ int32_t mgmtInitSystem() {
struct stat dirstat;
bool fileExist = (stat(tsMnodeDir, &dirstat) == 0);
bool asMaster = (strcmp(tsMasterIp, tsPrivateIp) == 0);
bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0);
if (asMaster || fileExist) {
if (mgmtStartSystem() != 0) {

65
src/mnode/src/mgmtMnode.c
View File

@ -30,7 +30,7 @@
#include "mgmtShell.h"
#include "mgmtUser.h"
void * tsMnodeSdb = NULL;
static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@ -71,7 +71,7 @@ static int32_t mgmtMnodeActionUpdate(SSdbOper *pOper) {
memcpy(pSaved, pMnode, pOper->rowSize);
free(pMnode);
}
mgmtDecMnodeRef(pSaved);
return TSDB_CODE_SUCCESS;
}
@ -97,7 +97,7 @@ static int32_t mgmtMnodeActionRestored() {
mgmtGetNextMnode(NULL, &pMnode);
if (pMnode != NULL) {
pMnode->role = TAOS_SYNC_ROLE_MASTER;
mgmtReleaseMnode(pMnode);
mgmtDecMnodeRef(pMnode);
}
}
return TSDB_CODE_SUCCESS;
@ -148,7 +148,11 @@ void *mgmtGetMnode(int32_t mnodeId) {
return sdbGetRow(tsMnodeSdb, &mnodeId);
}
void mgmtReleaseMnode(SMnodeObj *pMnode) {
void mgmtIncMnodeRef(SMnodeObj *pMnode) {
sdbIncRef(tsMnodeSdb, pMnode);
}
void mgmtDecMnodeRef(SMnodeObj *pMnode) {
sdbDecRef(tsMnodeSdb, pMnode);
}
@ -171,27 +175,23 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
}
}
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet, bool usePublicIp) {
void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
void *pNode = NULL;
while (1) {
SMnodeObj *pMnode = NULL;
pNode = mgmtGetNextMnode(pNode, &pMnode);
if (pMnode == NULL) break;
if (usePublicIp) {
ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->publicIp);
} else {
ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->privateIp);
}
strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
ipSet->inUse = ipSet->numOfIps;
}
ipSet->numOfIps++;
ipSet->port = htons(pMnode->pDnode->mnodeShellPort);
mgmtReleaseMnode(pMnode);
mgmtDecMnodeRef(pMnode);
}
}
@ -207,16 +207,13 @@ void mgmtGetMnodeInfos(void *param) {
if (pMnode == NULL) break;
mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
mnodes->nodeInfos[index].nodeIp = htonl(pMnode->pDnode->privateIp);
mnodes->nodeInfos[index].nodePort = htons(pMnode->pDnode->mnodeDnodePort);
mnodes->nodeInfos[index].syncPort = htons(pMnode->pDnode->syncPort);
strcpy(mnodes->nodeInfos[index].nodeName, pMnode->pDnode->dnodeName);
strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
mnodes->inUse = index;
}
index++;
mgmtReleaseMnode(pMnode);
mgmtDecMnodeRef(pMnode);
}
mnodes->nodeNum = index;
@ -242,8 +239,17 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
return code;
}
void mgmtDropMnodeLocal(int32_t dnodeId) {
SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
if (pMnode != NULL) {
SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
sdbDeleteRow(&oper);
mgmtDecMnodeRef(pMnode);
}
}
int32_t mgmtDropMnode(int32_t dnodeId) {
SMnodeObj *pMnode = sdbGetRow(tsMnodeSdb, &dnodeId);
SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
if (pMnode == NULL) {
return TSDB_CODE_DNODE_NOT_EXIST;
}
@ -265,7 +271,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
sdbUpdateMnodeRoles();
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->pAcct->user, "root") != 0) {
@ -282,15 +288,9 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "private ip");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "public ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -327,7 +327,6 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
int32_t cols = 0;
SMnodeObj *pMnode = NULL;
char *pWrite;
char ipstr[32];
while (numOfRows < rows) {
pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode);
@ -339,14 +338,8 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
*(int16_t *)pWrite = pMnode->mnodeId;
cols++;
tinet_ntoa(ipstr, pMnode->pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
cols++;
tinet_ntoa(ipstr, pMnode->pDnode->publicIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@ -359,7 +352,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
numOfRows++;
mgmtReleaseMnode(pMnode);
mgmtDecMnodeRef(pMnode);
}
pShow->numOfReads += numOfRows;

66
src/mnode/src/mgmtProfile.c
View File

@ -561,17 +561,6 @@ int32_t mgmtKillConnection(char *qidstr, void *pConn) {
return TSDB_CODE_INVALID_CONNECTION;
}
bool mgmtCheckQhandle(uint64_t qhandle) {
return true;
}
void mgmtSaveQhandle(void *qhandle) {
mTrace("qhandle:%p is allocated", qhandle);
}
void mgmtFreeQhandle(void *qhandle) {
mTrace("qhandle:%p is freed", qhandle);
}
int mgmtGetConns(SShowObj *pShow, void *pConn) {
// SAcctObj * pAcct = pConn->pAcct;
@ -686,7 +675,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
rpcSendResponse(&rpcRsp);
@ -710,7 +699,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
rpcSendResponse(&rpcRsp);
@ -734,7 +723,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
if (pUser == NULL) {
rpcRsp.code = TSDB_CODE_INVALID_USER;
rpcSendResponse(&rpcRsp);
@ -771,52 +760,3 @@ int32_t mgmtInitProfile() {
void mgmtCleanUpProfile() {
}
void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
bool usePublicIp = false;
SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle, &usePublicIp);
if (pUser == NULL) {
return NULL;
}
SQueuedMsg *pMsg = calloc(1, sizeof(SQueuedMsg));
pMsg->thandle = rpcMsg->handle;
pMsg->msgType = rpcMsg->msgType;
pMsg->contLen = rpcMsg->contLen;
pMsg->pCont = rpcMsg->pCont;
pMsg->pUser = pUser;
pMsg->usePublicIp = usePublicIp;
return pMsg;
}
void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
if (pMsg != NULL) {
rpcFreeCont(pMsg->pCont);
if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
if (pMsg->pDnode) mgmtDecDnodeRef(pMsg->pDnode);
free(pMsg);
}
}
void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
SQueuedMsg *pDestMsg = calloc(1, sizeof(SQueuedMsg));
pDestMsg->thandle = pSrcMsg->thandle;
pDestMsg->msgType = pSrcMsg->msgType;
pDestMsg->pCont = pSrcMsg->pCont;
pDestMsg->contLen = pSrcMsg->contLen;
pDestMsg->retry = pSrcMsg->retry;
pDestMsg->maxRetry= pSrcMsg->maxRetry;
pDestMsg->pUser = pSrcMsg->pUser;
pDestMsg->usePublicIp = pSrcMsg->usePublicIp;
pSrcMsg->pCont = NULL;
pSrcMsg->pUser = NULL;
return pDestMsg;
}

35
src/mnode/src/mgmtSdb.c
View File

@ -40,7 +40,7 @@ typedef enum {
typedef enum {
SDB_STATUS_OFFLINE,
SDB_STATUS_SERVING,
SDB_ACTION_CLOSING
SDB_STATUS_CLOSING
} ESdbStatus;
typedef struct _SSdbTable {
@ -107,6 +107,10 @@ bool sdbIsMaster() {
return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER;
}
bool sdbIsServing() {
return tsSdbObj.status == SDB_STATUS_SERVING;
}
static char *sdbGetActionStr(int32_t action) {
switch (action) {
case SDB_ACTION_INSERT:
@ -180,7 +184,7 @@ void sdbUpdateMnodeRoles() {
if (pMnode != NULL) {
pMnode->role = roles.role[i];
sdbPrint("mnode:%d, role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
mgmtReleaseMnode(pMnode);
mgmtDecMnodeRef(pMnode);
}
}
}
@ -231,8 +235,8 @@ void sdbUpdateSync() {
for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
SDMMnodeInfo *node = &mnodes->nodeInfos[i];
syncCfg.nodeInfo[i].nodeId = node->nodeId;
syncCfg.nodeInfo[i].nodeIp = node->nodeIp;
strcpy(syncCfg.nodeInfo[i].name, node->nodeName);
taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[i].nodeFqdn, &syncCfg.nodeInfo[i].nodePort);
syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
index++;
}
@ -244,21 +248,16 @@ void sdbUpdateSync() {
if (pMnode == NULL) break;
syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
syncCfg.nodeInfo[index].nodeIp = pMnode->pDnode->privateIp;
strcpy(syncCfg.nodeInfo[index].name, pMnode->pDnode->dnodeName);
syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC;
strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
index++;
mgmtReleaseMnode(pMnode);
mgmtDecMnodeRef(pMnode);
}
}
syncCfg.replica = index;
syncCfg.arbitratorIp = syncCfg.nodeInfo[0].nodeIp;
if (syncCfg.replica == 1) {
syncCfg.quorum = 1;
} else {
syncCfg.quorum = 2;
}
syncCfg.quorum = (syncCfg.replica == 1) ? 1:2;
bool hasThisDnode = false;
for (int32_t i = 0; i < syncCfg.replica; ++i) {
@ -271,10 +270,9 @@ void sdbUpdateSync() {
if (!hasThisDnode) return;
if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
sdbPrint("work as mnode, replica:%d arbitratorIp:%s", syncCfg.replica, taosIpStr(syncCfg.arbitratorIp));
sdbPrint("work as mnode, replica:%d", syncCfg.replica);
for (int32_t i = 0; i < syncCfg.replica; ++i) {
sdbPrint("mnode:%d, ip:%s name:%s", syncCfg.nodeInfo[i].nodeId, taosIpStr(syncCfg.nodeInfo[i].nodeIp),
syncCfg.nodeInfo[i].name);
sdbPrint("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort);
}
SSyncInfo syncInfo;
@ -320,6 +318,7 @@ int32_t sdbInit() {
void sdbCleanUp() {
if (tsSdbObj.status != SDB_STATUS_SERVING) return;
tsSdbObj.status = SDB_STATUS_CLOSING;
syncStop(tsSdbObj.sync);
free(tsSdbObj.sync);
walClose(tsSdbObj.wal);
@ -333,7 +332,7 @@ void sdbIncRef(void *handle, void *pRow) {
SSdbTable *pTable = handle;
int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos);
atomic_add_fetch_32(pRefCount, 1);
if (0 && strcmp(pTable->tableName, "accounts") == 0) {
if (0 && pTable->tableId == SDB_TABLE_CTABLE) {
sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
*pRefCount);
}
@ -345,7 +344,7 @@ void sdbDecRef(void *handle, void *pRow) {
SSdbTable *pTable = handle;
int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos);
int32_t refCount = atomic_sub_fetch_32(pRefCount, 1);
if (0 && strcmp(pTable->tableName, "accounts") == 0) {
if (0 && pTable->tableId == SDB_TABLE_CTABLE) {
sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
*pRefCount);
}

151
src/mnode/src/mgmtShell.c
View File

@ -23,6 +23,7 @@
#include "ttimer.h"
#include "tgrant.h"
#include "tglobal.h"
#include "tcache.h"
#include "dnode.h"
#include "mgmtDef.h"
#include "mgmtLog.h"
@ -50,10 +51,11 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
extern void *tsMgmtTmr;
void *tsMgmtTmr;
static void *tsMgmtShellRpc = NULL;
static void *tsMgmtTranQhandle = NULL;
static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SQueuedMsg *) = {0};
static void *tsQhandleCache = NULL;
static SShowMetaFp tsMgmtShowMetaFp[TSDB_MGMT_TABLE_MAX] = {0};
static SShowRetrieveFp tsMgmtShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
@ -64,7 +66,9 @@ int32_t mgmtInitShell() {
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mgmtProcessUseMsg);
tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
tsQhandleCache = taosCacheInit(tsMgmtTmr, 2);
int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0;
if (numOfThreads < 1) {
@ -72,7 +76,6 @@ int32_t mgmtInitShell() {
}
SRpcInit rpcInit = {0};
rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;
rpcInit.localPort = tsMnodeShellPort;
rpcInit.label = "MND-shell";
rpcInit.numOfThreads = numOfThreads;
@ -103,6 +106,12 @@ void mgmtCleanUpShell() {
tsMgmtShellRpc = NULL;
mPrint("server connection to shell is closed");
}
if (tsQhandleCache) {
taosCacheEmpty(tsQhandleCache);
taosCacheCleanup(tsQhandleCache);
tsQhandleCache = NULL;
}
}
void mgmtAddShellMsgHandle(uint8_t showType, void (*fp)(SQueuedMsg *queuedMsg)) {
@ -148,14 +157,12 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
if (!sdbIsMaster()) {
SRpcConnInfo connInfo;
rpcGetConnInfo(rpcMsg->handle, &connInfo);
bool usePublicIp = (connInfo.serverIp == tsPublicIpInt);
SRpcIpSet ipSet = {0};
ipSet.port = tsMnodeShellPort;
dnodeGetMnodeIpSet(&ipSet, usePublicIp);
mgmtGetMnodeIpSet(&ipSet);
mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i]));
mTrace("index:%d ip:%s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
}
rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
@ -236,14 +243,15 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
return;
}
SShowObj *pShow = (SShowObj *) calloc(1, sizeof(SShowObj) + htons(pShowMsg->payloadLen));
int32_t showObjSize = sizeof(SShowObj) + htons(pShowMsg->payloadLen);
SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
pShow->signature = pShow;
pShow->type = pShowMsg->type;
pShow->payloadLen = htons(pShowMsg->payloadLen);
strcpy(pShow->db, pShowMsg->db);
memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
mgmtSaveQhandle(pShow);
pShow = mgmtSaveQhandle(pShow, showObjSize);
pShowRsp->qhandle = htobe64((uint64_t) pShow);
mTrace("show:%p, type:%s, start to get meta", pShow, mgmtGetShowTypeStr(pShowMsg->type));
@ -258,10 +266,10 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
rpcSendResponse(&rpcRsp);
} else {
mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mgmtGetShowTypeStr(pShowMsg->type), tstrerror(code));
mgmtFreeQhandle(pShow);
mgmtFreeQhandle(pShow, false);
SRpcMsg rpcRsp = {
.handle = pMsg->thandle,
.code = code
.handle = pMsg->thandle,
.code = code
};
rpcSendResponse(&rpcRsp);
}
@ -287,26 +295,20 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
mTrace("show:%p, type:%s, retrieve data", pShow, mgmtGetShowTypeStr(pShow->type));
if (!mgmtCheckQhandle(pRetrieve->qhandle)) {
mError("pShow:%p, query memory is corrupted", pShow);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MEMORY_CORRUPTED);
return;
} else {
if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsToRead = pShow->numOfRows - pShow->numOfReads;
}
/* return no more than 100 meters in one round trip */
if (rowsToRead > 100) rowsToRead = 100;
/*
* the actual number of table may be larger than the value of pShow->numOfRows, if a query is
* issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
*/
if (rowsToRead < 0) rowsToRead = 0;
size = pShow->rowSize * rowsToRead;
if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsToRead = pShow->numOfRows - pShow->numOfReads;
}
/* return no more than 100 meters in one round trip */
if (rowsToRead > 100) rowsToRead = 100;
/*
* the actual number of table may be larger than the value of pShow->numOfRows, if a query is
* issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
*/
if (rowsToRead < 0) rowsToRead = 0;
size = pShow->rowSize * rowsToRead;
size += 100;
SRetrieveTableRsp *pRsp = rpcMallocCont(size);
@ -316,6 +318,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
if (rowsRead < 0) { // TSDB_CODE_ACTION_IN_PROGRESS;
rpcFreeCont(pRsp);
mgmtFreeQhandle(pShow, false);
return;
}
@ -332,7 +335,9 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
rpcSendResponse(&rpcRsp);
if (rowsToRead == 0) {
mgmtFreeQhandle(pShow);
mgmtFreeQhandle(pShow, true);
} else {
mgmtFreeQhandle(pShow, false);
}
}
@ -343,7 +348,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
return;
}
mgmtGetMnodeIpSet(&pHBRsp->ipList, pMsg->usePublicIp);
mgmtGetMnodeIpSet(&pHBRsp->ipList);
/*
* TODO
@ -416,6 +421,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
code = TSDB_CODE_INVALID_DB;
goto connect_over;
}
mgmtDecDbRef(pDb);
}
SCMConnectRsp *pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp));
@ -429,7 +435,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
pConnectRsp->writeAuth = pUser->writeAuth;
pConnectRsp->superAuth = pUser->superAuth;
mgmtGetMnodeIpSet(&pConnectRsp->ipList, pMsg->usePublicIp);
mgmtGetMnodeIpSet(&pConnectRsp->ipList);
connect_over:
rpcRsp.code = code;
@ -449,9 +455,8 @@ static void mgmtProcessUseMsg(SQueuedMsg *pMsg) {
SCMUseDbMsg *pUseDbMsg = pMsg->pCont;
// todo check for priority of current user
pMsg->pDb = mgmtGetDb(pUseDbMsg->db);
int32_t code = TSDB_CODE_SUCCESS;
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pUseDbMsg->db);
if (pMsg->pDb == NULL) {
code = TSDB_CODE_INVALID_DB;
}
@ -465,7 +470,7 @@ static void mgmtProcessUseMsg(SQueuedMsg *pMsg) {
*/
static bool mgmtCheckTableMetaMsgReadOnly(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
pMsg->pTable = mgmtGetTable(pInfo->tableId);
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId);
if (pMsg->pTable != NULL) return true;
// If table does not exists and autoCreate flag is set, we add the handler into task queue
@ -514,3 +519,79 @@ void mgmtSendSimpleResp(void *thandle, int32_t code) {
};
rpcSendResponse(&rpcRsp);
}
bool mgmtCheckQhandle(uint64_t qhandle) {
void *pSaved = taosCacheAcquireByData(tsQhandleCache, (void *)qhandle);
if (pSaved == (void *)qhandle) {
mTrace("qhandle:%p is retrived", qhandle);
return true;
} else {
mTrace("qhandle:%p is already freed", qhandle);
return false;
}
}
void* mgmtSaveQhandle(void *qhandle, int32_t size) {
if (tsQhandleCache != NULL) {
char key[24];
sprintf(key, "show:%p", qhandle);
void *newQhandle = taosCachePut(tsQhandleCache, key, qhandle, size, 60);
free(qhandle);
mTrace("qhandle:%p is saved", newQhandle);
return newQhandle;
}
return NULL;
}
void mgmtFreeQhandle(void *qhandle, bool forceRemove) {
mTrace("qhandle:%p is freed", qhandle);
taosCacheRelease(tsQhandleCache, &qhandle, forceRemove);
}
void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
if (pUser == NULL) {
return NULL;
}
SQueuedMsg *pMsg = calloc(1, sizeof(SQueuedMsg));
pMsg->thandle = rpcMsg->handle;
pMsg->msgType = rpcMsg->msgType;
pMsg->contLen = rpcMsg->contLen;
pMsg->pCont = rpcMsg->pCont;
pMsg->pUser = pUser;
return pMsg;
}
void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
if (pMsg != NULL) {
rpcFreeCont(pMsg->pCont);
if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
if (pMsg->pDnode) mgmtDecDnodeRef(pMsg->pDnode);
free(pMsg);
}
}
void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
SQueuedMsg *pDestMsg = calloc(1, sizeof(SQueuedMsg));
pDestMsg->thandle = pSrcMsg->thandle;
pDestMsg->msgType = pSrcMsg->msgType;
pDestMsg->pCont = pSrcMsg->pCont;
pDestMsg->contLen = pSrcMsg->contLen;
pDestMsg->retry = pSrcMsg->retry;
pDestMsg->maxRetry= pSrcMsg->maxRetry;
pDestMsg->pUser = pSrcMsg->pUser;
pSrcMsg->pCont = NULL;
pSrcMsg->pUser = NULL;
return pDestMsg;
}

277
src/mnode/src/mgmtTable.c
View File

@ -41,8 +41,8 @@
#include "mgmtVgroup.h"
#include "tcompare.h"
void * tsChildTableSdb;
void * tsSuperTableSdb;
static void * tsChildTableSdb;
static void * tsSuperTableSdb;
static int32_t tsChildTableUpdateSize;
static int32_t tsSuperTableUpdateSize;
static void * mgmtGetChildTable(char *tableId);
@ -74,6 +74,7 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg);
static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg);
static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg);
static void mgmtGetChildTableMeta(SQueuedMsg *pMsg);
static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg);
static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg);
static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg);
@ -96,14 +97,14 @@ static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) {
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId);
mError("ctable:%s, not in vgId:%d", pTable->info.tableId, pTable->vgId);
return TSDB_CODE_INVALID_VGROUP_ID;
}
mgmtDecVgroupRef(pVgroup);
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
return TSDB_CODE_INVALID_DB;
}
mgmtDecDbRef(pDb);
@ -116,6 +117,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) {
mgmtDecAcctRef(pAcct);
if (pTable->info.type == TSDB_CHILD_TABLE) {
// add ref
pTable->superTable = mgmtGetSuperTable(pTable->superTableId);
mgmtAddTableIntoStable(pTable->superTable, pTable);
grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
@ -145,7 +147,7 @@ static int32_t mgmtChildTableActionDelete(SSdbOper *pOper) {
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
mError("ctable:%s, vgId:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
return TSDB_CODE_INVALID_DB;
}
mgmtDecDbRef(pDb);
@ -185,6 +187,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
free(oldSql);
free(oldSchema);
}
mgmtDecTableRef(pTable);
return TSDB_CODE_SUCCESS;
}
@ -249,7 +252,7 @@ static int32_t mgmtChildTableActionRestored() {
while (1) {
pLastNode = pNode;
mgmtDecTableRef(pTable);
pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable);
pNode = mgmtGetNextChildTable(pNode, &pTable);
if (pTable == NULL) break;
SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId);
@ -267,7 +270,7 @@ static int32_t mgmtChildTableActionRestored() {
SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
mError("ctable:%s, failed to get vgId:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
pTable->vgId = 0;
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
@ -280,7 +283,7 @@ static int32_t mgmtChildTableActionRestored() {
mgmtDecVgroupRef(pVgroup);
if (strcmp(pVgroup->dbName, pDb->name) != 0) {
mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it",
pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
pTable->vgId = 0;
SSdbOper desc = {0};
@ -293,7 +296,7 @@ static int32_t mgmtChildTableActionRestored() {
}
if (pVgroup->tableList == NULL) {
mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId);
mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId);
pTable->vgId = 0;
SSdbOper desc = {0};
desc.type = SDB_OPER_LOCAL;
@ -360,7 +363,7 @@ static void mgmtCleanUpChildTables() {
static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
if (pStable->vgLen == 0) {
pStable->vgLen = 10;
pStable->vgLen = 8;
pStable->vgList = calloc(pStable->vgLen, sizeof(int32_t));
}
@ -430,11 +433,11 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
void *oldSchema = pTable->schema;
memcpy(pTable, pNew, pOper->rowSize);
pTable->schema = pNew->schema;
free(pNew);
free(pNew->vgList);
free(pNew);
free(oldSchema);
}
mgmtDecTableRef(pTable);
return TSDB_CODE_SUCCESS;
}
@ -557,20 +560,28 @@ static void *mgmtGetSuperTable(char *tableId) {
return sdbGetRow(tsSuperTableSdb, tableId);
}
STableObj *mgmtGetTable(char *tableId) {
STableObj *tableInfo = sdbGetRow(tsSuperTableSdb, tableId);
if (tableInfo != NULL) {
return tableInfo;
void *mgmtGetTable(char *tableId) {
void *pTable = mgmtGetSuperTable(tableId);
if (pTable != NULL) {
return pTable;
}
tableInfo = sdbGetRow(tsChildTableSdb, tableId);
if (tableInfo != NULL) {
return tableInfo;
pTable = mgmtGetChildTable(tableId);
if (pTable != NULL) {
return pTable;
}
return NULL;
}
void *mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable) {
return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable);
}
void *mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable) {
return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable);
}
void mgmtIncTableRef(void *p1) {
STableObj *pTable = (STableObj *)p1;
if (pTable->type == TSDB_SUPER_TABLE) {
@ -612,9 +623,19 @@ static void mgmtExtractTableName(char* tableId, char* name) {
static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->pCont;
pMsg->pTable = mgmtGetTable(pCreate->tableId);
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pCreate->db);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to create, db not selected", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pCreate->tableId);
if (pMsg->pTable != NULL && pMsg->retry == 0) {
if (pCreate->igExists) {
if (pCreate->getMeta) {
mTrace("table:%s, continue to get meta", pCreate->tableId);
mgmtGetChildTableMeta(pMsg);
} else if (pCreate->igExists) {
mTrace("table:%s, is already exist", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
} else {
@ -624,13 +645,6 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
return;
}
pMsg->pDb = mgmtGetDb(pCreate->db);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to create, db not selected", pCreate->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
if (pCreate->numOfTags != 0) {
mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->thandle);
mgmtProcessCreateSuperTableMsg(pMsg);
@ -642,7 +656,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
SCMDropTableMsg *pDrop = pMsg->pCont;
pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId);
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to drop table, db not selected", pDrop->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
@ -655,8 +669,8 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
return;
}
pMsg->pTable = mgmtGetTable(pDrop->tableId);
if (pMsg->pTable == NULL) {
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pDrop->tableId);
if (pMsg->pTable == NULL) {
if (pDrop->igNotExists) {
mTrace("table:%s, table is not exist, think drop success", pDrop->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@ -679,17 +693,24 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle);
pInfo->createFlag = htons(pInfo->createFlag);
mTrace("table:%s, table meta msg is received from thandle:%p, createFlag:%d", pInfo->tableId, pMsg->thandle, pInfo->createFlag);
pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to get table meta, db not selected", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
return;
}
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId);
if (pMsg->pTable == NULL) {
mgmtGetChildTableMeta(pMsg);
if (!pInfo->createFlag) {
mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
} else {
mgmtAutoCreateChildTable(pMsg);
}
} else {
if (pMsg->pTable->type != TSDB_SUPER_TABLE) {
mgmtGetChildTableMeta(pMsg);
@ -776,8 +797,6 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
mgmtDecVgroupRef(pVgroup);
}
}
//mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
//mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
} else {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -835,8 +854,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
.pObj = pStable
};
int32_t code = sdbUpdateRow(&oper);
@ -863,8 +881,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
.pObj = pStable
};
int32_t code = sdbUpdateRow(&oper);
@ -900,8 +917,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
.pObj = pStable
};
int32_t code = sdbUpdateRow(&oper);
@ -966,8 +982,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
.pObj = pStable
};
int32_t code = sdbUpdateRow(&oper);
@ -1004,8 +1019,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsSuperTableSdb,
.pObj = pStable,
.rowSize = tsSuperTableUpdateSize
.pObj = pStable
};
int32_t code = sdbUpdateRow(&oper);
@ -1088,7 +1102,8 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
char stableName[TSDB_TABLE_NAME_LEN] = {0};
while (numOfRows < rows) {
pShow->pNode = sdbFetchRow(tsSuperTableSdb, pShow->pNode, (void **) &pTable);
mgmtDecTableRef(pTable);
pShow->pNode = mgmtGetNextSuperTable(pShow->pNode, &pTable);
if (pTable == NULL) break;
if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
continue;
@ -1124,8 +1139,6 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
cols++;
numOfRows++;
mgmtDecTableRef(pTable);
}
pShow->numOfReads += numOfRows;
@ -1144,7 +1157,8 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name);
while (1) {
pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **)&pTable);
pLastNode = pNode;
pNode = mgmtGetNextSuperTable(pNode, &pTable);
if (pTable == NULL) break;
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@ -1202,14 +1216,13 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
SCMSTableVgroupMsg *pInfo = pMsg->pCont;
SSuperTableObj *pTable = mgmtGetSuperTable(pInfo->tableId);
pMsg->pTable = (STableObj *)pTable;
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetSuperTable(pInfo->tableId);
if (pMsg->pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
}
SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * pTable->vgLen;
SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
@ -1230,8 +1243,8 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
if (pDnode == NULL) break;
pRsp->vgroups[vg].ipAddr[vn].ip = htonl(pDnode->privateIp);
pRsp->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort);
strcpy(pRsp->vgroups[vg].ipAddr[vn].fqdn, pDnode->dnodeFqdn);
pRsp->vgroups[vg].ipAddr[vn].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL);
pRsp->vgroups[vg].numOfIps++;
}
@ -1419,17 +1432,21 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
}
int32_t sid = taosAllocateId(pVgroup->idPool);
if (sid < 0) {
mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId);
if (sid <= 0) {
mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId);
mgmtCreateVgroup(mgmtCloneQueuedMsg(pMsg), pMsg->pDb);
return;
}
if (pMsg->retry == 0) {
pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
if (pMsg->pTable == NULL) {
pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
mgmtIncTableRef(pMsg->pTable);
}
} else {
pMsg->pTable = mgmtGetTable(pCreate->tableId);
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pCreate->tableId);
}
if (pMsg->pTable == NULL) {
mgmtSendSimpleResp(pMsg->thandle, terrno);
return;
@ -1445,7 +1462,6 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
newMsg->ahandle = pMsg->pTable;
newMsg->maxRetry = 5;
mgmtIncTableRef(pMsg->pTable);
SRpcMsg rpcMsg = {
.handle = newMsg,
.pCont = pMDCreate,
@ -1459,8 +1475,8 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
SVgObj *pVgroup = pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) {
mError("table:%s, failed to drop ctable, vgroup not exist", pTable->info.tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
return;
@ -1479,7 +1495,7 @@ static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
pDrop->sid = htonl(pTable->sid);
pDrop->uid = htobe64(pTable->uid);
SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pMsg->pVgroup);
mTrace("table:%s, send drop ctable msg", pDrop->tableId);
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
@ -1545,8 +1561,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
.rowSize = tsChildTableUpdateSize
.pObj = pTable
};
int32_t code = sdbUpdateRow(&oper);
@ -1578,8 +1593,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsChildTableSdb,
.pObj = pTable,
.rowSize = tsChildTableUpdateSize
.pObj = pTable
};
int32_t code = sdbUpdateRow(&oper);
@ -1607,7 +1621,6 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pT
static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
SDbObj *pDb = pMsg->pDb;
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
int8_t usePublicIp = pMsg->usePublicIp;
pMeta->uid = htobe64(pTable->uid);
pMeta->sid = htonl(pTable->sid);
@ -1628,68 +1641,57 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable);
}
SVgObj *pVgroup = pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
if (pMsg->pVgroup == NULL) pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
if (pMsg->pVgroup == NULL) {
mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId);
return TSDB_CODE_INVALID_VGROUP_ID;
}
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
for (int32_t i = 0; i < pMsg->pVgroup->numOfVnodes; ++i) {
SDnodeObj *pDnode = mgmtGetDnode(pMsg->pVgroup->vnodeGid[i].dnodeId);
if (pDnode == NULL) break;
if (usePublicIp) {
pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->publicIp);
pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort);
} else {
pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->privateIp);
pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort);
}
strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn);
pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL);
pMeta->vgroup.numOfIps++;
mgmtDecDnodeRef(pDnode);
}
pMeta->vgroup.vgId = htonl(pVgroup->vgId);
pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId);
mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
return TSDB_CODE_SUCCESS;
}
void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
SCMTableInfoMsg *pInfo = pMsg->pCont;
if (pTable == NULL) {
if (htons(pInfo->createFlag) != 1) {
mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
return;
} else {
//TODO: on demand create table from super table if table does not exists
int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId));
SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
memcpy(newMsg, pMsg, sizeof(SQueuedMsg));
pMsg->pCont = NULL;
newMsg->ahandle = newMsg->pCont;
newMsg->pCont = pCreateMsg;
mTrace("table:%s, start to create in demand", pInfo->tableId);
mgmtAddToShellQueue(newMsg);
return;
}
int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData);
SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
if (pCreateMsg == NULL) {
mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId));
strcpy(pCreateMsg->db, pMsg->pDb->name);
pCreateMsg->igExists = 1;
pCreateMsg->getMeta = 1;
pCreateMsg->contLen = htonl(contLen);
memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
pMsg->pCont = newMsg->pCont;
newMsg->msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
newMsg->pCont = pCreateMsg;
mTrace("table:%s, start to create on demand", pInfo->tableId);
mgmtAddToShellQueue(newMsg);
}
static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) {
STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS);
if (pMeta == NULL) {
mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId);
mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
return;
}
@ -1715,7 +1717,8 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name);
while (1) {
pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable);
pLastNode = pNode;
pNode = mgmtGetNextChildTable(pNode, &pTable);
if (pTable == NULL) break;
if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@ -1743,7 +1746,8 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables);
while (1) {
pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable);
pLastNode = pNode;
pNode = mgmtGetNextChildTable(pNode, &pTable);
if (pTable == NULL) break;
if (pTable->superTable == pStable) {
@ -1763,16 +1767,13 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
}
static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) {
SDnodeObj *pObj = mgmtGetDnode(dnodeId);
static SChildTableObj* mgmtGetTableByPos(int32_t vnode, int32_t sid) {
SVgObj *pVgroup = mgmtGetVgroup(vnode);
if (pVgroup == NULL) return NULL;
if (pObj == NULL || pVgroup == NULL) {
return NULL;
}
SChildTableObj *pTable = pVgroup->tableList[sid];
SChildTableObj *pTable = pVgroup->tableList[sid - 1];
mgmtIncTableRef((STableObj *)pTable);
mgmtDecVgroupRef(pVgroup);
return pTable;
}
@ -1784,7 +1785,7 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
pCfg->sid = htonl(pCfg->sid);
mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
SChildTableObj *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid);
SChildTableObj *pTable = mgmtGetTableByPos(pCfg->vnode, pCfg->sid);
if (pTable == NULL) {
mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE);
@ -1800,7 +1801,8 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
return;
}
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode);
SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnode);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
SRpcMsg rpcRsp = {
.handle = NULL,
.pCont = pMDCreate,
@ -1809,7 +1811,9 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
.msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
};
mgmtSendMsgToDnode(&ipSet, &rpcRsp);
mgmtDecTableRef(pTable);
mgmtDecDnodeRef(pDnode);
}
// handle drop child response
@ -1830,8 +1834,8 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
return;
}
SVgObj *pVgroup = queueMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
if (pVgroup == NULL) {
if (queueMsg->pVgroup == NULL) queueMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
if (queueMsg->pVgroup == NULL) {
mError("table:%s, failed to get vgroup", pTable->info.tableId);
mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID);
return;
@ -1850,9 +1854,9 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
return;
}
if (pVgroup->numOfTables <= 0) {
mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId);
mgmtDropVgroup(pVgroup, NULL);
if (queueMsg->pVgroup->numOfTables <= 0) {
mPrint("vgId:%d, all tables is dropped, drop vgroup", queueMsg->pVgroup->vgId);
mgmtDropVgroup(queueMsg->pVgroup, NULL);
}
mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS);
@ -1893,9 +1897,10 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
} else {
mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle,
tstrerror(rpcMsg->code));
if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) {
mTrace("table:%s, start to get meta", pTable->info.tableId);
SCMCreateTableMsg *pCreate = queueMsg->pCont;
if (pCreate->getMeta) {
mTrace("table:%s, continue to get meta", pTable->info.tableId);
queueMsg->retry = 0;
mgmtAddToShellQueue(queueMsg);
} else {
mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
@ -1928,8 +1933,8 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
SChildTableObj *pTable = mgmtGetChildTable(tableId);
if (pTable == NULL) continue;
SDbObj *pDb = mgmtGetDbByTableId(tableId);
if (pDb == NULL) continue;
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(tableId);
if (pMsg->pDb == NULL) continue;
int availLen = totalMallocLen - pMultiMeta->contLen;
if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) {
@ -2028,7 +2033,8 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
int32_t prefixLen = strlen(prefix);
while (numOfRows < rows) {
pShow->pNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable);
mgmtDecTableRef(pTable);
pShow->pNode = mgmtGetNextChildTable(pShow->pNode, &pTable);
if (pTable == NULL) break;
// not belong to current db
@ -2072,7 +2078,6 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
cols++;
numOfRows++;
mgmtDecTableRef(pTable);
}
pShow->numOfReads += numOfRows;
@ -2088,7 +2093,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
SCMAlterTableMsg *pAlter = pMsg->pCont;
mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle);
pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId);
if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("table:%s, failed to alter table, db not selected", pAlter->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
@ -2101,7 +2106,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
return;
}
pMsg->pTable = mgmtGetTable(pAlter->tableId);
if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pAlter->tableId);
if (pMsg->pTable == NULL) {
mError("table:%s, failed to alter table, table not exist", pMsg->pTable->tableId);
mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);

14
src/mnode/src/mgmtUser.c
View File

@ -168,8 +168,7 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsUserSdb,
.pObj = pUser,
.rowSize = tsUserUpdateSize
.pObj = pUser
};
int32_t code = sdbUpdateRow(&oper);
@ -249,7 +248,7 @@ static int32_t mgmtDropUser(SUserObj *pUser) {
}
static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
SUserObj *pUser = mgmtGetUserFromConn(pConn);
if (pUser == NULL) {
return TSDB_CODE_NO_USER_FROM_CONN;
}
@ -298,7 +297,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
char *pWrite;
while (numOfRows < rows) {
pShow->pNode = sdbFetchRow(tsUserSdb, pShow->pNode, (void **) &pUser);
pShow->pNode = mgmtGetNextUser(pShow->pNode, &pUser);
if (pUser == NULL) break;
cols = 0;
@ -329,12 +328,9 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
return numOfRows;
}
SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) {
SUserObj *mgmtGetUserFromConn(void *pConn) {
SRpcConnInfo connInfo;
if (rpcGetConnInfo(pConn, &connInfo) == 0) {
if (usePublicIp) {
*usePublicIp = (connInfo.serverIp == tsPublicIpInt);
}
return mgmtGetUser(connInfo.user);
} else {
mError("can not get user from conn:%p", pConn);
@ -510,7 +506,7 @@ void mgmtDropAllUsers(SAcctObj *pAcct) {
while (1) {
pLastNode = pNode;
pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser);
pNode = mgmtGetNextUser(pNode, &pUser);
if (pUser == NULL) break;
if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {

153
src/mnode/src/mgmtVgroup.c
View File

@ -36,8 +36,8 @@
#include "mgmtTable.h"
#include "mgmtVgroup.h"
void *tsVgroupSdb = NULL;
int32_t tsVgUpdateSize = 0;
static void *tsVgroupSdb = NULL;
static int32_t tsVgUpdateSize = 0;
static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@ -62,6 +62,8 @@ static int32_t mgmtVgroupActionDestroy(SSdbOper *pOper) {
static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
// refer to db
SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
if (pDb == NULL) {
return TSDB_CODE_INVALID_DB;
@ -74,13 +76,13 @@ static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *));
if (pVgroup->tableList == NULL) {
mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
mError("vgId:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
return -1;
}
pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables);
if (pVgroup->idPool == NULL) {
mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
mError("vgId:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
tfree(pVgroup->tableList);
return -1;
}
@ -101,7 +103,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
static int32_t mgmtVgroupActionDelete(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
if (pVgroup->pDb != NULL) {
mgmtRemoveVgroupFromDb(pVgroup);
}
@ -140,6 +142,7 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
if (pDnode != NULL) {
atomic_add_fetch_32(&pDnode->openVnodes, 1);
}
mgmtDecDnodeRef(pDnode);
}
}
@ -147,14 +150,15 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
SDbObj *pDb = pVgroup->pDb;
if (pDb != NULL) {
if (pDb->cfg.maxTables != oldTables) {
mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
}
}
mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes);
mgmtDecVgroupRef(pVgroup);
mTrace("vgId:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes);
return TSDB_CODE_SUCCESS;
}
@ -162,7 +166,7 @@ static int32_t mgmtVgroupActionEncode(SSdbOper *pOper) {
SVgObj *pVgroup = pOper->pObj;
memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
SVgObj *pTmpVgroup = pOper->rowData;
for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) {
for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
pTmpVgroup->vnodeGid[i].pDnode = NULL;
pTmpVgroup->vnodeGid[i].role = 0;
}
@ -237,8 +241,7 @@ void mgmtUpdateVgroup(SVgObj *pVgroup) {
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
.rowSize = tsVgUpdateSize
.pObj = pVgroup
};
sdbUpdateRow(&oper);
@ -260,8 +263,8 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
}
if (!dnodeExist) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
mError("vgroup:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
return;
}
@ -273,7 +276,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
}
if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) {
mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion,
pVgroup->numOfVnodes);
mgmtSendCreateVgroupMsg(pVgroup, NULL);
@ -317,9 +320,9 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
return;
}
mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
mPrint("vgroup:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
}
pMsg->ahandle = pVgroup;
@ -331,7 +334,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
if (ahandle != NULL) {
mgmtSendDropVgroupMsg(pVgroup, ahandle);
} else {
mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mTrace("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
mgmtSendDropVgroupMsg(pVgroup, NULL);
SSdbOper oper = {
.type = SDB_OPER_GLOBAL,
@ -379,6 +382,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
if (pShow->payloadLen > 0 ) {
pTable = mgmtGetTable(pShow->payload);
if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
mgmtDecTableRef(pTable);
return TSDB_CODE_INVALID_TABLE_ID;
}
mgmtDecTableRef(pTable);
@ -401,9 +405,9 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 16;
pShow->bytes[cols] = 40;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip");
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
@ -440,7 +444,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
SVgObj *pVgroup = NULL;
int32_t maxReplica = 0;
int32_t cols = 0;
char ipstr[20];
char * pWrite;
SDbObj *pDb = mgmtGetDb(pShow->db);
@ -479,10 +482,10 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
SDnodeObj *pDnode = pVgroup->vnodeGid[i].pDnode;
if (pDnode != NULL) {
tinet_ntoa(ipstr, pDnode->privateIp);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, ipstr);
strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
strcpy(pWrite, mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role));
cols++;
@ -506,25 +509,28 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
}
void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) {
pVgroup->tableList[pTable->sid] = pTable;
if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] == NULL) {
pVgroup->tableList[pTable->sid - 1] = pTable;
taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid);
pVgroup->numOfTables++;
}
if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables)
mgmtAddVgroupIntoDbTail(pVgroup);
if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) {
mgmtMoveVgroupToTail(pVgroup);
}
mgmtIncVgroupRef(pVgroup);
}
void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) {
pVgroup->tableList[pTable->sid] = NULL;
if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] != NULL) {
pVgroup->tableList[pTable->sid - 1] = NULL;
taosFreeId(pVgroup->idPool, pTable->sid);
pVgroup->numOfTables--;
}
if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables)
mgmtAddVgroupIntoDbTail(pVgroup);
mgmtMoveVgroupToHead(pVgroup);
mgmtDecVgroupRef(pVgroup);
}
SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
@ -559,11 +565,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode;
if (pDnode != NULL) {
pNodes[j].nodeId = htonl(pDnode->dnodeId);
pNodes[j].nodeIp = htonl(pDnode->privateIp);
strcpy(pNodes[j].nodeName, pDnode->dnodeName);
if (j == 0) {
pCfg->arbitratorIp = htonl(pDnode->privateIp);
}
strcpy(pNodes[j].nodeEp, pDnode->dnodeEp);
}
}
@ -574,26 +576,26 @@ SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) {
SRpcIpSet ipSet = {
.numOfIps = pVgroup->numOfVnodes,
.inUse = 0,
.port = tsDnodeMnodePort
};
for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
ipSet.ip[i] = pVgroup->vnodeGid[i].pDnode->privateIp;
strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn);
ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEMNODE;
}
return ipSet;
}
SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip) {
SRpcIpSet ipSet = {
.ip[0] = ip,
.numOfIps = 1,
.inUse = 0,
.port = tsDnodeMnodePort
};
SRpcIpSet mgmtGetIpSetFromIp(char *ep) {
SRpcIpSet ipSet;
ipSet.numOfIps = 1;
ipSet.inUse = 0;
taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]);
ipSet.port[0] += TSDB_PORT_DNODEMNODE;
return ipSet;
}
void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
mTrace("vgroup:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle);
mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle);
SMDCreateVnodeMsg *pCreate = mgmtBuildCreateVnodeMsg(pVgroup);
SRpcMsg rpcMsg = {
.handle = ahandle,
@ -606,9 +608,9 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
}
void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
mTrace("vgId:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mgmtSendCreateVnodeMsg(pVgroup, &ipSet, ahandle);
}
}
@ -624,7 +626,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
}
SVgObj *pVgroup = queueMsg->ahandle;
mTrace("vgroup:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
mTrace("vgId:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
pVgroup->vgId, tstrerror(rpcMsg->code), queueMsg->received, queueMsg->successed, queueMsg->expected,
queueMsg->thandle, rpcMsg->handle);
@ -659,7 +661,7 @@ static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) {
}
void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
mTrace("vgroup:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
mTrace("vgId:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(vgId);
SRpcMsg rpcMsg = {
.handle = ahandle,
@ -672,15 +674,15 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
}
static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
mTrace("vgroup:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
mTrace("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle);
}
}
static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
mTrace("drop vnode rsp is received");
mTrace("drop vnode rsp is received, handle:%p", rpcMsg->handle);
if (rpcMsg->handle == NULL) return;
SQueuedMsg *queueMsg = rpcMsg->handle;
@ -691,7 +693,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
}
SVgObj *pVgroup = queueMsg->ahandle;
mTrace("vgroup:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
mTrace("vgId:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
pVgroup->vgId, tstrerror(rpcMsg->code), queueMsg->received, queueMsg->successed, queueMsg->expected,
queueMsg->thandle, rpcMsg->handle);
@ -737,24 +739,22 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
}
void mgmtDropAllVgroups(SDbObj *pDropDb) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfVgroups = 0;
int32_t dbNameLen = strlen(pDropDb->name);
void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
void * pNode = NULL;
void * pLastNode = NULL;
SVgObj *pVgroup = NULL;
mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
int32_t numOfVgroups = 0;
while (1) {
pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
pLastNode = pNode;
pNode = mgmtGetNextVgroup(pNode, &pVgroup);
if (pVgroup == NULL) break;
if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) {
if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
@ -763,9 +763,36 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfVgroups++;
continue;
}
mgmtDecVgroupRef(pVgroup);
}
}
void mgmtDropAllDbVgroups(SDbObj *pDropDb) {
void *pNode = NULL;
void *pLastNode = NULL;
int32_t numOfVgroups = 0;
SVgObj *pVgroup = NULL;
mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
while (1) {
pLastNode = pNode;
pNode = mgmtGetNextVgroup(pNode, &pVgroup);
if (pVgroup == NULL) break;
if (pVgroup->pDb == pDropDb) {
SSdbOper oper = {
.type = SDB_OPER_LOCAL,
.table = tsVgroupSdb,
.pObj = pVgroup,
};
sdbDeleteRow(&oper);
pNode = pLastNode;
numOfVgroups++;
mgmtSendDropVgroupMsg(pVgroup, NULL);
}
mgmtSendDropVgroupMsg(pVgroup, NULL);
mgmtDecVgroupRef(pVgroup);
}

8
src/os/darwin/src/darwinEnv.c
View File

@ -23,8 +23,8 @@ void osInit() {
strcpy(tsVnodeDir, "");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(dataDir, "/var/lib/taos");
strcpy(logDir, "~/TDengineLog");
strcpy(scriptDir, "/etc/taos");
strcpy(osName, "Darwin");
strcpy(tsDataDir, "/var/lib/taos");
strcpy(tsLogDir, "~/TDengineLog");
strcpy(tsScriptDir, "/etc/taos");
strcpy(tsOsName, "Darwin");
}

63
src/os/darwin/src/darwinPlatform.c
View File

@ -151,67 +151,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle
return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
}
int taosOpenUDClientSocket(char *ip, uint16_t port) {
int sockFd = 0;
struct sockaddr_un serverAddr;
int ret;
char name[128];
sprintf(name, "%s.%hu", ip, port);
sockFd = socket(AF_UNIX, SOCK_STREAM, 0);
if (sockFd < 0) {
uError("failed to open the UD socket:%s, reason:%s", name, strerror(errno));
return -1;
}
memset((char *)&serverAddr, 0, sizeof(serverAddr));
serverAddr.sun_family = AF_UNIX;
strcpy(serverAddr.sun_path + 1, name);
ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
if (ret != 0) {
uError("failed to connect UD socket, name:%d, reason: %s", name, strerror(errno));
sockFd = -1;
}
return sockFd;
}
int taosOpenUDServerSocket(char *ip, uint16_t port) {
struct sockaddr_un serverAdd;
int sockFd;
char name[128];
uTrace("open ud socket:%s", name);
sprintf(name, "%s.%hu", ip, port);
bzero((char *)&serverAdd, sizeof(serverAdd));
serverAdd.sun_family = AF_UNIX;
strcpy(serverAdd.sun_path + 1, name);
unlink(name);
if ((sockFd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
uError("failed to open UD socket:%s, reason:%s", name, strerror(errno));
return -1;
}
/* bind socket to server address */
if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
uError("bind socket:%s failed, reason:%s", name, strerror(errno));
tclose(sockFd);
return -1;
}
if (listen(sockFd, 10) < 0) {
uError("listen socket:%s failed, reason:%s", name, strerror(errno));
return -1;
}
return sockFd;
}
int taosInitTimer(void (*callback)(int), int ms) {
signal(SIGALRM, callback);
@ -443,4 +382,4 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
return writeLen;
}
void taosSetCoreDump() {}
void taosSetCoreDump() {}

8
src/os/linux/src/linuxEnv.c
View File

@ -23,8 +23,8 @@ void osInit() {
strcpy(tsVnodeDir, "");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(dataDir, "/var/lib/taos");
strcpy(logDir, "/var/log/taos");
strcpy(scriptDir, "/etc/taos");
strcpy(osName, "Linux");
strcpy(tsDataDir, "/var/lib/taos");
strcpy(tsLogDir, "/var/log/taos");
strcpy(tsScriptDir, "/etc/taos");
strcpy(tsOsName, "Linux");
}

62
src/os/linux/src/linuxPlatform.c
View File

@ -143,68 +143,6 @@ int taosSetNonblocking(int sock, int on) {
int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
}
int taosOpenUDClientSocket(char *ip, uint16_t port) {
int sockFd = 0;
struct sockaddr_un serverAddr;
int ret;
char name[128];
sprintf(name, "%s.%hu", ip, port);
sockFd = socket(AF_UNIX, SOCK_STREAM, 0);
if (sockFd < 0) {
uError("failed to open the UD socket:%s, reason:%s", name, strerror(errno));
return -1;
}
memset((char *)&serverAddr, 0, sizeof(serverAddr));
serverAddr.sun_family = AF_UNIX;
strcpy(serverAddr.sun_path + 1, name);
ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
if (ret != 0) {
uError("failed to connect UD socket, name:%d, reason: %s", name, strerror(errno));
sockFd = -1;
}
return sockFd;
}
int taosOpenUDServerSocket(char *ip, uint16_t port) {
struct sockaddr_un serverAdd;
int sockFd;
char name[128];
uTrace("open ud socket:%s", name);
sprintf(name, "%s.%hu", ip, port);
bzero((char *)&serverAdd, sizeof(serverAdd));
serverAdd.sun_family = AF_UNIX;
strcpy(serverAdd.sun_path + 1, name);
unlink(name);
if ((sockFd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
uError("failed to open UD socket:%s, reason:%s", name, strerror(errno));
return -1;
}
/* bind socket to server address */
if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
uError("bind socket:%s failed, reason:%s", name, strerror(errno));
tclose(sockFd);
return -1;
}
if (listen(sockFd, 10) < 0) {
uError("listen socket:%s failed, reason:%s", name, strerror(errno));
return -1;
}
return sockFd;
}
static void taosDeleteTimer(void *tharg) {
timer_t *pTimer = tharg;
timer_delete(*pTimer);

23
src/os/linux/src/linuxSysPara.c
View File

@ -290,11 +290,12 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
bool taosGetDisk() {
struct statvfs info;
const double unit = 1024 * 1024 * 1024;
if (tscEmbedded) {
if (statvfs(dataDir, &info)) {
tsTotalDataDirGB = 0;
tsAvailDataDirGB = 0;
if (statvfs(tsDataDir, &info)) {
//tsTotalDataDirGB = 0;
//tsAvailDataDirGB = 0;
uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
return false;
} else {
tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
@ -302,9 +303,10 @@ bool taosGetDisk() {
}
}
if (statvfs(logDir, &info)) {
tsTotalLogDirGB = 0;
tsAvailLogDirGB = 0;
if (statvfs(tsLogDir, &info)) {
//tsTotalLogDirGB = 0;
//tsAvailLogDirGB = 0;
uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
return false;
} else {
tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
@ -312,8 +314,9 @@ bool taosGetDisk() {
}
if (statvfs("/tmp", &info)) {
tsTotalTmpDirGB = 0;
tsAvailTmpDirGB = 0;
//tsTotalTmpDirGB = 0;
//tsAvailTmpDirGB = 0;
uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
return false;
} else {
tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
@ -361,6 +364,8 @@ static bool taosGetCardName(char *ip, char *name) {
static bool taosGetCardInfo(int64_t *bytes) {
static char tsPublicCard[1000] = {0};
static char tsPrivateIp[40];
if (tsPublicCard[0] == 0) {
if (!taosGetCardName(tsPrivateIp, tsPublicCard)) {
uError("can't get card name from ip:%s", tsPrivateIp);

8
src/os/windows/src/twinenv.c
View File

@ -23,8 +23,8 @@ void osInit() {
strcpy(tsVnodeDir, "C:/TDengine/data");
strcpy(tsDnodeDir, "");
strcpy(tsMnodeDir, "");
strcpy(dataDir, "C:/TDengine/data");
strcpy(logDir, "C:/TDengine/log");
strcpy(scriptDir, "C:/TDengine/script");
strcpy(osName, "Windows");
strcpy(tsDataDir, "C:/TDengine/data");
strcpy(tsLogDir, "C:/TDengine/log");
strcpy(tsScriptDir, "C:/TDengine/script");
strcpy(tsOsName, "Windows");
}

2
src/plugins/http/inc/httpHandle.h
View File

@ -210,7 +210,7 @@ typedef struct HttpThread {
typedef struct HttpServer {
char label[HTTP_LABEL_SIZE];
char serverIp[16];
uint32_t serverIp;
uint16_t serverPort;
int cacheContext;
int sessionExpire;

4
src/plugins/http/src/httpSystem.c
View File

@ -48,7 +48,7 @@ int httpInitSystem() {
memset(httpServer, 0, sizeof(HttpServer));
strcpy(httpServer->label, "rest");
strcpy(httpServer->serverIp, tsHttpIp);
httpServer->serverIp = 0;
httpServer->serverPort = tsHttpPort;
httpServer->cacheContext = tsHttpCacheSessions;
httpServer->sessionExpire = tsHttpSessionExpire;
@ -117,7 +117,7 @@ void httpCleanUpSystem() {
httpPrint("http service cleanup");
httpStopSystem();
#if 1
#if 0
if (httpServer == NULL) {
return;
}

20
src/plugins/monitor/src/monitorMain.c
View File

@ -68,7 +68,7 @@ typedef enum {
typedef struct {
void * conn;
void * timer;
char privateIpStr[TSDB_IPv4ADDR_LEN];
char ep[TSDB_FQDN_LEN];
int8_t cmdIndex;
int8_t state;
char sql[SQL_LENGTH];
@ -112,14 +112,8 @@ static void monitorInitConn(void *para, void *unused) {
monitorPrint("starting to initialize monitor service ..");
tsMonitorConn.state = MONITOR_STATE_INITIALIZING;
if (tsMonitorConn.privateIpStr[0] == 0) {
strcpy(tsMonitorConn.privateIpStr, tsPrivateIp);
for (int32_t i = 0; i < TSDB_IPv4ADDR_LEN; ++i) {
if (tsMonitorConn.privateIpStr[i] == '.') {
tsMonitorConn.privateIpStr[i] = '_';
}
}
}
if (tsMonitorConn.ep[0] == 0)
strcpy(tsMonitorConn.ep, tsLocalEp);
if (tsMonitorConn.conn == NULL) {
taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, &tsMonitorConn, &(tsMonitorConn.conn));
@ -163,7 +157,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) {
tsMonitorDbName, IP_LEN_STR + 1);
} else if (cmd == MONITOR_CMD_CREATE_TB_DN) {
snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName,
tsMonitorConn.privateIpStr, tsMonitorDbName, tsPrivateIp);
tsMonitorConn.ep, tsMonitorDbName, tsLocalEp);
} else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) {
snprintf(sql, SQL_LENGTH,
"create table if not exists %s.acct(ts timestamp "
@ -214,7 +208,7 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) {
if (-code == TSDB_CODE_TABLE_ALREADY_EXIST || -code == TSDB_CODE_DB_ALREADY_EXIST || code >= 0) {
monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql);
if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) {
monitorPrint("dnode:%s is started", tsPrivateIp);
monitorPrint("dnode:%s is started", tsLocalEp);
}
tsMonitorConn.cmdIndex++;
monitorInitDatabase();
@ -346,7 +340,7 @@ static void monitorSaveSystemInfo() {
int64_t ts = taosGetTimestampUs();
char * sql = tsMonitorConn.sql;
int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.privateIpStr, ts);
int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.ep, ts);
pos += monitorBuildCpuSql(sql + pos);
pos += monitorBuildMemorySql(sql + pos);
@ -414,7 +408,7 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
va_end(argpointer);
if (len > max_length) len = max_length;
len += sprintf(sql + len, "', '%s')", tsPrivateIp);
len += sprintf(sql + len, "', '%s')", tsLocalEp);
sql[len++] = 0;
monitorTrace("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql);

213
src/query/src/sql.c
View File

@ -27,7 +27,6 @@
#include <assert.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "qsqlparser.h"
@ -203,61 +202,62 @@ typedef union {
** yy_default[] Default action for each state.
**
*********** Begin parsing tables **********************************************/
#define YY_ACTTAB_COUNT (529)
#define YY_ACTTAB_COUNT (531)
static const YYACTIONTYPE yy_action[] = {
/* 0 */ 752, 440, 132, 150, 244, 10, 616, 246, 132, 441,
/* 10 */ 132, 155, 821, 41, 43, 20, 35, 36, 820, 154,
/* 20 */ 821, 29, 741, 440, 200, 39, 37, 40, 38, 131,
/* 30 */ 499, 441, 96, 34, 33, 100, 151, 32, 31, 30,
/* 40 */ 41, 43, 741, 35, 36, 152, 136, 163, 29, 727,
/* 50 */ 749, 200, 39, 37, 40, 38, 185, 100, 225, 224,
/* 60 */ 34, 33, 162, 730, 32, 31, 30, 400, 401, 402,
/* 0 */ 752, 440, 133, 151, 244, 10, 616, 246, 133, 441,
/* 10 */ 133, 156, 821, 41, 43, 20, 35, 36, 820, 155,
/* 20 */ 821, 29, 741, 440, 201, 39, 37, 40, 38, 132,
/* 30 */ 499, 441, 97, 34, 33, 101, 152, 32, 31, 30,
/* 40 */ 41, 43, 741, 35, 36, 153, 137, 164, 29, 727,
/* 50 */ 749, 201, 39, 37, 40, 38, 186, 101, 225, 224,
/* 60 */ 34, 33, 163, 730, 32, 31, 30, 400, 401, 402,
/* 70 */ 403, 404, 405, 406, 407, 408, 409, 410, 411, 245,
/* 80 */ 730, 41, 43, 188, 35, 36, 215, 236, 197, 29,
/* 90 */ 58, 20, 200, 39, 37, 40, 38, 32, 31, 30,
/* 100 */ 56, 34, 33, 75, 730, 32, 31, 30, 43, 236,
/* 110 */ 35, 36, 776, 817, 195, 29, 20, 20, 200, 39,
/* 120 */ 37, 40, 38, 164, 570, 727, 227, 34, 33, 440,
/* 130 */ 167, 32, 31, 30, 238, 35, 36, 441, 7, 816,
/* 140 */ 29, 61, 110, 200, 39, 37, 40, 38, 223, 228,
/* 80 */ 730, 41, 43, 189, 35, 36, 216, 236, 198, 29,
/* 90 */ 58, 20, 201, 39, 37, 40, 38, 32, 31, 30,
/* 100 */ 56, 34, 33, 76, 730, 32, 31, 30, 43, 236,
/* 110 */ 35, 36, 776, 817, 196, 29, 20, 20, 201, 39,
/* 120 */ 37, 40, 38, 165, 570, 727, 227, 34, 33, 440,
/* 130 */ 168, 32, 31, 30, 238, 35, 36, 441, 7, 816,
/* 140 */ 29, 61, 111, 201, 39, 37, 40, 38, 223, 228,
/* 150 */ 727, 727, 34, 33, 50, 728, 32, 31, 30, 15,
/* 160 */ 214, 237, 213, 212, 211, 210, 209, 208, 207, 206,
/* 160 */ 215, 237, 214, 213, 212, 211, 210, 209, 208, 207,
/* 170 */ 712, 51, 701, 702, 703, 704, 705, 706, 707, 708,
/* 180 */ 709, 710, 711, 159, 583, 11, 815, 574, 100, 577,
/* 190 */ 100, 580, 168, 159, 583, 222, 221, 574, 16, 577,
/* 200 */ 20, 580, 34, 33, 145, 26, 32, 31, 30, 238,
/* 210 */ 86, 85, 139, 174, 657, 156, 157, 123, 144, 199,
/* 220 */ 182, 715, 179, 714, 148, 156, 157, 159, 583, 531,
/* 230 */ 60, 574, 149, 577, 726, 580, 237, 16, 39, 37,
/* 180 */ 709, 710, 711, 160, 583, 11, 815, 574, 101, 577,
/* 190 */ 101, 580, 169, 160, 583, 222, 221, 574, 16, 577,
/* 200 */ 20, 580, 34, 33, 146, 26, 32, 31, 30, 238,
/* 210 */ 87, 86, 140, 175, 657, 157, 158, 124, 145, 200,
/* 220 */ 183, 715, 180, 714, 149, 157, 158, 160, 583, 531,
/* 230 */ 60, 574, 150, 577, 726, 580, 237, 16, 39, 37,
/* 240 */ 40, 38, 27, 775, 26, 59, 34, 33, 551, 552,
/* 250 */ 32, 31, 30, 137, 113, 114, 219, 64, 67, 156,
/* 260 */ 157, 95, 515, 666, 184, 512, 123, 513, 26, 514,
/* 270 */ 523, 147, 127, 125, 240, 88, 87, 187, 42, 158,
/* 280 */ 73, 77, 239, 84, 76, 572, 528, 729, 42, 582,
/* 290 */ 79, 17, 658, 165, 166, 123, 243, 242, 92, 582,
/* 250 */ 32, 31, 30, 138, 114, 115, 68, 64, 67, 157,
/* 260 */ 158, 96, 515, 666, 185, 512, 124, 513, 26, 514,
/* 270 */ 523, 148, 128, 126, 240, 89, 88, 188, 42, 159,
/* 280 */ 74, 78, 239, 85, 77, 572, 528, 729, 42, 582,
/* 290 */ 80, 17, 658, 166, 167, 124, 243, 242, 93, 582,
/* 300 */ 47, 542, 543, 600, 581, 45, 13, 12, 584, 576,
/* 310 */ 138, 579, 12, 575, 581, 578, 2, 72, 71, 48,
/* 320 */ 505, 573, 42, 743, 45, 504, 204, 9, 8, 21,
/* 330 */ 21, 140, 519, 582, 520, 517, 141, 518, 83, 82,
/* 340 */ 142, 143, 134, 130, 135, 830, 133, 786, 581, 785,
/* 350 */ 160, 782, 781, 161, 751, 721, 768, 226, 97, 767,
/* 360 */ 111, 112, 516, 668, 205, 109, 128, 24, 218, 220,
/* 370 */ 829, 69, 26, 828, 826, 115, 186, 686, 25, 22,
/* 380 */ 90, 129, 655, 78, 653, 80, 651, 650, 169, 538,
/* 390 */ 124, 648, 189, 647, 646, 644, 636, 193, 52, 740,
/* 400 */ 126, 642, 640, 638, 49, 755, 756, 101, 769, 44,
/* 410 */ 198, 196, 194, 28, 192, 190, 217, 74, 229, 230,
/* 420 */ 202, 232, 231, 614, 233, 234, 53, 235, 241, 170,
/* 430 */ 146, 62, 171, 65, 173, 172, 613, 176, 175, 178,
/* 440 */ 649, 177, 612, 89, 91, 117, 687, 118, 116, 119,
/* 450 */ 120, 643, 104, 102, 122, 725, 106, 103, 105, 121,
/* 460 */ 107, 1, 108, 23, 180, 181, 605, 183, 187, 525,
/* 470 */ 55, 539, 153, 98, 57, 191, 18, 63, 4, 544,
/* 480 */ 99, 5, 585, 3, 19, 14, 201, 6, 203, 480,
/* 490 */ 479, 478, 477, 476, 475, 474, 473, 471, 45, 444,
/* 500 */ 66, 446, 21, 501, 216, 68, 500, 498, 54, 465,
/* 510 */ 46, 463, 455, 70, 461, 457, 459, 453, 451, 472,
/* 520 */ 470, 81, 426, 442, 93, 415, 94, 413, 618,
/* 310 */ 139, 579, 12, 575, 581, 578, 2, 73, 72, 48,
/* 320 */ 505, 573, 42, 743, 45, 504, 205, 9, 8, 21,
/* 330 */ 21, 141, 519, 582, 520, 517, 142, 518, 84, 83,
/* 340 */ 143, 144, 135, 131, 136, 830, 134, 786, 581, 785,
/* 350 */ 161, 782, 781, 162, 751, 721, 768, 226, 98, 767,
/* 360 */ 112, 113, 516, 668, 206, 110, 129, 24, 219, 665,
/* 370 */ 220, 829, 26, 70, 828, 826, 187, 116, 686, 25,
/* 380 */ 91, 22, 130, 655, 79, 653, 81, 651, 650, 538,
/* 390 */ 170, 125, 190, 648, 647, 646, 644, 194, 52, 740,
/* 400 */ 636, 127, 642, 640, 638, 49, 755, 102, 756, 44,
/* 410 */ 769, 199, 197, 195, 193, 191, 28, 218, 75, 229,
/* 420 */ 230, 231, 232, 233, 234, 235, 203, 53, 241, 614,
/* 430 */ 171, 172, 147, 62, 65, 174, 613, 177, 173, 179,
/* 440 */ 612, 176, 649, 178, 181, 643, 123, 687, 117, 119,
/* 450 */ 118, 120, 121, 90, 103, 725, 108, 104, 105, 122,
/* 460 */ 106, 107, 109, 92, 1, 23, 182, 188, 605, 184,
/* 470 */ 525, 55, 539, 57, 99, 154, 192, 18, 63, 4,
/* 480 */ 544, 100, 480, 585, 3, 19, 5, 14, 202, 6,
/* 490 */ 204, 479, 478, 477, 476, 475, 474, 473, 471, 45,
/* 500 */ 217, 444, 66, 21, 501, 500, 46, 498, 54, 465,
/* 510 */ 463, 455, 461, 457, 69, 459, 71, 453, 451, 472,
/* 520 */ 470, 82, 426, 442, 94, 415, 413, 618, 617, 617,
/* 530 */ 95,
};
static const YYCODETYPE yy_lookahead[] = {
/* 0 */ 207, 1, 256, 206, 207, 256, 204, 205, 256, 9,
@ -300,20 +300,20 @@ static const YYCODETYPE yy_lookahead[] = {
/* 370 */ 207, 207, 103, 207, 207, 207, 240, 207, 207, 207,
/* 380 */ 59, 207, 207, 207, 207, 207, 207, 207, 207, 107,
/* 390 */ 207, 207, 259, 207, 207, 207, 207, 259, 117, 253,
/* 400 */ 207, 207, 207, 207, 119, 208, 208, 252, 208, 116,
/* 410 */ 111, 115, 110, 121, 109, 108, 75, 84, 83, 49,
/* 420 */ 208, 82, 80, 5, 53, 81, 208, 79, 75, 132,
/* 430 */ 208, 212, 5, 212, 58, 132, 5, 5, 132, 58,
/* 440 */ 208, 132, 5, 209, 209, 220, 222, 216, 221, 219,
/* 450 */ 217, 208, 249, 251, 215, 240, 247, 250, 248, 218,
/* 460 */ 246, 213, 245, 210, 132, 58, 86, 124, 104, 97,
/* 470 */ 105, 97, 1, 96, 101, 96, 101, 72, 112, 97,
/* 480 */ 96, 112, 97, 96, 101, 96, 98, 96, 98, 9,
/* 490 */ 5, 5, 5, 5, 1, 5, 5, 5, 101, 76,
/* 500 */ 72, 58, 101, 5, 15, 127, 5, 97, 96, 5,
/* 510 */ 16, 5, 5, 127, 5, 5, 5, 5, 5, 5,
/* 520 */ 5, 58, 58, 76, 21, 59, 21, 58, 0, 267,
/* 530 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 400 */ 207, 207, 207, 207, 207, 119, 208, 252, 208, 116,
/* 410 */ 208, 111, 115, 110, 109, 108, 121, 75, 84, 83,
/* 420 */ 49, 80, 82, 53, 81, 79, 208, 208, 75, 5,
/* 430 */ 132, 5, 208, 212, 212, 58, 5, 5, 132, 58,
/* 440 */ 5, 132, 208, 132, 132, 208, 215, 222, 221, 216,
/* 450 */ 220, 219, 217, 209, 251, 240, 246, 250, 249, 218,
/* 460 */ 248, 247, 245, 209, 213, 210, 58, 104, 86, 124,
/* 470 */ 97, 105, 97, 101, 96, 1, 96, 101, 72, 112,
/* 480 */ 97, 96, 9, 97, 96, 101, 112, 96, 98, 96,
/* 490 */ 98, 5, 5, 5, 5, 1, 5, 5, 5, 101,
/* 500 */ 15, 76, 72, 101, 5, 5, 16, 97, 96, 5,
/* 510 */ 5, 5, 5, 5, 127, 5, 127, 5, 5, 5,
/* 520 */ 5, 58, 58, 76, 21, 59, 58, 0, 267, 267,
/* 530 */ 21, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 540 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 550 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 560 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
@ -333,41 +333,41 @@ static const YYCODETYPE yy_lookahead[] = {
/* 700 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 710 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 720 */ 267, 267, 267, 267, 267, 267, 267, 267, 267, 267,
/* 730 */ 267, 267,
/* 730 */ 267, 267, 267, 267,
};
#define YY_SHIFT_COUNT (246)
#define YY_SHIFT_MIN (0)
#define YY_SHIFT_MAX (528)
#define YY_SHIFT_MAX (527)
static const unsigned short int yy_shift_ofst[] = {
/* 0 */ 141, 74, 182, 226, 128, 128, 128, 128, 128, 128,
/* 10 */ 0, 22, 226, 260, 260, 260, 102, 128, 128, 128,
/* 20 */ 128, 128, 31, 149, 9, 9, 529, 192, 226, 226,
/* 20 */ 128, 128, 31, 149, 9, 9, 531, 192, 226, 226,
/* 30 */ 226, 226, 226, 226, 226, 226, 226, 226, 226, 226,
/* 40 */ 226, 226, 226, 226, 226, 260, 260, 25, 25, 25,
/* 50 */ 25, 25, 25, 42, 25, 165, 128, 128, 135, 135,
/* 60 */ 185, 128, 128, 128, 128, 128, 128, 128, 128, 128,
/* 70 */ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
/* 80 */ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
/* 90 */ 128, 128, 128, 128, 128, 269, 321, 321, 282, 282,
/* 100 */ 321, 281, 285, 293, 299, 296, 302, 305, 307, 292,
/* 110 */ 269, 321, 321, 341, 341, 321, 333, 335, 370, 342,
/* 120 */ 339, 371, 344, 348, 321, 353, 321, 353, 529, 529,
/* 130 */ 27, 68, 68, 68, 94, 119, 213, 213, 213, 216,
/* 140 */ 169, 169, 169, 169, 190, 208, 67, 89, 60, 60,
/* 150 */ 236, 173, 204, 205, 206, 211, 304, 308, 284, 220,
/* 160 */ 199, 53, 223, 228, 229, 327, 330, 191, 201, 266,
/* 170 */ 418, 297, 427, 303, 376, 431, 306, 432, 309, 381,
/* 180 */ 437, 332, 407, 380, 343, 364, 372, 365, 373, 374,
/* 190 */ 377, 471, 379, 382, 384, 375, 366, 383, 369, 385,
/* 200 */ 387, 389, 388, 391, 390, 405, 480, 485, 486, 487,
/* 210 */ 488, 493, 490, 491, 492, 397, 423, 489, 428, 443,
/* 220 */ 494, 378, 386, 401, 498, 501, 410, 412, 401, 504,
/* 230 */ 506, 507, 509, 510, 511, 512, 513, 514, 515, 463,
/* 240 */ 464, 447, 503, 505, 466, 469, 528,
/* 90 */ 128, 128, 128, 128, 128, 128, 269, 321, 321, 282,
/* 100 */ 282, 321, 281, 286, 293, 300, 297, 303, 305, 307,
/* 110 */ 295, 269, 321, 321, 342, 342, 321, 334, 336, 371,
/* 120 */ 341, 340, 370, 343, 346, 321, 353, 321, 353, 531,
/* 130 */ 531, 27, 68, 68, 68, 94, 119, 213, 213, 213,
/* 140 */ 216, 169, 169, 169, 169, 190, 208, 67, 89, 60,
/* 150 */ 60, 236, 173, 204, 205, 206, 211, 304, 308, 284,
/* 160 */ 220, 199, 53, 223, 228, 229, 327, 330, 191, 201,
/* 170 */ 266, 424, 298, 426, 306, 377, 431, 309, 432, 311,
/* 180 */ 381, 435, 312, 408, 382, 345, 363, 373, 366, 372,
/* 190 */ 375, 378, 474, 380, 383, 385, 376, 367, 384, 374,
/* 200 */ 386, 388, 391, 390, 393, 392, 406, 473, 486, 487,
/* 210 */ 488, 489, 494, 491, 492, 493, 398, 425, 485, 430,
/* 220 */ 490, 387, 389, 402, 499, 500, 410, 412, 402, 504,
/* 230 */ 505, 506, 507, 508, 510, 512, 513, 514, 515, 463,
/* 240 */ 464, 447, 503, 509, 466, 468, 527,
};
#define YY_REDUCE_COUNT (129)
#define YY_REDUCE_COUNT (130)
#define YY_REDUCE_MIN (-254)
#define YY_REDUCE_MAX (253)
#define YY_REDUCE_MAX (255)
static const short yy_reduce_ofst[] = {
/* 0 */ -198, -53, -254, -246, -150, -172, -192, -116, -91, -90,
/* 10 */ -207, -203, -248, -179, -162, -138, -218, -175, -19, -17,
@ -378,10 +378,11 @@ static const short yy_reduce_ofst[] = {
/* 60 */ 121, 153, 154, 156, 157, 159, 160, 161, 162, 163,
/* 70 */ 164, 166, 167, 168, 170, 171, 172, 174, 175, 176,
/* 80 */ 177, 178, 179, 180, 181, 183, 184, 186, 187, 188,
/* 90 */ 189, 193, 194, 195, 196, 136, 197, 198, 133, 138,
/* 100 */ 200, 146, 155, 202, 207, 203, 210, 209, 214, 217,
/* 110 */ 215, 212, 218, 219, 221, 222, 224, 227, 225, 231,
/* 120 */ 230, 233, 241, 239, 232, 234, 243, 235, 248, 253,
/* 90 */ 189, 193, 194, 195, 196, 197, 136, 198, 200, 133,
/* 100 */ 138, 202, 146, 155, 203, 207, 209, 212, 214, 210,
/* 110 */ 217, 215, 218, 219, 221, 222, 224, 225, 227, 230,
/* 120 */ 233, 232, 235, 241, 231, 234, 244, 237, 254, 251,
/* 130 */ 255,
};
static const YYACTIONTYPE yy_default[] = {
/* 0 */ 615, 667, 823, 823, 615, 615, 615, 615, 615, 615,
@ -391,21 +392,21 @@ static const YYACTIONTYPE yy_default[] = {
/* 40 */ 615, 615, 615, 615, 615, 615, 615, 615, 615, 615,
/* 50 */ 615, 615, 615, 615, 615, 615, 615, 615, 772, 772,
/* 60 */ 746, 615, 615, 615, 615, 615, 615, 615, 615, 615,
/* 70 */ 615, 615, 615, 615, 615, 615, 615, 615, 654, 615,
/* 80 */ 652, 615, 615, 615, 615, 615, 615, 615, 615, 615,
/* 90 */ 615, 615, 641, 615, 615, 615, 635, 635, 615, 615,
/* 100 */ 635, 779, 783, 777, 765, 773, 764, 760, 759, 787,
/* 110 */ 615, 635, 635, 664, 664, 635, 685, 683, 681, 673,
/* 120 */ 679, 675, 677, 671, 635, 662, 635, 662, 700, 713,
/* 130 */ 615, 788, 822, 778, 806, 805, 818, 812, 811, 615,
/* 140 */ 810, 809, 808, 807, 615, 615, 615, 615, 814, 813,
/* 150 */ 615, 615, 615, 615, 615, 615, 615, 615, 615, 790,
/* 160 */ 784, 780, 615, 615, 615, 615, 615, 615, 615, 615,
/* 70 */ 615, 615, 615, 615, 615, 615, 615, 615, 615, 654,
/* 80 */ 615, 652, 615, 615, 615, 615, 615, 615, 615, 615,
/* 90 */ 615, 615, 615, 641, 615, 615, 615, 635, 635, 615,
/* 100 */ 615, 635, 779, 783, 777, 765, 773, 764, 760, 759,
/* 110 */ 787, 615, 635, 635, 664, 664, 635, 685, 683, 681,
/* 120 */ 673, 679, 675, 677, 671, 635, 662, 635, 662, 700,
/* 130 */ 713, 615, 788, 822, 778, 806, 805, 818, 812, 811,
/* 140 */ 615, 810, 809, 808, 807, 615, 615, 615, 615, 814,
/* 150 */ 813, 615, 615, 615, 615, 615, 615, 615, 615, 615,
/* 160 */ 790, 784, 780, 615, 615, 615, 615, 615, 615, 615,
/* 170 */ 615, 615, 615, 615, 615, 615, 615, 615, 615, 615,
/* 180 */ 615, 615, 615, 615, 615, 745, 615, 615, 754, 615,
/* 190 */ 615, 615, 615, 615, 615, 774, 615, 766, 615, 615,
/* 200 */ 615, 615, 615, 615, 722, 615, 615, 615, 615, 615,
/* 210 */ 615, 615, 615, 615, 615, 688, 615, 615, 615, 615,
/* 180 */ 615, 615, 615, 615, 615, 615, 745, 615, 615, 754,
/* 190 */ 615, 615, 615, 615, 615, 615, 774, 615, 766, 615,
/* 200 */ 615, 615, 615, 615, 615, 722, 615, 615, 615, 615,
/* 210 */ 615, 615, 615, 615, 615, 615, 688, 615, 615, 615,
/* 220 */ 615, 615, 615, 827, 615, 615, 615, 716, 825, 615,
/* 230 */ 615, 615, 615, 615, 615, 615, 615, 615, 615, 615,
/* 240 */ 615, 615, 639, 637, 615, 631, 615,
@ -1038,7 +1039,7 @@ static const char *const yyRuleName[] = {
/* 44 */ "ifexists ::=",
/* 45 */ "ifnotexists ::= IF NOT EXISTS",
/* 46 */ "ifnotexists ::=",
/* 47 */ "cmd ::= CREATE DNODE IPTOKEN",
/* 47 */ "cmd ::= CREATE DNODE ids",
/* 48 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr",
/* 49 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr",
/* 50 */ "cmd ::= CREATE USER ids PASS ids",
@ -1711,7 +1712,7 @@ static const struct {
{ 209, 0 }, /* (44) ifexists ::= */
{ 212, -3 }, /* (45) ifnotexists ::= IF NOT EXISTS */
{ 212, 0 }, /* (46) ifnotexists ::= */
{ 205, -3 }, /* (47) cmd ::= CREATE DNODE IPTOKEN */
{ 205, -3 }, /* (47) cmd ::= CREATE DNODE ids */
{ 205, -6 }, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */
{ 205, -5 }, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */
{ 205, -5 }, /* (50) cmd ::= CREATE USER ids PASS ids */
@ -2122,7 +2123,7 @@ static void yy_reduce(
case 45: /* ifnotexists ::= IF NOT EXISTS */
{yymsp[-2].minor.yy0.n = 1;}
break;
case 47: /* cmd ::= CREATE DNODE IPTOKEN */
case 47: /* cmd ::= CREATE DNODE ids */
{ setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);}
break;
case 48: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */

4
src/query/tests/astTest.cpp
View File

@ -582,7 +582,7 @@ void exprSerializeTest1() {
tExprTreeDestroy(&p1, nullptr);
tExprTreeDestroy(&p2, nullptr);
tbufClose(&bw);
// tbufClose(&bw);
}
void exprSerializeTest2() {
@ -627,7 +627,7 @@ void exprSerializeTest2() {
tExprTreeDestroy(&p1, nullptr);
tExprTreeDestroy(&p2, nullptr);
tbufClose(&bw);
// tbufClose(&bw);
}
} // namespace
TEST(testCase, astTest) {

4
src/rpc/inc/rpcCache.h
View File

@ -22,8 +22,8 @@ extern "C" {
void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer);
void rpcCloseConnCache(void *handle);
void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, int8_t connType);
void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connType);
void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType);
void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType);
#ifdef __cplusplus
}

6
src/rpc/inc/rpcTcp.h
View File

@ -20,12 +20,12 @@
extern "C" {
#endif
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void taosCleanUpTcpServer(void *param);
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle);
void taosCleanUpTcpClient(void *chandle);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port);
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
void taosCloseTcpConnection(void *chandle);
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);

4
src/rpc/inc/rpcUdp.h
View File

@ -22,10 +22,10 @@ extern "C" {
#include "taosdef.h"
void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int, void *fp, void *shandle);
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle);
void taosCleanUpUdpConnection(void *handle);
int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port);
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port);
void taosFreeMsgHdr(void *hdr);
int taosMsgHdrSize(void *hdr);

32
src/rpc/src/rpcCache.c
View File

@ -23,7 +23,7 @@
#include "rpcCache.h"
typedef struct SConnHash {
uint32_t ip;
char fqdn[TSDB_FQDN_LEN];
uint16_t port;
char connType;
struct SConnHash *prev;
@ -46,7 +46,7 @@ typedef struct {
int64_t *lockedBy;
} SConnCache;
static int rpcHashConn(void *handle, uint32_t ip, uint16_t port, int8_t connType);
static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType);
static void rpcLockCache(int64_t *lockedBy);
static void rpcUnlockCache(int64_t *lockedBy);
static void rpcCleanConnCache(void *handle, void *tmrId);
@ -114,7 +114,7 @@ void rpcCloseConnCache(void *handle) {
free(pCache);
}
void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, int8_t connType) {
void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) {
int hash;
SConnHash * pNode;
SConnCache *pCache;
@ -125,9 +125,9 @@ void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, i
assert(pCache);
assert(data);
hash = rpcHashConn(pCache, ip, port, connType);
hash = rpcHashConn(pCache, fqdn, port, connType);
pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool);
pNode->ip = ip;
strcpy(pNode->fqdn, fqdn);
pNode->port = port;
pNode->connType = connType;
pNode->data = data;
@ -147,12 +147,12 @@ void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, i
pCache->total++;
tTrace("%p ip:0x%x:%hu:%d:%d:%p added into cache, connections:%d", data, ip, port, connType, hash, pNode, pCache->count[hash]);
tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]);
return;
}
void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connType) {
void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType) {
int hash;
SConnHash * pNode;
SConnCache *pCache;
@ -163,7 +163,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT
uint64_t time = taosGetTimestampMs();
hash = rpcHashConn(pCache, ip, port, connType);
hash = rpcHashConn(pCache, fqdn, port, connType);
rpcLockCache(pCache->lockedBy+hash);
pNode = pCache->connHashList[hash];
@ -174,7 +174,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT
break;
}
if (pNode->ip == ip && pNode->port == port && pNode->connType == connType) break;
if (strcmp(pNode->fqdn, fqdn) == 0 && pNode->port == port && pNode->connType == connType) break;
pNode = pNode->next;
}
@ -201,7 +201,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT
rpcUnlockCache(pCache->lockedBy+hash);
if (pData) {
tTrace("%p ip:0x%x:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, ip, port, connType, hash, pNode, pCache->count[hash]);
tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]);
}
return pData;
@ -239,7 +239,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
pNext = pNode->next;
pCache->total--;
pCache->count[hash]--;
tTrace("%p ip:0x%x:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->ip, pNode->port, pNode->connType, hash, pNode,
tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode,
pCache->count[hash]);
taosMemPoolFree(pCache->connHashMemPool, (char *)pNode);
pNode = pNext;
@ -251,12 +251,16 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash
pCache->connHashList[hash] = NULL;
}
static int rpcHashConn(void *handle, uint32_t ip, uint16_t port, int8_t connType) {
static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) {
SConnCache *pCache = (SConnCache *)handle;
int hash = 0;
char *temp = fqdn;
while (*temp) {
hash += *temp;
++temp;
}
hash = ip >> 16;
hash += (unsigned short)(ip & 0xFFFF);
hash += port;
hash += connType;

61
src/rpc/src/rpcMain.c
View File

@ -44,7 +44,6 @@ typedef struct {
int sessions; // number of sessions allowed
int numOfThreads; // number of threads to process incoming messages
int idleTime; // milliseconds;
char localIp[TSDB_IPv4ADDR_LEN];
uint16_t localPort;
int8_t connType;
int index; // for UDP server only, round robin for multiple threads
@ -101,9 +100,8 @@ typedef struct SRpcConn {
uint16_t localPort; // for UDP only
uint32_t linkUid; // connection unique ID assigned by client
uint32_t peerIp; // peer IP
uint32_t destIp; // server destination IP to handle NAT
uint16_t peerPort; // peer port
char peerIpstr[TSDB_IPv4ADDR_LEN]; // peer IP string
char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string
uint16_t tranId; // outgoing transcation ID, for build message
uint16_t outTranId; // outgoing transcation ID
uint16_t inTranId; // transcation ID for incoming msg
@ -140,7 +138,7 @@ int tsRpcOverhead;
#define RPC_CONN_TCPC 3
#define RPC_CONN_TCP 2
void *(*taosInitConn[])(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
taosInitUdpConnection,
taosInitUdpConnection,
taosInitTcpServer,
@ -161,7 +159,7 @@ int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *cha
taosSendTcpData
};
void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = {
void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = {
taosOpenUdpConnection,
taosOpenUdpConnection,
NULL,
@ -175,7 +173,7 @@ void (*taosCloseConn[])(void *chandle) = {
taosCloseTcpConnection
};
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType);
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType);
static void rpcCloseConn(void *thandle);
static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext);
static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc);
@ -217,7 +215,6 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime;
pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
if (pInit->localIp) strcpy(pRpc->localIp, pInit->localIp);
pRpc->localPort = pInit->localPort;
pRpc->afp = pInit->afp;
pRpc->sessions = pInit->sessions;
@ -229,13 +226,13 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp;
pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(pRpc->localIp, pRpc->localPort, pRpc->label,
pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label,
pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
pRpc->udphandle = (*taosInitConn[pRpc->connType])(pRpc->localIp, pRpc->localPort, pRpc->label,
pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label,
pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) {
tError("%s failed to init network, %s:%d", pRpc->label, pRpc->localIp, pRpc->localPort);
tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort);
rpcClose(pRpc);
return NULL;
}
@ -457,7 +454,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
pInfo->clientIp = pConn->peerIp;
pInfo->clientPort = pConn->peerPort;
pInfo->serverIp = pConn->destIp;
// pInfo->serverIp = pConn->destIp;
strcpy(pInfo->user, pConn->user);
return 0;
@ -490,27 +487,32 @@ static void rpcFreeMsg(void *msg) {
}
}
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType) {
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) {
SRpcConn *pConn;
uint32_t peerIp = taosGetIpFromFqdn(peerFqdn);
if (peerIp == -1) {
tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn);
return NULL;
}
pConn = rpcAllocateClientConn(pRpc);
if (pConn) {
strcpy(pConn->peerIpstr, peerIpStr);
pConn->peerIp = inet_addr(peerIpStr);
strcpy(pConn->peerFqdn, peerFqdn);
pConn->peerIp = peerIp;
pConn->peerPort = peerPort;
strcpy(pConn->user, pRpc->user);
pConn->connType = connType;
if (taosOpenConn[connType]) {
void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle;
pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIpstr, pConn->peerPort);
pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort);
if (pConn->chandle) {
tTrace("%s %p, rpc connection is set up, sid:%d id:%s ip:%s:%hu connType:%d", pRpc->label,
pConn, pConn->sid, pRpc->user, pConn->peerIpstr, pConn->peerPort, pConn->connType);
tTrace("%s %p, rpc connection is set up, sid:%d id:%s %s:%hu connType:%d", pRpc->label,
pConn, pConn->sid, pRpc->user, peerFqdn, pConn->peerPort, pConn->connType);
} else {
tError("%s %p, failed to set up connection to ip:%s:%hu", pRpc->label, pConn,
pConn->peerIpstr, pConn->peerPort);
tError("%s %p, failed to set up connection to %s:%hu", pRpc->label, pConn, peerFqdn, pConn->peerPort);
terrno = TSDB_CODE_NETWORK_UNAVAIL;
rpcCloseConn(pConn);
pConn = NULL;
@ -661,12 +663,9 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) {
SRpcInfo *pRpc = pContext->pRpc;
SRpcIpSet *pIpSet = &pContext->ipSet;
pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->ip[pIpSet->inUse], pIpSet->port, pContext->connType);
pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType);
if ( pConn == NULL || pConn->user[0] == 0) {
char ipstr[20] = {0};
tinet_ntoa(ipstr, pIpSet->ip[pIpSet->inUse]);
pConn = rpcOpenConn(pRpc, ipstr, pIpSet->port, pContext->connType);
if (pConn) pConn->destIp = pIpSet->ip[pIpSet->inUse];
pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType);
} else {
tTrace("%s %p, connection is retrieved from cache", pRpc->label, pConn);
}
@ -789,7 +788,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
pConn->peerIp = pRecv->ip;
char ipstr[20] = {0};
tinet_ntoa(ipstr, pRecv->ip);
strcpy(pConn->peerIpstr, ipstr);
strcpy(pConn->peerFqdn, ipstr);
}
if (pRecv->port) pConn->peerPort = pRecv->port;
@ -922,7 +921,6 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
if ( rpcIsReq(pHead->msgType) ) {
rpcMsg.handle = pConn;
pConn->destIp = pHead->destIp;
taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer);
(*(pRpc->cfp))(&rpcMsg);
} else {
@ -932,7 +930,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
pConn->pContext = NULL;
// for UDP, port may be changed by server, the port in ipSet shall be used for cache
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerIp, pContext->ipSet.port, pConn->connType);
rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType);
if (pHead->code == TSDB_CODE_REDIRECT) {
pContext->redirect = 1;
@ -1053,7 +1051,6 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
pHead->tranId = pConn->tranId;
pHead->sourceId = pConn->ownId;
pHead->destId = pConn->peerId;
pHead->destIp = pConn->destIp;
pHead->port = 0;
pHead->linkUid = pConn->linkUid;
if (!pConn->secured) memcpy(pHead->user, pConn->user, tListLen(pHead->user));
@ -1081,12 +1078,12 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
if ( rpcIsReq(pHead->msgType)) {
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
tTrace("%s %p, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d",
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr,
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn,
pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
} else {
if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, pConn->peerPort,
pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort,
htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
}
@ -1141,13 +1138,13 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
if (pConn->retry < 4) {
tTrace("%s %p, re-send msg:%s to %s:%hud", pRpc->label, pConn,
taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort);
taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen);
taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer);
} else {
// close the connection
tTrace("%s %p, failed to send msg:%s to %s:%hu", pRpc->label, pConn,
taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort);
taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort);
reportDisc = 1;
}
} else {

26
src/rpc/src/rpcTcp.c
View File

@ -40,7 +40,7 @@ typedef struct SThreadObj {
SFdObj * pHead;
pthread_mutex_t mutex;
pthread_cond_t fdReady;
char ipstr[TSDB_IPv4ADDR_LEN];
uint32_t ip;
int pollFd;
int numOfFds;
int threadId;
@ -50,7 +50,7 @@ typedef struct SThreadObj {
} SThreadObj;
typedef struct {
char ip[TSDB_IPv4ADDR_LEN];
uint32_t ip;
uint16_t port;
char label[12];
int numOfThreads;
@ -65,12 +65,12 @@ static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void taosAcceptTcpConnection(void *arg);
void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
SServerObj *pServerObj;
SThreadObj *pThreadObj;
pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
strcpy(pServerObj->ip, ip);
pServerObj->ip = ip;
pServerObj->port = port;
strcpy(pServerObj->label, label);
pServerObj->numOfThreads = numOfThreads;
@ -138,7 +138,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads,
free(pServerObj);
pServerObj = NULL;
} else {
tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
tTrace("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
}
return (void *)pServerObj;
@ -222,14 +222,14 @@ static void taosAcceptTcpConnection(void *arg) {
}
}
void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
SThreadObj *pThreadObj;
pthread_attr_t thattr;
pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj));
memset(pThreadObj, 0, sizeof(SThreadObj));
strcpy(pThreadObj->label, label);
strcpy(pThreadObj->ipstr, ip);
pThreadObj->ip = ip;
pThreadObj->shandle = shandle;
if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) {
@ -284,21 +284,19 @@ void taosCleanUpTcpClient(void *chandle) {
tfree(pThreadObj);
}
void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
SThreadObj * pThreadObj = shandle;
struct in_addr destIp;
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr);
int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
if (fd <= 0) return NULL;
inet_aton(ip, &destIp);
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
if (pFdObj) {
pFdObj->thandle = thandle;
pFdObj->port = port;
pFdObj->ip = destIp.s_addr;
tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d",
pFdObj->ip = ip;
tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds);
} else {
close(fd);
@ -403,7 +401,7 @@ static void *taosProcessTcpData(void *param) {
continue;
}
// tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen);
// tTrace("%s TCP data is received, ip:0x%x:%u len:%d", pThreadObj->label, pFdObj->ip, pFdObj->port, msgLen);
memcpy(msg, &rpcHead, sizeof(SRpcHead));
recvInfo.msg = msg;

14
src/rpc/src/rpcUdp.c
View File

@ -51,7 +51,7 @@ typedef struct {
typedef struct {
int index;
int server;
char ip[16]; // local IP
uint32_t ip; // local IP
uint16_t port; // local Port
void *shandle; // handle passed by upper layer during server initialization
int threads;
@ -77,7 +77,7 @@ static void *taosRecvUdpData(void *param);
static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port);
static void taosProcessUdpBufTimer(void *param, void *tmrId);
void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) {
SUdpConn *pConn;
SUdpConnSet *pSet;
@ -89,7 +89,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
}
memset(pSet, 0, (size_t)size);
strcpy(pSet->ip, ip);
pSet->ip = ip;
pSet->port = port;
pSet->shandle = shandle;
pSet->fp = fp;
@ -111,7 +111,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
ownPort = (port ? port + i : 0);
pConn->fd = taosOpenUdpSocket(ip, ownPort);
if (pConn->fd < 0) {
tError("%s failed to open UDP socket %s:%hu", label, ip, port);
tError("%s failed to open UDP socket %x:%hu", label, ip, port);
taosCleanUpUdpConnection(pSet);
return NULL;
}
@ -157,7 +157,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v
++pSet->threads;
}
tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads);
tTrace("%s UDP connection is initialized, ip:%x port:%hu threads:%d", label, ip, port, threads);
return pSet;
}
@ -190,7 +190,7 @@ void taosCleanUpUdpConnection(void *handle) {
tfree(pSet);
}
void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
SUdpConnSet *pSet = (SUdpConnSet *)shandle;
pSet->index = (pSet->index + 1) % pSet->threads;
@ -198,7 +198,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por
SUdpConn *pConn = pSet->udpConn + pSet->index;
pConn->port = port;
tTrace("%s UDP connection is setup, ip: %s:%hu, local: %s:%d", pConn->label, ip, port, pSet->ip,
tTrace("%s UDP connection is setup, ip:%x:%hu, local:%x:%d", pConn->label, ip, port, pSet->ip,
ntohs((uint16_t)pConn->localPort));
return pConn;

17
src/rpc/test/rclient.c
View File

@ -88,13 +88,13 @@ int main(int argc, char *argv[]) {
// server info
ipSet.numOfIps = 1;
ipSet.inUse = 0;
ipSet.port = 7000;
ipSet.ip[0] = inet_addr(serverIp);
ipSet.ip[1] = inet_addr("192.168.0.1");
ipSet.port[0] = 7000;
ipSet.port[1] = 7000;
strcpy(ipSet.fqdn[0], serverIp);
strcpy(ipSet.fqdn[1], "192.168.0.1");
// client info
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = "0.0.0.0";
rpcInit.localPort = 0;
rpcInit.label = "APP";
rpcInit.numOfThreads = 1;
@ -110,11 +110,9 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port = atoi(argv[++i]);
ipSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
ipSet.ip[0] = inet_addr(argv[++i]);
} else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
strcpy(rpcInit.localIp, argv[++i]);
strcpy(ipSet.fqdn[0], argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -138,10 +136,9 @@ int main(int argc, char *argv[]) {
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-l localIp]: local IP address, default is:%s\n", rpcInit.localIp);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-a threads]: number of app threads, default is:%d\n", appThreads);
printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);

18
src/rpc/test/rsclient.c
View File

@ -89,13 +89,14 @@ int main(int argc, char *argv[]) {
// server info
ipSet.numOfIps = 1;
ipSet.inUse = 0;
ipSet.port = 7000;
ipSet.ip[0] = inet_addr(serverIp);
ipSet.ip[1] = inet_addr("192.168.0.1");
ipSet.port[0] = 7000;
ipSet.port[1] = 7000;
strcpy(ipSet.fqdn[0], serverIp);
strcpy(ipSet.fqdn[1], "192.168.0.1");
// client info
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = "0.0.0.0";
//rpcInit.localIp = "0.0.0.0";
rpcInit.localPort = 0;
rpcInit.label = "APP";
rpcInit.numOfThreads = 1;
@ -111,11 +112,9 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
ipSet.port = atoi(argv[++i]);
ipSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
ipSet.ip[0] = inet_addr(argv[++i]);
} else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
strcpy(rpcInit.localIp, argv[++i]);
strcpy(ipSet.fqdn[0], argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -139,10 +138,9 @@ int main(int argc, char *argv[]) {
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: first server IP address, default is:%s\n", serverIp);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port);
printf(" [-p port]: server port number, default is:%d\n", ipSet.port[0]);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
printf(" [-l localIp]: local IP address, default is:%s\n", rpcInit.localIp);
printf(" [-m msgSize]: message body size, default is:%d\n", msgSize);
printf(" [-a threads]: number of app threads, default is:%d\n", appThreads);
printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);

5
src/rpc/test/rserver.c
View File

@ -126,10 +126,8 @@ void processRequestMsg(SRpcMsg *pMsg) {
int main(int argc, char *argv[]) {
SRpcInit rpcInit;
char dataName[20] = "server.data";
char localIp[40] = "0.0.0.0";
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localIp = localIp;
rpcInit.localPort = 7000;
rpcInit.label = "SER";
rpcInit.numOfThreads = 1;
@ -141,8 +139,6 @@ int main(int argc, char *argv[]) {
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
rpcInit.localPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i")==0 && i < argc-1) {
strcpy(rpcInit.localIp, argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
@ -159,7 +155,6 @@ int main(int argc, char *argv[]) {
uDebugFlag = rpcDebugFlag;
} else {
printf("\nusage: %s [options] \n", argv[0]);
printf(" [-i ip]: server IP address, default is:%s\n", rpcInit.localIp);
printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort);
printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);

11
src/tsdb/inc/tsdbMain.h
View File

@ -153,17 +153,16 @@ typedef struct {
} SCacheMem;
typedef struct {
int maxBytes;
int cacheBlockSize;
int totalCacheBlocks;
STsdbCachePool pool;
STsdbCacheBlock *curBlock;
SCacheMem * mem;
SCacheMem * imem;
TsdbRepoT * pRepo;
TsdbRepoT * pRepo;
} STsdbCache;
STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo);
STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo);
void tsdbFreeCache(STsdbCache *pCache);
void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);
@ -297,7 +296,7 @@ typedef struct {
// TODO: take pre-calculation into account
typedef struct {
int16_t colId; // Column ID
int16_t len; // Column length
int16_t len; // Column length // TODO: int16_t is not enough
int32_t type : 8;
int32_t offset : 24;
} SCompCol;
@ -426,6 +425,8 @@ typedef struct {
SCompData *pCompData;
SDataCols *pDataCols[2];
void *blockBuffer; // Buffer to hold the whole data block
void *compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper;
// --------- Helper state
@ -445,13 +446,11 @@ typedef struct {
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
// int tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg);
void tsdbDestroyHelper(SRWHelper *pHelper);
void tsdbResetHelper(SRWHelper *pHelper);
// --------- For set operations
int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
// void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema);
void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);

12
src/tsdb/src/tsdbCache.c
View File

@ -21,29 +21,25 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache);
static void tsdbFreeBlockList(SList *list);
static void tsdbFreeCacheMem(SCacheMem *mem);
STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) {
STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo) {
STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache));
if (pCache == NULL) return NULL;
if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
cacheBlockSize *= (1024 * 1024);
if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS;
if (totalBlocks <= 1) totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
pCache->maxBytes = maxBytes;
pCache->cacheBlockSize = cacheBlockSize;
pCache->totalCacheBlocks = totalBlocks;
pCache->pRepo = pRepo;
int nBlocks = maxBytes / cacheBlockSize + 1;
if (nBlocks <= 1) nBlocks = 2;
pCache->totalCacheBlocks = nBlocks;
STsdbCachePool *pPool = &(pCache->pool);
pPool->index = 0;
pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *));
if (pPool->memPool == NULL) goto _err;
for (int i = 0; i < nBlocks; i++) {
for (int i = 0; i < totalBlocks; i++) {
STsdbCacheBlock *pBlock = (STsdbCacheBlock *)malloc(sizeof(STsdbCacheBlock) + cacheBlockSize);
if (pBlock == NULL) {
goto _err;

33
src/tsdb/src/tsdbMain.c
View File

@ -5,6 +5,7 @@
#include "tsdb.h"
#include "tsdbMain.h"
#include "tscompression.h"
#include "tchecksum.h"
#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI // default precision
#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
@ -157,7 +158,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
for (int i = 0; i < pRepo->config.maxTables; i++) {
for (int i = 1; i < pRepo->config.maxTables; i++) {
STable * pTable = pMeta->tables[i];
SCompIdx *pIdx = &rhelper.pCompIdx[i];
@ -202,7 +203,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
return NULL;
}
pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo);
pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TsdbRepoT *)pRepo);
if (pRepo->tsdbCache == NULL) {
tsdbFreeMeta(pRepo->tsdbMeta);
free(pRepo->rootDir);
@ -259,7 +260,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
}
pRepo->commit = 1;
// Loop to move pData to iData
for (int i = 0; i < pRepo->config.maxTables; i++) {
for (int i = 1; i < pRepo->config.maxTables; i++) {
STable *pTable = pRepo->tsdbMeta->tables[i];
if (pTable != NULL && pTable->mem != NULL) {
pTable->imem = pTable->mem;
@ -312,7 +313,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
}
pRepo->commit = 1;
// Loop to move pData to iData
for (int i = 0; i < pRepo->config.maxTables; i++) {
for (int i = 1; i < pRepo->config.maxTables; i++) {
STable *pTable = pRepo->tsdbMeta->tables[i];
if (pTable != NULL && pTable->mem != NULL) {
pTable->imem = pTable->mem;
@ -367,6 +368,15 @@ int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
return 0;
}
TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
if (pTable == NULL) return -1;
return TSDB_GET_TABLE_LAST_KEY(pTable);
}
int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
if (repo == NULL) return -1;
STsdbRepo *pRepo = (STsdbRepo *)repo;
@ -601,6 +611,10 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) return -1;
}
// Since tableId starts from 1, we increase maxTables by 1
// TODO: take a fancier way to do this
pCfg->maxTables++;
// Check daysPerFile
if (pCfg->daysPerFile == -1) {
pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
@ -823,7 +837,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max
static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) {
if (iters == NULL) return;
for (int tid = 0; tid < maxTables; tid++) {
for (int tid = 1; tid < maxTables; tid++) {
if (iters[tid] == NULL) continue;
tSkipListDestroyIter(iters[tid]);
}
@ -835,7 +849,7 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
SSkipListIterator **iters = (SSkipListIterator **)calloc(maxTables, sizeof(SSkipListIterator *));
if (iters == NULL) return NULL;
for (int tid = 0; tid < maxTables; tid++) {
for (int tid = 1; tid < maxTables; tid++) {
STable *pTable = pMeta->tables[tid];
if (pTable == NULL || pTable->imem == NULL) continue;
@ -902,7 +916,7 @@ _exit:
free(pCache->imem);
pCache->imem = NULL;
pRepo->commit = 0;
for (int i = 0; i < pCfg->maxTables; i++) {
for (int i = 1; i < pCfg->maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable && pTable->imem) {
tsdbFreeMemTable(pTable->imem);
@ -915,7 +929,7 @@ _exit:
}
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) {
char dataDir[128] = {0};
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
STsdbCfg * pCfg = &pRepo->config;
@ -936,7 +950,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err;
// Loop to commit data in each table
for (int tid = 0; tid < pCfg->maxTables; tid++) {
for (int tid = 1; tid < pCfg->maxTables; tid++) {
STable * pTable = pMeta->tables[tid];
if (pTable == NULL) continue;
@ -974,7 +988,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
// Write the SCompBlock part
if (tsdbWriteCompInfo(pHelper) < 0) goto _err;
}
if (tsdbWriteCompIdx(pHelper) < 0) goto _err;

11
src/tsdb/src/tsdbMeta.c
View File

@ -129,7 +129,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
void tsdbOrgMeta(void *pHandle) {
STsdbMeta *pMeta = (STsdbMeta *)pHandle;
for (int i = 0; i < pMeta->maxTables; i++) {
for (int i = 1; i < pMeta->maxTables; i++) {
STable *pTable = pMeta->tables[i];
if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) {
tsdbAddTableIntoIndex(pMeta, pTable);
@ -179,7 +179,7 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
tsdbCloseMetaFile(pMeta->mfh);
for (int i = 0; i < pMeta->maxTables; i++) {
for (int i = 1; i < pMeta->maxTables; i++) {
if (pMeta->tables[i] != NULL) {
tsdbFreeTable(pMeta->tables[i]);
}
@ -242,7 +242,7 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId id, int32_t colId, int16_t*
assert(pCol != NULL);
SDataRow row = (SDataRow)pTable->tagVal;
char* d = dataRowAt(row, TD_DATA_ROW_HEAD_SIZE);
char* d = dataRowTuple(row);
*val = d;
*type = pCol->type;
@ -451,9 +451,8 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
// Update the pMeta->maxCols and pMeta->maxRowBytes
if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE) {
if (schemaNCols(pTable->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pTable->schema);
int bytes = tdMaxRowBytesFromSchema(pTable->schema);
int bytes = dataRowMaxBytesFromSchema(pTable->schema);
if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
tdUpdateSchema(pTable->schema);
}
return tsdbAddTableIntoMap(pMeta, pTable);
@ -524,5 +523,5 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
char *getTupleKey(const void * data) {
SDataRow row = (SDataRow)data;
return dataRowAt(row, TD_DATA_ROW_HEAD_SIZE);
return POINTER_DRIFT(row, TD_DATA_ROW_HEAD_SIZE);
}

220
src/tsdb/src/tsdbRWHelper.c
View File

@ -131,6 +131,11 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t
// Init block part
if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
pHelper->blockBuffer =
tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols +
pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM));
if (pHelper->blockBuffer == NULL) goto _err;
return 0;
_err:
@ -149,6 +154,8 @@ int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
void tsdbDestroyHelper(SRWHelper *pHelper) {
if (pHelper) {
tzfree(pHelper->blockBuffer);
tzfree(pHelper->compBuffer);
tsdbDestroyHelperFile(pHelper);
tsdbDestroyHelperTable(pHelper);
tsdbDestroyHelperBlock(pHelper);
@ -330,7 +337,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks);
if (pCompBlock == NULL) { // No key overlap, must has last block, just merge with the last block
ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last);
ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last);
rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols);
if (rowsToWrite < 0) goto _err;
} else { // Has key overlap
@ -552,61 +559,97 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx,
return 0;
}
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfPoints,
int maxPoints, char *buffer, int bufferSize) {
// Verify by checksum
if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1;
// Decode the data
if (comp) {
// // Need to decompress
pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))(
content, len - sizeof(TSCKSUM), numOfPoints, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
pDataCol->len += (sizeof(int32_t) * maxPoints);
dataColSetOffset(pDataCol, numOfPoints);
}
} else {
// No need to decompress, just memcpy it
switch (pDataCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
pDataCol->len = sizeof(int32_t) * maxPoints;
memcpy((char *)pDataCol->pData + pDataCol->len, content, len - sizeof(TSCKSUM));
pDataCol->len += (len - sizeof(TSCKSUM));
dataColSetOffset(pDataCol, numOfPoints);
break;
default:
pDataCol->len = len - sizeof(TSCKSUM);
memcpy(pDataCol->pData, content, pDataCol->len);
break;
}
}
return 0;
}
/**
* Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1)
*/
static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
ASSERT(pCompBlock->numOfSubBlocks <= 1);
SCompData *pCompData = (SCompData *)malloc(pCompBlock->len);
if (pCompData == NULL) return -1;
ASSERT(tsizeof(pHelper->blockBuffer) >= pCompBlock->len);
SCompData *pCompData = (SCompData *)pHelper->blockBuffer;
int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err;
if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err;
ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
// TODO : check the checksum
size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err;
for (int i = 0; i < pCompData->numOfCols; i++) {
// TODO: check the data checksum
// if (!taosCheckChecksumWhole())
}
ASSERT(pCompBlock->numOfCols == pCompData->numOfCols);
pDataCols->numOfPoints = pCompBlock->numOfPoints;
int ccol = 0, dcol = 0;
while (true) {
if (ccol >= pDataCols->numOfCols) {
// TODO: Fill rest NULL
break;
// Recover the data
int ccol = 0;
int dcol = 0;
while (dcol < pDataCols->numOfCols) {
SDataCol *pDataCol = &(pDataCols->cols[dcol]);
if (ccol >= pCompData->numOfCols) {
// Set current column as NULL and forward
dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
dcol++;
continue;
}
if (dcol >= pCompData->numOfCols) break;
SCompCol *pCompCol = &(pCompData->cols[ccol]);
SDataCol *pDataCol = &(pDataCols->cols[dcol]);
if (pCompCol->colId == pDataCol->colId) {
// TODO: uncompress
memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tsize + pCompCol->offset), pCompCol->len);
if (pCompBlock->algorithm == TWO_STAGE_COMP) {
pHelper->compBuffer = trealloc(pHelper->compBuffer, pCompCol->len + COMP_OVERFLOW_BYTES);
if (pHelper->compBuffer == NULL) goto _err;
}
if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len,
pCompBlock->algorithm, pCompBlock->numOfPoints, pDataCols->maxPoints,
pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0)
goto _err;
dcol++;
ccol++;
} else if (pCompCol->colId < pDataCol->colId) {
ccol++;
dcol++;
} else if (pCompCol->colId > pDataCol->colId) {
// TODO: Fill NULL
dcol++;
} else {
ccol++;
// Set current column as NULL and forward
dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
dcol++;
}
}
tfree(pCompData);
return 0;
_err:
tfree(pCompData);
return -1;
}
@ -634,36 +677,6 @@ _err:
return -1;
}
// static int tsdbCheckHelperCfg(SHelperCfg *pCfg) {
// // TODO
// return 0;
// }
// static void tsdbClearHelperFile(SHelperFile *pHFile) {
// pHFile->fid = -1;
// if (pHFile->headF.fd > 0) {
// close(pHFile->headF.fd);
// pHFile->headF.fd = -1;
// }
// if (pHFile->dataF.fd > 0) {
// close(pHFile->dataF.fd);
// pHFile->dataF.fd = -1;
// }
// if (pHFile->lastF.fd > 0) {
// close(pHFile->lastF.fd);
// pHFile->lastF.fd = -1;
// }
// if (pHFile->nHeadF.fd > 0) {
// close(pHFile->nHeadF.fd);
// pHFile->nHeadF.fd = -1;
// }
// if (pHFile->nLastF.fd > 0) {
// close(pHFile->nLastF.fd);
// pHFile->nLastF.fd = -1;
// }
// }
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
ASSERT(pHelper->files.lastF.fd > 0);
struct stat st;
@ -677,81 +690,94 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints &&
rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
SCompData *pCompData = NULL;
SCompData *pCompData = (SCompData *)(pHelper->blockBuffer);
int64_t offset = 0;
offset = lseek(pFile->fd, 0, SEEK_END);
if (offset < 0) goto _err;
pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols + sizeof(TSCKSUM));
if (pCompData == NULL) goto _err;
int nColsNotAllNull = 0;
int32_t toffset = 0;
for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
SDataCol *pDataCol = pDataCols->cols + ncol;
SCompCol *pCompCol = pCompData->cols + nColsNotAllNull;
if (0) {
// TODO: all data to commit are NULL
if (isNEleNull(pDataCol, rowsToWrite)) {
// all data to commit are NULL, just ignore it
continue;
}
// Compress the data here
{
// TODO
}
pCompCol->colId = pDataCol->colId;
pCompCol->type = pDataCol->type;
pCompCol->len = TYPE_BYTES[pCompCol->type] * rowsToWrite; // TODO: change it
pCompCol->offset = toffset;
nColsNotAllNull++;
toffset += pCompCol->len;
}
ASSERT(nColsNotAllNull > 0 && nColsNotAllNull <= pDataCols->numOfCols);
// Compress the data if neccessary
int tcol = 0;
int32_t toffset = 0;
int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
int32_t lsize = tsize;
for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
if (tcol >= nColsNotAllNull) break;
SDataCol *pDataCol = pDataCols->cols + ncol;
SCompCol *pCompCol = pCompData->cols + tcol;
if (pDataCol->colId != pCompCol->colId) continue;
void *tptr = (void *)((char *)pCompData + lsize);
pCompCol->offset = toffset;
int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
if (pHelper->config.compress) {
if (pHelper->config.compress == TWO_STAGE_COMP) {
pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES);
if (pHelper->compBuffer == NULL) goto _err;
}
pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
(char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->blockBuffer) - lsize,
pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer));
} else {
pCompCol->len = tlen;
memcpy(tptr, pDataCol->pData, pCompCol->len);
}
// Add checksum
pCompCol->len += sizeof(TSCKSUM);
taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len);
toffset += pCompCol->len;
lsize += pCompCol->len;
tcol++;
}
pCompData->delimiter = TSDB_FILE_DELIMITER;
pCompData->uid = pHelper->tableInfo.uid;
pCompData->numOfCols = nColsNotAllNull;
// Write SCompData + SCompCol part
size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err;
// Write true data part
int nCompCol = 0;
for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
ASSERT(nCompCol < nColsNotAllNull);
SDataCol *pDataCol = pDataCols->cols + ncol;
SCompCol *pCompCol = pCompData->cols + nCompCol;
if (pDataCol->colId == pCompCol->colId) {
if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err;
tsize += pCompCol->len;
nCompCol++;
}
}
// Write the whole block to file
if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) goto _err;
// Update pCompBlock membership vairables
pCompBlock->last = isLast;
pCompBlock->offset = offset;
pCompBlock->algorithm = pHelper->config.compress;
pCompBlock->numOfPoints = rowsToWrite;
pCompBlock->sversion = pHelper->tableInfo.sversion;
pCompBlock->len = (int32_t)tsize;
pCompBlock->len = (int32_t)lsize;
pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0;
pCompBlock->numOfCols = nColsNotAllNull;
pCompBlock->keyFirst = dataColsKeyFirst(pDataCols);
pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1);
tfree(pCompData);
return 0;
_err:
tfree(pCompData);
return -1;
}
@ -782,7 +808,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
TSKEY keyFirst = dataColsKeyFirst(pDataCols);
SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
ASSERT(blkIdx < pIdx->numOfSuperBlocks);
ASSERT(blkIdx < pIdx->numOfBlocks);
// SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
ASSERT(blockAtIdx(pHelper, blkIdx)->numOfSubBlocks >= 1);
@ -790,7 +816,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
// ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfSuperBlocks-1);
ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1);
int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface
rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfPoints), pDataCols->numOfPoints);
@ -961,7 +987,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) {
static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfSuperBlocks);
ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks);
ASSERT(pCompBlock->numOfSubBlocks == 1);
// Adjust memory if no more room
@ -1004,7 +1030,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
ASSERT(pCompBlock->numOfSubBlocks == 0);
SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks);
SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS);
@ -1088,7 +1114,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks);
SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;

17
src/tsdb/tests/tsdbTests.cpp
View File

@ -27,7 +27,7 @@ typedef struct {
static int insertData(SInsertInfo *pInfo) {
SSubmitMsg *pMsg =
(SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
(SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowMaxBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
if (pMsg == NULL) return -1;
TSKEY start_time = pInfo->startTime;
@ -52,11 +52,12 @@ static int insertData(SInsertInfo *pInfo) {
tdInitDataRow(row, pInfo->pSchema);
for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
STColumn *pTCol = schemaColAt(pInfo->pSchema, j);
if (j == 0) { // Just for timestamp
tdAppendColVal(row, (void *)(&start_time), schemaColAt(pInfo->pSchema, j));
tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->bytes, pTCol->offset);
} else { // For int
int val = 10;
tdAppendColVal(row, (void *)(&val), schemaColAt(pInfo->pSchema, j));
tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset);
}
}
pBlock->len += dataRowLen(row);
@ -105,9 +106,9 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
}
@ -149,9 +150,9 @@ TEST(TsdbTest, createRepo) {
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
}
@ -244,7 +245,7 @@ TEST(TsdbTest, DISABLED_openRepo) {
// tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
// STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
// SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
// SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5);
// tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);

6
src/util/CMakeLists.txt
View File

@ -11,7 +11,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil pthread os m rt)
TARGET_LINK_LIBRARIES(tutil pthread os m rt lz4)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)
ADD_DEFINITIONS(-DUSE_LIBICONV)
@ -68,7 +68,7 @@ ELSEIF (TD_WINDOWS_64)
LIST(APPEND SRC ./src/tutil.c)
LIST(APPEND SRC ./src/version.c)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32)
TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32 lz4)
ELSEIF(TD_DARWIN_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
LIST(APPEND SRC ./src/hash.c)
@ -105,7 +105,7 @@ ELSEIF(TD_DARWIN_64)
LIST(APPEND SRC ./src/version.c)
LIST(APPEND SRC ./src/hash.c)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil iconv pthread os)
TARGET_LINK_LIBRARIES(tutil iconv pthread os lz4)
ENDIF()
# TARGET_LINK_LIBRARIES(tutil mstorage)

251
src/util/inc/tscompression.h
View File

@ -21,7 +21,9 @@ extern "C" {
#endif
#include "taosdef.h"
#include "tutil.h"
#define COMP_OVERFLOW_BYTES 2
#define BITS_PER_BYTE 8
// Masks
#define INT64MASK(_x) ((1ul << _x) - 1)
@ -32,43 +34,220 @@ extern "C" {
#define ONE_STAGE_COMP 1
#define TWO_STAGE_COMP 2
int tsCompressTinyint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorithm,
char* const buffer, int bufferSize);
int tsCompressSmallint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
int tsCompressInt(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
int tsCompressBigint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
int tsCompressBool(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorithm,
char* const buffer, int bufferSize);
int tsCompressString(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
int tsCompressFloat(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
int tsCompressDouble(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
int tsCompressTimestamp(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
char* const buffer, int bufferSize);
extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
extern int tsDecompressBoolImp(const char *const input, const int nelements, char *const output);
extern int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize);
extern int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize);
extern int tsCompressTimestampImp(const char *const input, const int nelements, char *const output);
extern int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output);
extern int tsCompressDoubleImp(const char *const input, const int nelements, char *const output);
extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
int tsDecompressTinyint(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
int tsDecompressSmallint(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
int tsDecompressInt(const char* const input, int compressedSize, const int nelements, char* const output, int outputSize,
char algorithm, char* const buffer, int bufferSize);
int tsDecompressBigint(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
int tsDecompressBool(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
int tsDecompressString(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
int tsDecompressFloat(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
int tsDecompressDouble(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorith, char* const buffer, int bufferSize);
int tsDecompressTimestamp(const char* const input, int compressedSize, const int nelements, char* const output,
int outputSize, char algorithm, char* const buffer, int bufferSize);
static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressBoolImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressBoolImp(buffer, nelements, output);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
return tsCompressStringImp(input, inputSize, output, outputSize);
}
static FORCE_INLINE int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
return tsDecompressStringImp(input, compressedSize, output, outputSize);
}
static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressFloatImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressFloatImp(buffer, nelements, output);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressDoubleImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressDoubleImp(buffer, nelements, output);
} else {
assert(0);
}
}
static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressTimestampImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressTimestampImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressTimestampImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressTimestampImp(buffer, nelements, output);
} else {
assert(0);
}
}
#ifdef __cplusplus
}

43
src/util/inc/tsocket.h
View File

@ -20,44 +20,23 @@
extern "C" {
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
int taosNonblockwrite(int fd, char *ptr, int nbytes);
int taosReadn(int sock, char *buffer, int len);
int taosWriteMsg(int fd, void *ptr, int nbytes);
int taosReadMsg(int fd, void *ptr, int nbytes);
int taosNonblockwrite(int fd, char *ptr, int nbytes);
int taosCopyFds(int sfd, int dfd, int64_t len);
int taosSetNonblocking(int sock, int on);
int taosOpenUdpSocket(char *ip, uint16_t port);
int taosOpenTcpClientSocket(char *ip, uint16_t port, char *localIp);
int taosOpenTcpServerSocket(char *ip, uint16_t port);
int taosKeepTcpAlive(int sockFd);
int taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
int taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
int taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int taosKeepTcpAlive(int sockFd);
void taosCloseTcpSocket(int sockFd);
int taosOpenUDServerSocket(char *ip, uint16_t port);
int taosOpenUDClientSocket(char *ip, uint16_t port);
int taosOpenRawSocket(char *ip);
int taosCopyFds(int sfd, int dfd, int64_t len);
int taosGetPublicIp(char *const ip);
int taosGetPrivateIp(char *const ip);
void tinet_ntoa(char *ipstr, unsigned int ip);
int taosSetNonblocking(int sock, int on);
int taosGetFqdn(char *);
uint32_t taosGetIpFromFqdn(const char *);
void tinet_ntoa(char *ipstr, unsigned int ip);
uint32_t ip2uint(const char *const ip_addr);
#ifdef __cplusplus
}

5
src/util/inc/tutil.h
View File

@ -44,7 +44,10 @@ extern "C" {
#define tclose(x) taosCloseSocket(x)
#ifdef ASSERTION
// Pointer p drift right by b bytes
#define POINTER_DRIFT(p, b) ((void *)((char *)(p) + (b)))
#ifndef NDEBUG
#define ASSERT(x) assert(x)
#else
#define ASSERT(x)

3
src/util/src/hash.c
View File

@ -521,6 +521,7 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
static SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
assert(pIter != NULL);
pIter->entryIndex++;
while (pIter->entryIndex < pIter->pHashObj->capacity) {
SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
if (pEntry->next == NULL) {
@ -540,7 +541,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) {
}
size_t size = taosHashGetSize(pIter->pHashObj);
if (size == 0 || pIter->num >= size) {
if (size == 0) {
return false;
}

217
src/util/src/tcompression.c
View File

@ -56,223 +56,6 @@ const int TEST_NUMBER = 1;
#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0)
#define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951L)
// Function declarations
int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
int tsDecompressBoolImp(const char *const input, const int nelements, char *const output);
int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize);
int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize);
int tsCompressTimestampImp(const char *const input, const int nelements, char *const output);
int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output);
int tsCompressDoubleImp(const char *const input, const int nelements, char *const output);
int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
/* ----------------------------------------------Compression function used by
* others ---------------------------------------------- */
int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else {
assert(0);
}
}
int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else {
assert(0);
}
}
int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
} else {
assert(0);
}
}
int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else {
assert(0);
}
}
int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressBoolImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressBoolImp(buffer, nelements, output);
} else {
assert(0);
}
}
int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
return tsCompressStringImp(input, inputSize, output, outputSize);
}
int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
return tsDecompressStringImp(input, compressedSize, output, outputSize);
}
int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressFloatImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressFloatImp(buffer, nelements, output);
} else {
assert(0);
}
}
int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressDoubleImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressDoubleImp(buffer, nelements, output);
} else {
assert(0);
}
}
int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressTimestampImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int len = tsCompressTimestampImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
}
}
int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output,
int outputSize, char algorithm, char *const buffer, int bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressTimestampImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
return tsDecompressTimestampImp(buffer, nelements, output);
} else {
assert(0);
}
}
bool safeInt64Add(int64_t a, int64_t b) {
if ((a > 0 && b > INT64_MAX - a) || (a < 0 && b < INT64_MIN - a)) return false;
return true;

2
src/util/src/tconfig.c
View File

@ -249,7 +249,7 @@ void taosReadGlobalLogCfg() {
}
wordfree(&full_path);
taosReadLogOption("logDir", logDir);
taosReadLogOption("tsLogDir", tsLogDir);
sprintf(fileName, "%s/taos.cfg", configDir);
fp = fopen(fileName, "r");

8
src/util/src/tlog.c
View File

@ -66,7 +66,7 @@ int32_t tsAsyncLog = 1;
float tsTotalLogDirGB = 0;
float tsAvailLogDirGB = 0;
float tsMinimalLogDirGB = 0.1;
char logDir[TSDB_FILENAME_LEN] = "/var/log/taos";
char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos";
static SLogObj tsLogObj = { .fileNum = 1 };
static void * taosAsyncOutputLog(void *param);
@ -298,7 +298,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
void taosPrintLog(const char *const flags, int32_t dflag, const char *const format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB);
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
return;
}
@ -356,7 +356,7 @@ void taosPrintLog(const char *const flags, int32_t dflag, const char *const form
void taosDumpData(unsigned char *msg, int32_t len) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB);
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
return;
}
@ -385,7 +385,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
void taosPrintLongString(const char *const flags, int32_t dflag, const char *const format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB);
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB);
fflush(stdout);
return;
}

4
src/util/src/tnote.c
View File

@ -27,10 +27,10 @@ void taosInitNote(int numOfNoteLines, int maxNotes, char* lable)
if (strcasecmp(lable, "http_note") == 0) {
pNote = &m_HttpNote;
sprintf(temp, "%s/httpnote", logDir);
sprintf(temp, "%s/httpnote", tsLogDir);
} else if (strcasecmp(lable, "tsc_note") == 0) {
pNote = &m_TscNote;
sprintf(temp, "%s/tscnote-%d", logDir, getpid());
sprintf(temp, "%s/tscnote-%d", tsLogDir, getpid());
} else {
return;
}

154
src/util/src/tsocket.c
View File

@ -14,92 +14,30 @@
*/
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
#include "tsocket.h"
#include "tutil.h"
/*
* Function to get the public ip address of current machine. If get IP
* successfully, return 0, else, return -1. The return values is ip.
*
* Use:
* if (taosGetPublicIp(ip) != 0) {
* perror("Fail to get public IP address\n");
* exit(EXIT_FAILURE);
* }
*/
int taosGetPublicIp(char *const ip) {
/* bool flag; */
int flag;
int sock;
char ** pptr = NULL;
struct sockaddr_in destAddr;
struct hostent * ptr = NULL;
char destIP[128];
char szBuffer[] = {
"GET / HTTP/1.1\nHost: ident.me\nUser-Agent: curl/7.47.0\nAccept: "
"*/*\n\n"};
char res[1024];
int taosGetFqdn(char *fqdn) {
char hostname[1024];
hostname[1023] = '\0';
gethostname(hostname, 1023);
// Create socket
sock = (int)socket(AF_INET, SOCK_STREAM, 0);
if (sock == -1) {
struct hostent* h;
h = gethostbyname(hostname);
if (h != NULL) {
strcpy(fqdn, h->h_name);
return 0;
} else {
uError("failed to get host name");
return -1;
}
}
bzero((void *)&destAddr, sizeof(destAddr));
destAddr.sin_family = AF_INET;
destAddr.sin_port = htons(80);
ptr = gethostbyname("ident.me");
if (ptr == NULL) {
return -1;
}
// Loop to find a valid IP address
for (flag = 0, pptr = ptr->h_addr_list; NULL != *pptr; ++pptr) {
inet_ntop(ptr->h_addrtype, *pptr, destIP, sizeof(destIP));
destAddr.sin_addr.s_addr = inet_addr(destIP);
if (connect(sock, (struct sockaddr *)&destAddr, sizeof(struct sockaddr)) != -1) {
flag = 1;
break;
}
}
// Check if the host is available.
if (flag == 0) {
return -1;
}
// Check send.
if (strlen(szBuffer) != taosWriteSocket(sock, szBuffer, (size_t)strlen(szBuffer))) {
return -1;
}
// Receive response.
if (taosReadSocket(sock, res, 1024) == -1) {
return -1;
}
// Extract the IP address from the response.
int c_start = 0, c_end = 0;
for (; c_start < (int)strlen(res); c_start = c_end + 1) {
for (c_end = c_start; c_end < (int)strlen(res) && res[c_end] != '\n'; c_end++) {
}
if (c_end >= (int)strlen(res)) {
return -1;
}
if (res[c_start] >= '0' && res[c_start] <= '9') {
strncpy(ip, res + c_start, (size_t)(c_end - c_start));
ip[c_end - c_start] = '\0';
break;
}
}
return 0;
uint32_t taosGetIpFromFqdn(const char *fqdn) {
struct hostent * record = gethostbyname(fqdn);
if(record == NULL) return -1;
return ((struct in_addr *)record->h_addr)->s_addr;
}
// Function converting an IP address string to an unsigned int.
@ -259,7 +197,7 @@ int taosReadn(int fd, char *ptr, int nbytes) {
return (nbytes - nleft);
}
int taosOpenUdpSocket(char *ip, uint16_t port) {
int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in localAddr;
int sockFd;
int ttl = 128;
@ -270,7 +208,7 @@ int taosOpenUdpSocket(char *ip, uint16_t port) {
memset((char *)&localAddr, 0, sizeof(localAddr));
localAddr.sin_family = AF_INET;
localAddr.sin_addr.s_addr = inet_addr(ip);
localAddr.sin_addr.s_addr = ip;
localAddr.sin_port = (uint16_t)htons(port);
if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
@ -325,13 +263,11 @@ int taosOpenUdpSocket(char *ip, uint16_t port) {
return sockFd;
}
int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) {
int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) {
int sockFd = 0;
struct sockaddr_in serverAddr, clientAddr;
int ret;
// uTrace("open tcp client socket:%s:%d, local Ip:%s", destIp, destPort, clientIp);
sockFd = (int)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sockFd < 0) {
@ -339,16 +275,16 @@ int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) {
return -1;
}
if (clientIp && clientIp[0] && clientIp[0] != '0') {
if ( clientIp != 0) {
memset((char *)&clientAddr, 0, sizeof(clientAddr));
clientAddr.sin_family = AF_INET;
clientAddr.sin_addr.s_addr = inet_addr(clientIp);
clientAddr.sin_addr.s_addr = clientIp;
clientAddr.sin_port = 0;
/* bind socket to client address */
if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) {
uError("bind tcp client socket failed, client(%s:0), dest(%s:%d), reason:%d(%s)",
clientIp, destIp, destPort, errno, strerror(errno));
uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)",
clientIp, destIp, destPort, strerror(errno));
close(sockFd);
return -1;
}
@ -356,13 +292,13 @@ int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) {
memset((char *)&serverAddr, 0, sizeof(serverAddr));
serverAddr.sin_family = AF_INET;
serverAddr.sin_addr.s_addr = inet_addr(destIp);
serverAddr.sin_addr.s_addr = destIp;
serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort);
ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
if (ret != 0) {
//uError("failed to connect socket, ip:%s, port:%hu, reason: %s", destIp, destPort, strerror(errno));
//uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
taosCloseSocket(sockFd);
sockFd = -1;
}
@ -420,7 +356,7 @@ int taosKeepTcpAlive(int sockFd) {
return 0;
}
int taosOpenTcpServerSocket(char *ip, uint16_t port) {
int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
struct sockaddr_in serverAdd;
int sockFd;
int reuse;
@ -429,7 +365,7 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) {
bzero((char *)&serverAdd, sizeof(serverAdd));
serverAdd.sin_family = AF_INET;
serverAdd.sin_addr.s_addr = inet_addr(ip);
serverAdd.sin_addr.s_addr = ip;
serverAdd.sin_port = (uint16_t)htons(port);
if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
@ -447,7 +383,7 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) {
/* bind socket to server address */
if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
uError("bind tcp server socket failed, %s:%hu, reason:%d(%s)", ip, port, errno, strerror(errno));
uError("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
close(sockFd);
return -1;
}
@ -455,45 +391,13 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) {
if (taosKeepTcpAlive(sockFd) < 0) return -1;
if (listen(sockFd, 10) < 0) {
uError("listen tcp server socket failed, %s:%hu, reason:%d(%s)", ip, port, errno, strerror(errno));
uError("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
return -1;
}
return sockFd;
}
int taosOpenRawSocket(char *ip) {
int fd, hold;
struct sockaddr_in rawAdd;
uTrace("open udp raw socket:%s", ip);
fd = (int)socket(AF_INET, SOCK_RAW, IPPROTO_UDP);
if (fd < 0) {
uError("failed to open raw socket: %d (%s)", errno, strerror(errno));
return -1;
}
hold = 1;
if (taosSetSockOpt(fd, IPPROTO_IP, IP_HDRINCL, (void *)&hold, sizeof(hold)) < 0) {
uError("failed to set hold option: %d (%s)", errno, strerror(errno));
close(fd);
return -1;
}
bzero((char *)&rawAdd, sizeof(rawAdd));
rawAdd.sin_family = AF_INET;
rawAdd.sin_addr.s_addr = inet_addr(ip);
if (bind(fd, (struct sockaddr *)&rawAdd, sizeof(rawAdd)) < 0) {
uError("failed to bind RAW socket: %d (%s)", errno, strerror(errno));
close(fd);
return -1;
}
return fd;
}
void tinet_ntoa(char *ipstr, unsigned int ip) {
sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
}

2
src/util/src/tutil.c
View File

@ -686,4 +686,4 @@ void taosRemoveDir(char *rootDir) {
rmdir(rootDir);
uPrint("dir:%s is removed", rootDir);
}
}

74
src/vnode/src/vnodeMain.c
View File

@ -96,14 +96,16 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
}
STsdbCfg tsdbCfg = {0};
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression;;
tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId;
tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize;
tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks;
tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables;
tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile;
tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep;
tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep;
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression;;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
@ -311,7 +313,7 @@ void *vnodeGetWqueue(int32_t vgId) {
SVnodeObj *pVnode = vnodeAccquireVnode(vgId);
if (pVnode == NULL) return NULL;
return pVnode->wqueue;
}
}
void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
@ -341,10 +343,13 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) {
}
static void vnodeCleanUp(SVnodeObj *pVnode) {
taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
//syncStop(pVnode->sync);
if (pVnode->sync) {
syncStop(pVnode->sync);
pVnode->sync = NULL;
}
tsdbCloseRepo(pVnode->tsdb);
walClose(pVnode->wal);
vnodeSaveVersion(pVnode);
@ -379,11 +384,11 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId);
FILE *fp = fopen(cfgFile, "w");
if (!fp) {
dError("vgId:%d, failed to open vnode cfg file for write, error:%s", pVnodeCfg->cfg.vgId, strerror(errno));
dError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile,
strerror(errno));
return errno;
}
char ipStr[20];
int32_t len = 0;
int32_t maxLen = 1000;
char * content = calloc(1, maxLen + 1);
@ -408,19 +413,10 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals);
len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum);
uint32_t ipInt = pVnodeCfg->cfg.arbitratorIp;
sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24));
len += snprintf(content + len, maxLen - len, " \"arbitratorIp\": \"%s\",\n", ipStr);
len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n");
for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) {
len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId);
uint32_t ipInt = pVnodeCfg->nodes[i].nodeIp;
sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24));
len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", ipStr);
len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", pVnodeCfg->nodes[i].nodeName);
len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp);
if (i < pVnodeCfg->cfg.replications - 1) {
len += snprintf(content + len, maxLen - len, " },{\n");
@ -444,7 +440,8 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId);
FILE *fp = fopen(cfgFile, "r");
if (!fp) {
dError("pVnode:%p vgId:%d, failed to open vnode cfg file for read, error:%s", pVnode, pVnode->vgId, strerror(errno));
dError("pVnode:%p vgId:%d, failed to open vnode cfg file for read, file:%s, error:%s", pVnode, pVnode->vgId,
cfgFile, strerror(errno));
return errno;
}
@ -585,13 +582,6 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
pVnode->syncCfg.quorum = (int8_t)quorum->valueint;
cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp");
if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring);
cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode, pVnode->vgId);
@ -615,27 +605,22 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
}
pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint;
cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp");
if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeIp not found", pVnode, pVnode->vgId);
cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
pVnode->syncCfg.nodeInfo[i].nodeIp = inet_addr(nodeIp->valuestring);
cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName");
if (!nodeName || nodeName->type != cJSON_String || nodeName->valuestring == NULL) {
dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeName not found", pVnode, pVnode->vgId);
goto PARSE_OVER;
}
strncpy(pVnode->syncCfg.nodeInfo[i].name, nodeName->valuestring, TSDB_NODE_NAME_LEN);
taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort);
pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC;
}
ret = 0;
dPrint("pVnode:%p vgId:%d, read vnode cfg successed, replcia:%d", pVnode, pVnode->vgId, pVnode->syncCfg.replica);
for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) {
dPrint("pVnode:%p vgId:%d, dnode:%d, ip:%s name:%s", pVnode, pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
taosIpStr(pVnode->syncCfg.nodeInfo[i].nodeIp), pVnode->syncCfg.nodeInfo[i].name);
dPrint("pVnode:%p vgId:%d, dnode:%d, %s:%d", pVnode, pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId,
pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort);
}
PARSE_OVER:
@ -645,13 +630,13 @@ PARSE_OVER:
return ret;
}
static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
char versionFile[TSDB_FILENAME_LEN + 30] = {0};
sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
FILE *fp = fopen(versionFile, "w");
if (!fp) {
dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno));
dError("pVnode:%p vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode, pVnode->vgId,
versionFile, strerror(errno));
return errno;
}
@ -667,7 +652,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
fclose(fp);
free(content);
dPrint("pVnode:%p vgId:%d, save vnode version successed", pVnode, pVnode->vgId);
dPrint("pVnode:%p vgId:%d, save vnode version:%" PRId64 " successed", pVnode, pVnode->vgId, pVnode->version);
return 0;
}
@ -675,9 +660,10 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
static bool vnodeReadVersion(SVnodeObj *pVnode) {
char versionFile[TSDB_FILENAME_LEN + 30] = {0};
sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
FILE *fp = fopen(versionFile, "w");
FILE *fp = fopen(versionFile, "r");
if (!fp) {
dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno));
dTrace("pVnode:%p vgId:%d, failed to open version file:%s error:%s", pVnode, pVnode->vgId,
versionFile, strerror(errno));
return false;
}
@ -714,4 +700,4 @@ PARSE_OVER:
cJSON_Delete(root);
fclose(fp);
return ret;
}
}

14
src/vnode/src/vnodeWrite.c
View File

@ -123,7 +123,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
STSchema *pDestSchema = tdNewSchema(numOfColumns);
for (int i = 0; i < numOfColumns; i++) {
tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
tdSchemaAddCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
}
tsdbTableSetSchema(&tCfg, pDestSchema, false);
tsdbTableSetName(&tCfg, pTable->tableId, false);
@ -131,7 +131,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
if (numOfTags != 0) {
STSchema *pDestTagSchema = tdNewSchema(numOfTags);
for (int i = numOfColumns; i < totalCols; i++) {
tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
tdSchemaAddCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
}
tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
tsdbTableSetSName(&tCfg, pTable->superTableId, false);
@ -141,7 +141,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
for (int i = 0; i < numOfTags; i++) {
tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
STColumn *pTCol = schemaColAt(pDestSchema, i);
tdAppendColVal(dataRow, pTagData + accumBytes, pTCol->type, pTCol->bytes, pTCol->offset);
accumBytes += htons(pSchema[i + numOfColumns].bytes);
}
tsdbTableSetTagValue(&tCfg, dataRow, false);
@ -188,14 +189,14 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
STSchema *pDestSchema = tdNewSchema(numOfColumns);
for (int i = 0; i < numOfColumns; i++) {
tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
tdSchemaAddCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
}
tsdbTableSetSchema(&tCfg, pDestSchema, false);
if (numOfTags != 0) {
STSchema *pDestTagSchema = tdNewSchema(numOfTags);
for (int i = numOfColumns; i < totalCols; i++) {
tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
tdSchemaAddCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
}
tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
@ -204,7 +205,8 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
for (int i = 0; i < numOfTags; i++) {
tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
STColumn *pTCol = schemaColAt(pDestTagSchema, i);
tdAppendColVal(dataRow, pTagData + accumBytes, pTCol->type, pTCol->bytes, pTCol->offset);
accumBytes += htons(pSchema[i + numOfColumns].bytes);
}
tsdbTableSetTagValue(&tCfg, dataRow, false);

11
tests/pytest/smoketest.sh
View File

@ -23,6 +23,17 @@ sleep 1
python3 ./test.py $1 -f insert/tinyint.py
python3 ./test.py -s $1
sleep 1
python3 ./test.py $1 -f table/column_name.py
python3 ./test.py -s $1
sleep 1
python3 ./test.py $1 -f table/column_num.py
python3 ./test.py -s $1
sleep 1
python3 ./test.py $1 -f table/db_table.py
python3 ./test.py -s $1
sleep 1
python3 ./test.py $1 -f import_merge/importDataLastTO.py
python3 ./test.py -s $1
sleep 1

181
tests/pytest/table/column_name.py Normal file
View File

@ -0,0 +1,181 @@
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
tdSql.prepare()
# TSIM: system sh/stop_dnodes.sh
# TSIM:
# TSIM: system sh/ip.sh -i 1 -s up
# TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
# TSIM: system sh/cfg.sh -n dnode1 -c commitLog -v 0
# TSIM: system sh/exec.sh -n dnode1 -s start
# TSIM:
# TSIM: sleep 3000
# TSIM: sql connect
# TSIM:
# TSIM: $i = 0
# TSIM: $dbPrefix = lm_cm_db
# TSIM: $tbPrefix = lm_cm_tb
# TSIM: $db = $dbPrefix . $i
# TSIM: $tb = $tbPrefix . $i
# TSIM:
# TSIM: print =============== step1
tdLog.info('=============== step1')
# TSIM: sql create database $db
# TSIM: sql use $db
# TSIM:
# TSIM: sql drop table dd -x step0
tdLog.info('drop table dd -x step0')
tdSql.error('drop table dd')
# TSIM: return -1
# TSIM: step0:
# TSIM:
# TSIM: sql create table $tb(ts timestamp, int) -x step1
tdLog.info('create table tb(ts timestamp, int) -x step1')
tdSql.error('create table tb(ts timestamp, int)')
# TSIM: return -1
# TSIM: step1:
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step2
tdLog.info('=============== step2')
# TSIM: sql create table $tb (ts timestamp, s int)
tdLog.info('create table tb (ts timestamp, s int)')
tdSql.execute('create table tb (ts timestamp, s int)')
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql drop table $tb
tdLog.info('drop table tb')
tdSql.execute('drop table tb')
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step3
tdLog.info('=============== step3')
# TSIM: sql create table $tb (ts timestamp, a0123456789 int)
tdLog.info('create table tb (ts timestamp, a0123456789 int)')
tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql drop table $tb
tdLog.info('drop table tb')
tdSql.execute('drop table tb')
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step4
tdLog.info('=============== step4')
# TSIM: sql create table $tb (ts timestamp,
# a0123456789012345678901234567890123456789 int)
tdLog.info(
'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
tdSql.execute(
'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
# TSIM: sql drop table $tb
tdLog.info('drop table tb')
tdSql.execute('drop table tb')
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step5
tdLog.info('=============== step5')
# TSIM: sql create table $tb (ts timestamp, a0123456789 int)
tdLog.info('create table tb (ts timestamp, a0123456789 int)')
tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql insert into $tb values (now , 1)
tdLog.info("insert into tb values (now , 1)")
tdSql.execute("insert into tb values (now , 1)")
# TSIM: sql select * from $tb
tdLog.info('select * from tb')
tdSql.query('select * from tb')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql drop database $db
tdLog.info('drop database db')
tdSql.execute('drop database db')
# TSIM: sql show databases
tdLog.info('show databases')
tdSql.query('show databases')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM:
# TSIM:
# TSIM:
# convert end
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

178
tests/pytest/table/column_num.py Normal file
View File

@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
tdSql.prepare()
# TSIM: system sh/stop_dnodes.sh
# TSIM:
# TSIM: system sh/ip.sh -i 1 -s up
# TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
# TSIM: system sh/cfg.sh -n dnode1 -c commitLog -v 0
# TSIM: system sh/exec.sh -n dnode1 -s start
# TSIM:
# TSIM: sleep 3000
# TSIM: sql connect
# TSIM:
# TSIM: $i = 0
# TSIM: $dbPrefix = lm_cn_db
# TSIM: $tbPrefix = lm_cn_tb
# TSIM: $db = $dbPrefix . $i
# TSIM: $tb = $tbPrefix . $i
# TSIM:
# TSIM: print =============== step1
tdLog.info('=============== step1')
# TSIM: sql create database $db
# TSIM: sql use $db
# TSIM:
# TSIM: sql create table $tb() -x step1
tdLog.info('create table tb() -x step1')
tdSql.error('create table tb()')
# TSIM: return -1
# TSIM: step1:
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step2
tdLog.info('=============== step2')
# TSIM: sql create table $tb (ts timestamp) -x step2
tdLog.info('create table tb (ts timestamp) -x step2')
tdSql.error('create table tb (ts timestamp) ')
# TSIM: return -1
# TSIM: step2:
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step3
tdLog.info('=============== step3')
# TSIM: sql create table $tb (ts int) -x step3
tdLog.info('create table tb (ts int) -x step3')
tdSql.error('create table tb (ts int) ')
# TSIM: return -1
# TSIM: step3:
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step4
tdLog.info('=============== step4')
# TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4
# int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12
# int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int,
# a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27
# int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int,
# b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12
# int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int,
# b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27
# int, b28 int,b29 int,b30 int,b31 int,b32 int)
tdLog.info('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)')
tdSql.execute('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)')
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: print =============== step5
tdLog.info('=============== step5')
# TSIM: $i = 1
# TSIM: $tb = $tbPrefix . $i
# TSIM:
# TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4
# int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12
# int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int,
# a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27
# int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int,
# b4 int, b5 int)
tdLog.info('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)')
tdSql.execute('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)')
# TSIM:
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 2 then
tdLog.info('tdSql.checkRow(2)')
tdSql.checkRows(2)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql insert into $tb values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8
# , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22
# , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)
tdLog.info("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)")
tdSql.execute("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)")
# TSIM: sql select * from $tb
tdLog.info('select * from tb1')
tdSql.query('select * from tb1')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql drop table $tb
tdLog.info('drop table tb1')
tdSql.execute('drop table tb1')
# TSIM: sql show tables
tdLog.info('show tables')
tdSql.query('show tables')
# TSIM: if $rows != 1 then
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
# TSIM: return -1
# TSIM: endi
# TSIM:
# TSIM: sql drop database $db
tdLog.info('drop database db')
tdSql.execute('drop database db')
# TSIM: sql show databases
tdLog.info('show databases')
tdSql.query('show databases')
# TSIM: if $rows != 0 then
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# TSIM: return -1
# TSIM: endi
# convert end
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

49
tests/pytest/table/db_table.py Normal file
View File

@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
tdSql.prepare()
tdLog.info('=============== step1')
tdLog.info('=============== step2')
tdLog.info('create table tb (ts timestamp, speed int)')
tdSql.execute('create table tb (ts timestamp, speed int)')
tdLog.info('=============== step3')
tdLog.info("insert into tb values (now, 1)")
tdSql.execute("insert into tb values (now, 1)")
tdLog.info('=============== step4')
tdLog.info('select * from tb')
tdSql.query('select * from tb')
tdLog.info('tdSql.checkRow(1)')
tdSql.checkRows(1)
tdLog.info('tdSql.checkData(0, 1, 1)')
tdSql.checkData(0, 1, 1)
tdLog.info('=============== step5')
tdLog.info('describe tb')
tdSql.query('describe tb')
tdLog.info('=============== step6')
tdLog.info('drop database db')
tdSql.execute('drop database db')
tdLog.info('show databases')
tdSql.query('show databases')
tdLog.info('tdSql.checkRow(0)')
tdSql.checkRows(0)
# convert end
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

8
tests/pytest/util/log.py
View File

@ -23,14 +23,14 @@ class TDLog:
self.path = ""
def info(self, info):
printf("%s %s" % (datetime.datetime.now(), info))
print("%s %s" % (datetime.datetime.now(), info))
def sleep(self, sec):
printf("%s sleep %d seconds" % (datetime.datetime.now(), sec))
print("%s sleep %d seconds" % (datetime.datetime.now(), sec))
time.sleep(sec)
def debug(self, err):
printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
def success(self, info):
printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info))
@ -43,7 +43,7 @@ class TDLog:
sys.exit(1)
def printNoPrefix(self, info):
printf("\033[1;36m%s\033[0m" % (info))
print("\033[1;36m%s\033[0m" % (info))
tdLog = TDLog()

28
tests/script/basicSuite.sim
View File

@ -1,8 +1,28 @@
#################################
run general/db/basic1.sim
run general/db/basic2.sim
run general/db/basic3.sim
run general/db/basic4.sim
run general/db/basic5.sim
run general/db/testSuite.sim
run general/insert/testSuite.sim
run general/table/testSuite.sim
run general/user/basicSuite.sim
run general/table/basic1.sim
run general/table/basic2.sim
run general/table/basic3.sim
run general/table/column_num.sim
run general/table/column_name.sim
run general/table/bigint.sim
run general/table/bool.sim
run general/table/double.sim
run general/table/float.sim
run general/table/int.sim
run general/table/smallint.sim
run general/table/tinyint.sim
run general/table/db.table.sim
run general/user/basic1.sim
run general/user/pass_alter.sim
run general/user/pass_len.sim
run general/user/user_create.sim
run general/user/user_len.sim
##################################

4
tests/script/general/agg/fill.sim
View File

@ -1,6 +1,6 @@
system sh/stop_dnodes.sh
system sh/ip.sh -i 1 -s up
system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c commitLog -v 0
system sh/exec.sh -n dnode1 -s start

4
tests/script/general/agg/stream.sim
View File

@ -1,7 +1,7 @@
system sh/stop_dnodes.sh
system sh/ip.sh -i 1 -s up
system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c clog -v 0
print ========== step1

4
tests/script/general/alter/cached_schema_after_alter.sim
View File

@ -1,6 +1,6 @@
system sh/stop_dnodes.sh
system sh/ip.sh -i 1 -s up
system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c clog -v 0
system sh/exec.sh -n dnode1 -s start
sleep 3000

4
tests/script/general/alter/count.sim
View File

@ -1,6 +1,6 @@
system sh/stop_dnodes.sh
system sh/ip.sh -i 1 -s up
system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c clog -v 0
system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1

4
tests/script/general/alter/import.sim
View File

@ -1,6 +1,6 @@
system sh/stop_dnodes.sh
system sh/ip.sh -i 1 -s up
system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c clog -v 0
system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1

Some files were not shown because too many files have changed in this diff Show More