diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index 44b3ad4671..79388a2edb 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -87,6 +87,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求。 - dataDir: 数据文件目录，所有的数据文件都将写入该目录。默认值：/var/lib/taos。 - logDir：日志文件目录，客户端和服务器的运行日志文件将写入该目录。默认值：/var/log/taos。 +- tempDir：临时文件目录，客户端和服务器的临时文件（主要是查询时用于保存中间结果的问题）将写入该目录。 默认值：Linux下为 /tmp/，Windows下为环境变量 tmp 或 temp 指向的目录。 - arbitrator：系统中裁决器的end point, 缺省值为空。 - role：dnode的可选角色。0-any; 既可作为mnode，也可分配vnode；1-mgmt;只能作为mnode，不能分配vnode；2-dnode;不能作为mnode，只能分配vnode - debugFlag：运行日志开关。131（输出错误和警告日志），135（ 输出错误、警告和调试日志），143（ 输出错误、警告、调试和跟踪日志）。默认值：131或135（不同模块有不同的默认值）。 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 4b274e05e6..1e36e6c5e6 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable ``` -示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44851T。TDengine大概需要消耗44851/5=8970T, 8.9P空间。 +示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。 diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index 0e29b32487..c5a955f43f 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -616,6 +616,43 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 - httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 - httpDebugFlag: 日志开关，131：仅错误和报警信息，135：调试信息，143：非常详细的调试信息，默认131 +## CSharp Connector + +在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。 + +#### 安装TDengine客户端 + +C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。 + +安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件 + +- TDengineDriver.cs 调用taos.dll文件的Native C方法 +- TDengineTest.cs 参考程序示例 + +在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件 + +#### 使用方法 + +- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中 +- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法 +- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中 + +#### 注意事项 + +- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。 +- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。 + +#### 第三方驱动 + +Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考 + +``` +//接口下载 +https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos +//用法说明 +https://www.taosdata.com/blog/2020/11/02/1901.html +``` + ## Go Connector diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 80deb889ef..757b6d9929 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -38,9 +38,9 @@ 6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的 -7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 +7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 -8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) +8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) 9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 974b2b05c1..ca88bca3c8 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -20,6 +20,9 @@ # data file's directory # dataDir /var/lib/taos +# temporary file's directory +# tempDir /tmp/ + # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only # arbitrator arbitrator_hostname:6042 @@ -256,3 +259,5 @@ # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden # maxBinaryDisplayWidth 30 +# enable/disable telemetry reporting +# telemetryReporting 1 \ No newline at end of file diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index e49baeffc7..54c3c9b279 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -2,7 +2,7 @@ %define cfg_install_dir /etc/taos %define __strip /bin/true -Name: TDengine +Name: tdengine Version: %{_version} Release: 3%{?dist} Summary: tdengine from taosdata diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index d91daaa5c4..569f316ff3 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -121,8 +121,11 @@ function install_config() { echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}" echo echo -e -n "${GREEN}OR leave it blank to build one${NC}:" - read firstEp - while true; do + #read firstEp + if exec < /dev/tty; then + read firstEp; + fi + while true; do if [ ! -z "$firstEp" ]; then # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 0e9bb85b25..4c687cb134 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -490,7 +490,7 @@ static bool balanceMontiorDropping() { if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue; - if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; + if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; if (mnodeGetDnodesNum() <= 1) continue; mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId, diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0d57dd18c3..6628e30874 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -87,8 +87,8 @@ typedef struct SJoinSupporter { } SJoinSupporter; typedef struct SVgroupTableInfo { - SCMVgroupInfo vgInfo; - SArray* itemList; //SArray + SVgroupInfo vgInfo; + SArray* itemList; //SArray } SVgroupTableInfo; static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { @@ -239,7 +239,7 @@ void tscDoQuery(SSqlObj* pSql); SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo); void* tscVgroupInfoClear(SVgroupsInfo *pInfo); -void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src); +void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src); /** * The create object function must be successful expect for the out of memory issue. * diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bcc9f8354d..1c2821638b 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -90,12 +90,12 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; -typedef struct SCMCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCMCorVgroupInfo; +typedef struct SCorVgroupInfo { + int32_t version; + int8_t inUse; + int8_t numOfEps; + SEpAddr1 epAddr[TSDB_MAX_REPLICA]; +} SCorVgroupInfo; typedef struct STableMeta { STableComInfo tableInfo; @@ -103,8 +103,8 @@ typedef struct STableMeta { int16_t sversion; int16_t tversion; char sTableId[TSDB_TABLE_FNAME_LEN]; - SCMVgroupInfo vgroupInfo; - SCMCorVgroupInfo corVgroupInfo; + SVgroupInfo vgroupInfo; + SCorVgroupInfo corVgroupInfo; STableId id; SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ac34d26d2f..5e65126ef6 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st } int32_t code = TSDB_CODE_TSC_INVALID_SQL; - char * tmpTokenBuf = calloc(1, 4096); // used for deleting Escape character: \\, \', \" + char * tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" if (NULL == tmpTokenBuf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index eb6843b0e4..acc5acd786 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -222,7 +222,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { } int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { - SCMHeartBeatMsg *pHeartbeat = pMsg; + SHeartBeatMsg *pHeartbeat = pMsg; int allocedQueriesNum = pHeartbeat->numOfQueries; int allocedStreamsNum = pHeartbeat->numOfStreams; @@ -277,7 +277,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { } int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) + - sizeof(SCMHeartBeatMsg); + sizeof(SHeartBeatMsg); pHeartbeat->connId = htonl(pObj->connId); pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries); pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ae5eb57bdf..3e7a64c828 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -113,7 +113,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString); static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); +static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); @@ -416,7 +416,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { char* pMsg = pCmd->payload; - SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg; + SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg; pDCL->a[0].n = strdequote(pDCL->a[0].z); strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n); @@ -618,6 +618,7 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; + const char* msg3 = "sliding cannot be used without interval"; SSqlCmd* pCmd = &pSql->cmd; @@ -625,6 +626,9 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) { + if (pQuerySql->sliding.n > 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } return TSDB_CODE_SUCCESS; } @@ -4471,8 +4475,8 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) { const char* msg0 = "only support order by primary timestamp"; const char* msg1 = "invalid column name"; - const char* msg2 = "only support order by primary timestamp and queried column"; - const char* msg3 = "only support order by primary timestamp and first tag in groupby clause"; + const char* msg2 = "only support order by primary timestamp or queried column"; + const char* msg3 = "only support order by primary timestamp or first tag in groupby clause"; setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -5233,7 +5237,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn return TSDB_CODE_SUCCESS; } -static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { const char* msg = "invalid number of options"; pMsg->daysToKeep = htonl(-1); @@ -5264,7 +5268,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* return TSDB_CODE_SUCCESS; } -static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { +static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { const char* msg = "invalid time precision"; pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default @@ -5288,7 +5292,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn return TSDB_CODE_SUCCESS; } -static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->maxTables = htonl(-1); // max tables can not be set anymore pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); @@ -5305,7 +5309,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { - SCMCreateDbMsg* pMsg = (SCMCreateDbMsg*)(pCmd->payload); + SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload); setCreateDBOption(pMsg, pCreateDbSql); if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { @@ -5823,7 +5827,7 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // can only perform the parameters based on the macro definitation -int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { +int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { char msg[512] = {0}; if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index ea493d2e25..441ad8d838 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -140,7 +140,7 @@ struct SSchema tscGetTbnameColumnSchema() { strcpy(s.name, TSQL_TBNAME_L); return s; } -static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) { +static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) { corVgroupInfo->version = 0; corVgroupInfo->inUse = 0; corVgroupInfo->numOfEps = vgroupInfo->numOfEps; @@ -166,7 +166,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; - SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; + SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps; pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7f16476d89..1584afe706 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -48,7 +48,7 @@ static int32_t getWaitingTimeInterval(int32_t count) { return initial * (2<<(count - 2)); } -static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { +static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); SRpcEpSet* pEpSet = &pSql->epSet; @@ -100,7 +100,7 @@ void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) { tscMgmtEpSet.epSet = *pEpSet; taosCorEndWrite(&tscMgmtEpSet.version); } -static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { +static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { if (pVgroupInfo == NULL) { return;} taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; @@ -117,7 +117,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; + SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); @@ -158,12 +158,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SSqlRes *pRes = &pSql->res; if (code == 0) { - SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; - SRpcEpSet * epSet = &pRsp->epSet; + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; + SRpcEpSet * epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); tscUpdateMgmtEpSet(epSet); - } + } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -584,7 +584,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - SCMVgroupInfo* pVgroupInfo = NULL; + SVgroupInfo* pVgroupInfo = NULL; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t index = pTableMetaInfo->vgroupIndex; assert(index >= 0); @@ -908,10 +908,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateDbMsg); + pCmd->payloadLen = sizeof(SCreateDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB; - SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload; + SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; assert(pCmd->numOfClause == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -922,13 +922,13 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateDnodeMsg); + pCmd->payloadLen = sizeof(SCreateDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload; + SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE; @@ -938,13 +938,13 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateAcctMsg); + pCmd->payloadLen = sizeof(SCreateAcctMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; + SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; SStrToken *pName = &pInfo->pDCLInfo->user.user; SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; @@ -983,14 +983,14 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCreateUserMsg); + pCmd->payloadLen = sizeof(SCreateUserMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateUserMsg *pAlterMsg = (SCMCreateUserMsg*)pCmd->payload; + SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; SUserInfo *pUser = &pInfo->pDCLInfo->user; strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); @@ -1015,21 +1015,21 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMCfgDnodeMsg); + pCmd->payloadLen = sizeof(SCfgDnodeMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_CONFIG_DNODE; return TSDB_CODE_SUCCESS; } int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropDbMsg); + pCmd->payloadLen = sizeof(SDropDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload; + SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db)); @@ -1059,13 +1059,13 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropDnodeMsg); + pCmd->payloadLen = sizeof(SDropDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload; + SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep)); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; @@ -1075,7 +1075,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropUserMsg); + pCmd->payloadLen = sizeof(SDropUserMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1083,7 +1083,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload; + SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); @@ -1092,7 +1092,7 @@ int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropUserMsg); + pCmd->payloadLen = sizeof(SDropUserMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { @@ -1100,7 +1100,7 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload; + SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); @@ -1109,14 +1109,14 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMUseDbMsg); + pCmd->payloadLen = sizeof(SUseDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload; + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); strcpy(pUseDbMsg->db, pTableMetaInfo->name); pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; @@ -1128,14 +1128,14 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STscObj *pObj = pSql->pTscObj; SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TSDB_MSG_TYPE_CM_SHOW; - pCmd->payloadLen = sizeof(SCMShowMsg) + 100; + pCmd->payloadLen = sizeof(SShowMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload; + SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); size_t nameLen = strlen(pTableMetaInfo->name); @@ -1162,13 +1162,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pEpAddr->n); } - pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen; + pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen; return TSDB_CODE_SUCCESS; } int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMKillQueryMsg); + pCmd->payloadLen = sizeof(SKillQueryMsg); switch (pCmd->command) { case TSDB_SQL_KILL_QUERY: @@ -1280,8 +1280,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - return minMsgSize() + sizeof(SCMAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + - TSDB_EXTRA_PAYLOAD_SIZE; + return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; } int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -1300,7 +1299,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; + SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db); strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); @@ -1349,10 +1348,10 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMAlterDbMsg); + pCmd->payloadLen = sizeof(SAlterDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB; - SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; + SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db)); @@ -1477,14 +1476,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STscObj *pObj = pSql->pTscObj; SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TSDB_MSG_TYPE_CM_CONNECT; - pCmd->payloadLen = sizeof(SCMConnectMsg); + pCmd->payloadLen = sizeof(SConnectMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMConnectMsg *pConnect = (SCMConnectMsg*)pCmd->payload; + SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; // TODO refactor full_name char *db; // ugly code to move the space @@ -1506,11 +1505,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload; + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; strcpy(pInfoMsg->tableId, pTableMetaInfo->name); pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); - char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); size_t len = htonl(pCmd->tagData.dataLen); if (pSql->cmd.autoCreated) { @@ -1529,7 +1528,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { /** * multi table meta req pkg format: - * | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * | SMgmtHead | SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... * no used 4B **/ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -1547,7 +1546,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize); memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db - SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); + SMultiTableInfoMsg *pInfoMsg = (SMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); pInfoMsg->numOfTables = htonl((int32_t)pCmd->count); if (pCmd->payloadLen > 0) { @@ -1556,7 +1555,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { taosTFree(tmpData); - pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg); + pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META; assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); @@ -1601,12 +1600,12 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char* pMsg = pCmd->payload; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg; + + SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); - pMsg += sizeof(SCMSTableVgroupMsg); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + pMsg += sizeof(SSTableVgroupMsg); + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); size_t size = sizeof(pTableMetaInfo->name); tstrncpy(pMsg, pTableMetaInfo->name, size); @@ -1639,7 +1638,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { numOfStreams++; } - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100; + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to create heartbeat msg", pSql); @@ -1647,7 +1646,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } // TODO the expired hb and client can not be identified by server till now. - SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; + SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); pHeartbeat->numOfQueries = numOfQueries; @@ -1738,7 +1737,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { /** * multi table meta rsp pkg format: - * | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 * |...... 1B 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { @@ -1755,9 +1754,9 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { rsp++; - SCMMultiTableInfoMsg *pInfo = (SCMMultiTableInfoMsg *)rsp; + SMultiTableInfoMsg *pInfo = (SMultiTableInfoMsg *)rsp; totalNum = htonl(pInfo->numOfTables); - rsp += sizeof(SCMMultiTableInfoMsg); + rsp += sizeof(SMultiTableInfoMsg); for (i = 0; i < totalNum; i++) { SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; @@ -1849,10 +1848,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SSqlRes* pRes = &pSql->res; // NOTE: the order of several table must be preserved. - SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp; + SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); - char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg); - + char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); + // master sqlObj locates in param SSqlObj* parent = pSql->param; assert(parent != NULL); @@ -1864,18 +1863,18 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg; pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg); + size_t size = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg); - size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + size_t vgroupsz = sizeof(SVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); pInfo->vgroupList = calloc(1, vgroupsz); assert(pInfo->vgroupList != NULL); pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { //just init, no need to lock - SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; + SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; pVgroups->vgId = htonl(vmsg->vgId); pVgroups->numOfEps = vmsg->numOfEps; @@ -1897,10 +1896,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { * current process do not use the cache at all */ int tscProcessShowRsp(SSqlObj *pSql) { - STableMetaMsg * pMetaMsg; - SCMShowRsp *pShow; - SSchema * pSchema; - char key[20]; + STableMetaMsg *pMetaMsg; + SShowRsp * pShow; + SSchema * pSchema; + char key[20]; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -1909,7 +1908,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pShow = (SCMShowRsp *)pRes->pRsp; + pShow = (SShowRsp *)pRes->pRsp; pShow->qhandle = htobe64(pShow->qhandle); pRes->qhandle = pShow->qhandle; @@ -2007,7 +2006,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; - SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp; + SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index faf016be23..f328a6c075 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -583,7 +583,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr SVgroupTableInfo info = {{0}}; for (int32_t m = 0; m < pvg->numOfVgroups; ++m) { if (tt->vgId == pvg->vgroups[m].vgId) { - tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]); + tscSVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]); break; } } @@ -1787,8 +1787,8 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); @@ -1990,7 +1990,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; @@ -2101,7 +2101,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; + SVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[trsupport->subqueryIndex]; // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 77d668c5af..839d5889f3 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -104,7 +104,6 @@ void taos_init_imp(void) { taosReadGlobalCfg(); taosCheckGlobalCfg(); - taosPrintGlobalCfg(); tscDebug("starting to initialize TAOS client ..."); tscDebug("Local End Point is:%s", tsLocalEp); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index effb2db6ac..879eeeaded 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2428,7 +2428,7 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { return NULL; } - size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; + size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupList->numOfVgroups; SVgroupsInfo* pNew = calloc(1, size); if (pNew == NULL) { return NULL; @@ -2437,9 +2437,9 @@ SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { pNew->numOfVgroups = vgroupList->numOfVgroups; for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i]; + SVgroupInfo* pNewVInfo = &pNew->vgroups[i]; - SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i]; + SVgroupInfo* pvInfo = &vgroupList->vgroups[i]; pNewVInfo->vgId = pvInfo->vgId; pNewVInfo->numOfEps = pvInfo->numOfEps; @@ -2458,7 +2458,7 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { } for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i]; + SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i]; for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { taosTFree(pVgroupInfo->epAddr[j].fqdn); @@ -2469,7 +2469,7 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { return NULL; } -void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) { +void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { dst->vgId = src->vgId; dst->numOfEps = src->numOfEps; for(int32_t i = 0; i < dst->numOfEps; ++i) { diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 4636eaac08..f059ec1566 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -51,6 +51,7 @@ extern char tsLocale[]; extern char tsCharset[]; // default encode string extern int32_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; +extern char tsTempDir[]; //query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing @@ -183,13 +184,13 @@ extern int32_t debugFlag; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -void taosInitGlobalCfg(); -bool taosCheckGlobalCfg(); -void taosSetAllDebugFlag(); -bool taosCfgDynamicOptions(char *msg); -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); - +void taosInitGlobalCfg(); +int32_t taosCheckGlobalCfg(); +void taosSetAllDebugFlag(); +bool taosCfgDynamicOptions(char *msg); +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); + #ifdef __cplusplus } #endif diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 32569e3982..cbeac421c9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -58,6 +58,7 @@ char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string int32_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; +char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -1310,13 +1311,23 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + + cfg.option = "tempDir"; + cfg.ptr = tsTempDir; + cfg.valType = TAOS_CFG_VTYPE_STRING; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 0; + cfg.ptrLength = tListLen(tsTempDir); + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); } void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -bool taosCheckGlobalCfg() { +int32_t taosCheckGlobalCfg() { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -1375,7 +1386,9 @@ bool taosCheckGlobalCfg() { tsSyncPort = tsServerPort + TSDB_PORT_SYNC; tsHttpPort = tsServerPort + TSDB_PORT_HTTP; - return true; + taosPrintGlobalCfg(); + + return 0; } int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { diff --git a/src/connector/go b/src/connector/go index 8d7bf74385..8c58c512b6 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h new file mode 100644 index 0000000000..35d8896460 --- /dev/null +++ b/src/dnode/inc/dnodeCfg.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_CFG_H +#define TDENGINE_DNODE_CFG_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t dnodeInitCfg(); +void dnodeCleanupCfg(); +void dnodeUpdateCfg(SDnodeCfg *cfg); +int32_t dnodeGetDnodeId(); +void dnodeGetCfg(int32_t *dnodeId, char *clusterId); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/dnode/inc/dnodeEps.h b/src/dnode/inc/dnodeEps.h new file mode 100644 index 0000000000..2a203498c1 --- /dev/null +++ b/src/dnode/inc/dnodeEps.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_EP_H +#define TDENGINE_DNODE_EP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosmsg.h" + +int32_t dnodeInitEps(); +void dnodeCleanupEps(); +void dnodeUpdateEps(SDnodeEps *eps); +void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); +bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h new file mode 100644 index 0000000000..9c3c85c47e --- /dev/null +++ b/src/dnode/inc/dnodeMInfos.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DNODE_MINFOS_H +#define TDENGINE_DNODE_MINFOS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosmsg.h" + +int32_t dnodeInitMInfos(); +void dnodeCleanupMInfos(); +void dnodeUpdateMInfos(SMnodeInfos *minfos); +void dnodeUpdateEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetMInfos(SMnodeInfos *minfos); +bool dnodeIsMasterEp(char *ep); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index e8f4a0823f..2038ef5286 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "trpc.h" + int32_t dnodeInitMgmt(); void dnodeCleanupMgmt(); int32_t dnodeInitMgmtTimer(); @@ -35,8 +37,8 @@ void* dnodeGetVnodeTsdb(void *pVnode); void dnodeReleaseVnode(void *pVnode); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetMnodeEpSetForPeer(void *epSet); -void dnodeGetMnodeEpSetForShell(void *epSet); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SRpcEpSet *epSet); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeVWrite.h b/src/dnode/inc/dnodeVWrite.h index 7da701a8e2..323405143f 100644 --- a/src/dnode/inc/dnodeVWrite.h +++ b/src/dnode/inc/dnodeVWrite.h @@ -20,9 +20,12 @@ extern "C" { #endif -int32_t dnodeInitVnodeWrite(); -void dnodeCleanupVnodeWrite(); -void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg); +int32_t dnodeInitVWrite(); +void dnodeCleanupVWrite(); +void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg); +void * dnodeAllocVWriteQueue(void *pVnode); +void dnodeFreeVWriteQueue(void *wqueue); +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c new file mode 100644 index 0000000000..16d109a13a --- /dev/null +++ b/src/dnode/src/dnodeCfg.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "dnode.h" +#include "dnodeInt.h" +#include "dnodeCfg.h" + +static SDnodeCfg tsCfg = {0}; +static pthread_mutex_t tsCfgMutex; + +static int32_t dnodeReadCfg(); +static int32_t dnodeWriteCfg(); +static void dnodeResetCfg(SDnodeCfg *cfg); +static void dnodePrintCfg(SDnodeCfg *cfg); + +int32_t dnodeInitCfg() { + pthread_mutex_init(&tsCfgMutex, NULL); + dnodeResetCfg(NULL); + int32_t ret = dnodeReadCfg(); + if (ret == 0) { + dInfo("dnode cfg is initialized"); + } + return ret; +} + +void dnodeCleanupCfg() { pthread_mutex_destroy(&tsCfgMutex); } + +void dnodeUpdateCfg(SDnodeCfg *cfg) { + if (tsCfg.dnodeId != 0) return; + dnodeResetCfg(cfg); +} + +int32_t dnodeGetDnodeId() { + int32_t dnodeId = 0; + pthread_mutex_lock(&tsCfgMutex); + dnodeId = tsCfg.dnodeId; + pthread_mutex_unlock(&tsCfgMutex); + return dnodeId; +} + +void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { + pthread_mutex_lock(&tsCfgMutex); + *dnodeId = tsCfg.dnodeId; + tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsCfgMutex); +} + +static void dnodeResetCfg(SDnodeCfg *cfg) { + if (cfg == NULL) return; + if (cfg->dnodeId == 0) return; + + pthread_mutex_lock(&tsCfgMutex); + tsCfg.dnodeId = cfg->dnodeId; + tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); + dnodePrintCfg(cfg); + dnodeWriteCfg(); + pthread_mutex_unlock(&tsCfgMutex); +} + +static void dnodePrintCfg(SDnodeCfg *cfg) { + dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId); +} + +static int32_t dnodeReadCfg() { + int32_t len = 0; + int32_t maxLen = 200; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SDnodeCfg cfg = {0}; + + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeCfg.json", tsDnodeDir); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("failed to read %s, file not exist", file); + goto PARSE_CFG_OVER; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s, content is null", file); + goto PARSE_CFG_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s, invalid json format", file); + goto PARSE_CFG_OVER; + } + + cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s, dnodeId not found", file); + goto PARSE_CFG_OVER; + } + cfg.dnodeId = dnodeId->valueint; + + cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_String) { + dError("failed to read %s, clusterId not found", file); + goto PARSE_CFG_OVER; + } + tstrncpy(cfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); + + dInfo("read file %s successed", file); + +PARSE_CFG_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + terrno = 0; + + dnodeResetCfg(&cfg); + return 0; +} + +static int32_t dnodeWriteCfg() { + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeCfg.json", tsDnodeDir); + + FILE *fp = fopen(file, "w"); + if (!fp) { + dError("failed to write %s, reason:%s", file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 200; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsCfg.dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsCfg.clusterId); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", file); + return 0; +} diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index 9b68fc1f6c..a9ee4ac649 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -15,9 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" #include "tglobal.h" -#include "mnode.h" #include "dnodeInt.h" #include "dnodeCheck.h" @@ -30,8 +28,8 @@ typedef struct { void (*stopFp)(); } SCheckItem; -static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; -int64_t tsMinFreeMemSizeForStart = 0; +static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; +int64_t tsMinFreeMemSizeForStart = 0; static int bindTcpPort(int port) { int serverSocket; @@ -264,8 +262,6 @@ int32_t dnodeInitCheck() { } } + dInfo("dnode check is initialized"); return 0; } - - - diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c new file mode 100644 index 0000000000..9c90c39181 --- /dev/null +++ b/src/dnode/src/dnodeEps.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "hash.h" +#include "dnode.h" +#include "dnodeInt.h" +#include "dnodeEps.h" + +static SDnodeEps *tsEps = NULL; +static SHashObj * tsEpsHash = NULL; +static pthread_mutex_t tsEpsMutex; + +static int32_t dnodeReadEps(); +static int32_t dnodeWriteEps(); +static void dnodeResetEps(SDnodeEps *eps); +static void dnodePrintEps(SDnodeEps *eps); + +int32_t dnodeInitEps() { + pthread_mutex_init(&tsEpsMutex, NULL); + tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + dnodeResetEps(NULL); + int32_t ret = dnodeReadEps(); + if (ret == 0) { + dInfo("dnode eps is initialized"); + } + return ret; +} + +void dnodeCleanupEps() { + pthread_mutex_lock(&tsEpsMutex); + if (tsEps) { + free(tsEps); + tsEps = NULL; + } + if (tsEpsHash) { + taosHashCleanup(tsEpsHash); + tsEpsHash = NULL; + } + pthread_mutex_unlock(&tsEpsMutex); + pthread_mutex_destroy(&tsEpsMutex); +} + +void dnodeUpdateEps(SDnodeEps *eps) { + if (eps == NULL) return; + + eps->dnodeNum = htonl(eps->dnodeNum); + for (int32_t i = 0; i < eps->dnodeNum; ++i) { + eps->dnodeEps[i].dnodeId = htonl(eps->dnodeEps[i].dnodeId); + eps->dnodeEps[i].dnodePort = htons(eps->dnodeEps[i].dnodePort); + } + + pthread_mutex_lock(&tsEpsMutex); + if (eps->dnodeNum != tsEps->dnodeNum) { + dnodeResetEps(eps); + dnodeWriteEps(); + } else { + int32_t size = sizeof(SDnodeEps) + eps->dnodeNum * sizeof(SDnodeEp); + if (memcmp(eps, tsEps, size) != 0) { + dnodeResetEps(eps); + dnodeWriteEps(); + } + } + pthread_mutex_unlock(&tsEpsMutex); +} + +bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr) { + bool changed = false; + pthread_mutex_lock(&tsEpsMutex); + SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t)); + if (ep != NULL) { + char epSaved[TSDB_EP_LEN + 1]; + snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); + changed = strcmp(epstr, epSaved) != 0; + tstrncpy(epstr, epSaved, TSDB_EP_LEN); + } + pthread_mutex_unlock(&tsEpsMutex); + return changed; +} + +void dnodeUpdateEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) { + pthread_mutex_lock(&tsEpsMutex); + SDnodeEp *ep = taosHashGet(tsEpsHash, &dnodeId, sizeof(int32_t)); + if (ep != NULL) { + if (port) *port = ep->dnodePort; + if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN); + if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort); + } + pthread_mutex_unlock(&tsEpsMutex); +} + +static void dnodeResetEps(SDnodeEps *eps) { + if (eps == NULL) { + int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp); + if (tsEps == NULL) { + tsEps = calloc(1, size); + } else { + tsEps->dnodeNum = 0; + } + } else { + assert(tsEps); + + int32_t size = sizeof(SDnodeEps) + sizeof(SDnodeEp) * eps->dnodeNum; + if (eps->dnodeNum > tsEps->dnodeNum) { + tsEps = realloc(tsEps, size); + } + memcpy(tsEps, eps, size); + dnodePrintEps(eps); + } + + for (int32_t i = 0; i < tsEps->dnodeNum; ++i) { + SDnodeEp *ep = &tsEps->dnodeEps[i]; + taosHashPut(tsEpsHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); + } +} + +static void dnodePrintEps(SDnodeEps *eps) { + dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum); + for (int32_t i = 0; i < eps->dnodeNum; i++) { + SDnodeEp *ep = &eps->dnodeEps[i]; + dDebug("dnodeId:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort); + } +} + +static int32_t dnodeReadEps() { + int32_t ret = -1; + int32_t len = 0; + int32_t maxLen = 30000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SDnodeEps *eps = NULL; + + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeEps.json", tsDnodeDir); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("failed to read %s, file not exist", file); + goto PRASE_EPS_OVER; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s, content is null", file); + goto PRASE_EPS_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s, invalid json format", file); + goto PRASE_EPS_OVER; + } + + cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum"); + if (!dnodeNum || dnodeNum->type != cJSON_Number) { + dError("failed to read %s, dnodeNum not found", file); + goto PRASE_EPS_OVER; + } + + cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos"); + if (!dnodeInfos || dnodeInfos->type != cJSON_Array) { + dError("failed to read %s, dnodeInfos not found", file); + goto PRASE_EPS_OVER; + } + + int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos); + if (dnodeInfosSize != dnodeNum->valueint) { + dError("failed to read %s, dnodeInfos size:%d not matched dnodeNum:%d", file, dnodeInfosSize, + (int32_t)dnodeNum->valueint); + goto PRASE_EPS_OVER; + } + + int32_t epsSize = sizeof(SDnodeEps) + dnodeInfosSize * sizeof(SDnodeEp); + eps = calloc(1, epsSize); + eps->dnodeNum = dnodeInfosSize; + + for (int32_t i = 0; i < dnodeInfosSize; ++i) { + cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i); + if (dnodeInfo == NULL) break; + + SDnodeEp *ep = &eps->dnodeEps[i]; + + cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); + if (!dnodeId || dnodeId->type != cJSON_Number) { + dError("failed to read %s, dnodeId not found", file); + goto PRASE_EPS_OVER; + } + ep->dnodeId = dnodeId->valueint; + + cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); + if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { + dError("failed to read %s, dnodeFqdn not found", file); + goto PRASE_EPS_OVER; + } + strncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + + cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); + if (!dnodePort || dnodePort->type != cJSON_Number) { + dError("failed to read %s, dnodePort not found", file); + goto PRASE_EPS_OVER; + } + ep->dnodePort = (uint16_t)dnodePort->valueint; + } + + ret = 0; + + dInfo("read file %s successed", file); + dnodePrintEps(eps); + +PRASE_EPS_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + if (ret != 0) { + if (eps) free(eps); + eps = NULL; + } + + dnodeResetEps(eps); + if (eps) free(eps); + + dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); + + terrno = 0; + return 0; +} + +static int32_t dnodeWriteEps() { + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/dnodeEps.json", tsDnodeDir); + + FILE *fp = fopen(file, "w"); + if (!fp) { + dError("failed to write %s, reason:%s", file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 30000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"dnodeNum\": %d,\n", tsEps->dnodeNum); + len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n"); + for (int32_t i = 0; i < tsEps->dnodeNum; ++i) { + SDnodeEp *ep = &tsEps->dnodeEps[i]; + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", ep->dnodeId); + len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn); + len += snprintf(content + len, maxLen - len, " \"dnodePort\": %u\n", ep->dnodePort); + if (i < tsEps->dnodeNum - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", file); + return 0; +} diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c new file mode 100644 index 0000000000..c985db371d --- /dev/null +++ b/src/dnode/src/dnodeMInfos.c @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "cJSON.h" +#include "tglobal.h" +#include "mnode.h" +#include "dnode.h" +#include "dnodeInt.h" +#include "dnodeMInfos.h" + +static SMnodeInfos tsMInfos; +static SRpcEpSet tsMEpSet; +static pthread_mutex_t tsMInfosMutex; + +static void dnodeResetMInfos(SMnodeInfos *minfos); +static void dnodePrintMInfos(SMnodeInfos *minfos); +static int32_t dnodeReadMInfos(); +static int32_t dnodeWriteMInfos(); + +int32_t dnodeInitMInfos() { + pthread_mutex_init(&tsMInfosMutex, NULL); + dnodeResetMInfos(NULL); + int32_t ret = dnodeReadMInfos(); + if (ret == 0) { + dInfo("dnode minfos is initialized"); + } + + return ret; +} + +void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); } + +void dnodeUpdateMInfos(SMnodeInfos *minfos) { + if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) { + dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum); + return; + } + + for (int32_t i = 0; i < minfos->mnodeNum; ++i) { + SMnodeInfo *minfo = &minfos->mnodeInfos[i]; + minfo->mnodeId = htonl(minfo->mnodeId); + if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) { + dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp); + return; + } + } + + pthread_mutex_lock(&tsMInfosMutex); + if (minfos->mnodeNum != tsMInfos.mnodeNum) { + dnodeResetMInfos(minfos); + dnodeWriteMInfos(); + sdbUpdateAsync(); + } else { + int32_t size = sizeof(SMnodeInfos); + if (memcmp(minfos, &tsMInfos, size) != 0) { + dnodeResetMInfos(minfos); + dnodeWriteMInfos(); + sdbUpdateAsync(); + } + } + pthread_mutex_unlock(&tsMInfosMutex); +} + +void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { + if (ep->numOfEps <= 0) { + dError("mnode EP list for peer is changed, but content is invalid, discard it"); + return; + } + + pthread_mutex_lock(&tsMInfosMutex); + dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); + for (int i = 0; i < ep->numOfEps; ++i) { + ep->port[i] -= TSDB_PORT_DNODEDNODE; + dInfo("mnode index:%d %s:%u", i, ep->fqdn[i], ep->port[i]); + } + tsMEpSet = *ep; + pthread_mutex_unlock(&tsMInfosMutex); +} + +bool dnodeIsMasterEp(char *ep) { + pthread_mutex_lock(&tsMInfosMutex); + bool isMaster = strcmp(ep, tsMInfos.mnodeInfos[tsMEpSet.inUse].mnodeEp) == 0; + pthread_mutex_unlock(&tsMInfosMutex); + + return isMaster; +} + +void dnodeGetMInfos(SMnodeInfos *minfos) { + pthread_mutex_lock(&tsMInfosMutex); + memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos)); + for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) { + minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); + } + pthread_mutex_unlock(&tsMInfosMutex); +} + +void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { + pthread_mutex_lock(&tsMInfosMutex); + *epSet = tsMEpSet; + for (int i = 0; i < epSet->numOfEps; ++i) { + epSet->port[i] += TSDB_PORT_DNODEDNODE; + } + pthread_mutex_unlock(&tsMInfosMutex); +} + +void dnodeGetEpSetForShell(SRpcEpSet *epSet) { + pthread_mutex_lock(&tsMInfosMutex); + *epSet = tsMEpSet; + pthread_mutex_unlock(&tsMInfosMutex); +} + +static void dnodePrintMInfos(SMnodeInfos *minfos) { + dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse); + for (int32_t i = 0; i < minfos->mnodeNum; i++) { + dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp); + } +} + +static void dnodeResetMInfos(SMnodeInfos *minfos) { + if (minfos == NULL) { + tsMEpSet.numOfEps = 1; + taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]); + + if (strcmp(tsSecond, tsFirst) != 0) { + tsMEpSet.numOfEps = 2; + taosGetFqdnPortFromEp(tsSecond, tsMEpSet.fqdn[1], &tsMEpSet.port[1]); + } + return; + } + + if (minfos->mnodeNum == 0) return; + + int32_t size = sizeof(SMnodeInfos); + memcpy(&tsMInfos, minfos, size); + + tsMEpSet.inUse = tsMInfos.inUse; + tsMEpSet.numOfEps = tsMInfos.mnodeNum; + for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) { + taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]); + } + + dnodePrintMInfos(minfos); +} + +static int32_t dnodeReadMInfos() { + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SMnodeInfos minfos = {0}; + + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); + + fp = fopen(file, "r"); + if (!fp) { + dDebug("failed to read %s, file not exist", file); + goto PARSE_MINFOS_OVER; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s, content is null", file); + goto PARSE_MINFOS_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s, invalid json format", file); + goto PARSE_MINFOS_OVER; + } + + cJSON *inUse = cJSON_GetObjectItem(root, "inUse"); + if (!inUse || inUse->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json, inUse not found"); + goto PARSE_MINFOS_OVER; + } + tsMInfos.inUse = inUse->valueint; + + cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); + if (!nodeNum || nodeNum->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json, nodeNum not found"); + goto PARSE_MINFOS_OVER; + } + minfos.mnodeNum = nodeNum->valueint; + + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + dError("failed to read mnodeEpSet.json, nodeInfos not found"); + goto PARSE_MINFOS_OVER; + } + + int size = cJSON_GetArraySize(nodeInfos); + if (size != minfos.mnodeNum) { + dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); + goto PARSE_MINFOS_OVER; + } + + for (int i = 0; i < size; ++i) { + cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + dError("failed to read mnodeEpSet.json, nodeId not found"); + goto PARSE_MINFOS_OVER; + } + minfos.mnodeInfos[i].mnodeId = nodeId->valueint; + + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { + dError("failed to read mnodeEpSet.json, nodeName not found"); + goto PARSE_MINFOS_OVER; + } + strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + } + + dInfo("read file %s successed", file); + dnodePrintMInfos(&minfos); + +PARSE_MINFOS_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + terrno = 0; + + for (int32_t i = 0; i < minfos.mnodeNum; ++i) { + SMnodeInfo *mInfo = &minfos.mnodeInfos[i]; + dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); + } + dnodeResetMInfos(&minfos); + return 0; +} + +static int32_t dnodeWriteMInfos() { + char file[TSDB_FILENAME_LEN + 20] = {0}; + sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); + + FILE *fp = fopen(file, "w"); + if (!fp) { + dError("failed to write %s, reason:%s", file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsMInfos.inUse); + len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsMInfos.mnodeNum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < tsMInfos.mnodeNum; i++) { + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMInfos.mnodeInfos[i].mnodeId); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMInfos.mnodeInfos[i].mnodeEp); + if (i < tsMInfos.mnodeNum - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", file); + return 0; +} diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 8414d79a98..0da7ab74e4 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -58,7 +58,7 @@ int32_t dnodeInitMnodePeer() { dDebug("dnode mpeer worker:%d is created", i); } - dDebug("dnode mpeer is opened, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset); + dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset); return 0; } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index fdcbb5889f..4ad787f26e 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -60,7 +60,7 @@ int32_t dnodeInitMnodeRead() { dDebug("dnode mread worker:%d is created", i); } - dDebug("dnode mread is opened, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset); + dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset); return 0; } diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 384a0fae75..600688b9fd 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -60,7 +60,7 @@ int32_t dnodeInitMnodeWrite() { dDebug("dnode mwrite worker:%d is created", i); } - dDebug("dnode mwrite is opened, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset); + dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset); return 0; } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 97e6f2ce6d..b4c174e29b 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -19,11 +19,15 @@ #include "tutil.h" #include "tconfig.h" #include "tglobal.h" +#include "twal.h" #include "dnode.h" #include "dnodeInt.h" #include "dnodeMgmt.h" #include "dnodePeer.h" #include "dnodeModule.h" +#include "dnodeEps.h" +#include "dnodeMInfos.h" +#include "dnodeCfg.h" #include "dnodeCheck.h" #include "dnodeVRead.h" #include "dnodeVWrite.h" @@ -33,26 +37,32 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" -static int32_t dnodeInitStorage(); -static void dnodeCleanupStorage(); -static void dnodeSetRunStatus(SDnodeRunStatus status); -static void dnodeCheckDataDirOpenned(char *dir); static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED; + +static int32_t dnodeInitStorage(); +static void dnodeCleanupStorage(); +static void dnodeSetRunStatus(SDnodeRunStatus status); +static void dnodeCheckDataDirOpenned(char *dir); static int32_t dnodeInitComponents(); -static void dnodeCleanupComponents(int32_t stepId); -static int dnodeCreateDir(const char *dir); +static void dnodeCleanupComponents(int32_t stepId); +static int dnodeCreateDir(const char *dir); typedef struct { const char *const name; - int (*init)(); - void (*cleanup)(); + int32_t (*init)(); + void (*cleanup)(); } SDnodeComponent; static const SDnodeComponent tsDnodeComponents[] = { {"storage", dnodeInitStorage, dnodeCleanupStorage}, + {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, + {"dnodeeps", dnodeInitEps, dnodeCleanupEps}, + {"globalcfg" ,taosCheckGlobalCfg, NULL}, + {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, + {"wal", walInit, walCleanUp}, {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! {"vread", dnodeInitVnodeRead, dnodeCleanupVnodeRead}, - {"vwrite", dnodeInitVnodeWrite, dnodeCleanupVnodeWrite}, + {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead}, {"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite}, {"mpeer", dnodeInitMnodePeer, dnodeCleanupMnodePeer}, @@ -75,7 +85,9 @@ static int dnodeCreateDir(const char *dir) { static void dnodeCleanupComponents(int32_t stepId) { for (int32_t i = stepId; i >= 0; i--) { - tsDnodeComponents[i].cleanup(); + if (tsDnodeComponents[i].cleanup) { + (*tsDnodeComponents[i].cleanup)(); + } } } @@ -112,14 +124,13 @@ int32_t dnodeInitSystem() { printf("failed to init log file\n"); } - if (!taosReadGlobalCfg() || !taosCheckGlobalCfg()) { + if (!taosReadGlobalCfg()) { taosPrintGlobalCfg(); dError("TDengine read global config failed"); return -1; } - taosPrintGlobalCfg(); - dInfo("start to initialize TDengine on %s", tsLocalEp); + dInfo("start to initialize TDengine"); if (dnodeInitComponents() != 0) { return -1; @@ -198,7 +209,7 @@ static int32_t dnodeInitStorage() { dnodeCheckDataDirOpenned(tsDnodeDir); - dInfo("storage directory is initialized"); + dInfo("dnode storage is initialized at %s", tsDnodeDir); return 0; } diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 968a8d9759..4a5dc31a9f 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -31,12 +31,13 @@ #include "mnode.h" #include "dnodeInt.h" #include "dnodeMgmt.h" +#include "dnodeEps.h" +#include "dnodeCfg.h" +#include "dnodeMInfos.h" #include "dnodeVRead.h" #include "dnodeVWrite.h" #include "dnodeModule.h" -#define MPEER_CONTENT_LEN 2000 - typedef struct { pthread_t thread; int32_t threadIndex; @@ -46,23 +47,13 @@ typedef struct { int32_t * vnodeList; } SOpenVnodeThread; -void * tsDnodeTmr = NULL; -static void * tsStatusTimer = NULL; -static uint32_t tsRebootTime; +void * tsDnodeTmr = NULL; +static void * tsStatusTimer = NULL; +static uint32_t tsRebootTime; +static taos_qset tsMgmtQset = NULL; +static taos_queue tsMgmtQueue = NULL; +static pthread_t tsQthread; -static SRpcEpSet tsDMnodeEpSet = {0}; -static SDMMnodeInfos tsDMnodeInfos = {0}; -static SDMDnodeCfg tsDnodeCfg = {0}; -static taos_qset tsMgmtQset = NULL; -static taos_queue tsMgmtQueue = NULL; -static pthread_t tsQthread; - -static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes); -static bool dnodeReadMnodeInfos(); -static void dnodeSaveMnodeInfos(); -static void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg); -static bool dnodeReadDnodeCfg(); -static void dnodeSaveDnodeCfg(); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); static void dnodeSendStatusMsg(void *handle, void *tmrId); static void *dnodeProcessMgmtQueue(void *param); @@ -74,7 +65,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); -static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg); static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); int32_t dnodeInitMgmt() { @@ -86,28 +77,8 @@ int32_t dnodeInitMgmt() { dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg; dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp); - dnodeReadDnodeCfg(); tsRebootTime = taosGetTimestampSec(); - if (!dnodeReadMnodeInfos()) { - memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet)); - memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos)); - - tsDMnodeEpSet.numOfEps = 1; - taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]); - - if (strcmp(tsSecond, tsFirst) != 0) { - tsDMnodeEpSet.numOfEps = 2; - taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]); - } - } else { - tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); - } - } - int32_t code = vnodeInitResources(); if (code != TSDB_CODE_SUCCESS) { dnodeCleanupMgmt(); @@ -381,7 +352,7 @@ static void dnodeCloseVnodes() { } static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { - SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; + SCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); @@ -404,7 +375,7 @@ static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { } static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { - SMDCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); + SCreateVnodeMsg *pCreate = dnodeParseVnodeMsg(rpcMsg); void *pVnode = vnodeAcquire(pCreate->cfg.vgId); if (pVnode != NULL) { @@ -418,7 +389,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { } static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { - SMDAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); + SAlterVnodeMsg *pAlter = dnodeParseVnodeMsg(rpcMsg); void *pVnode = vnodeAcquire(pAlter->cfg.vgId); if (pVnode != NULL) { @@ -433,14 +404,14 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { } static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { - SMDDropVnodeMsg *pDrop = rpcMsg->pCont; + SDropVnodeMsg *pDrop = rpcMsg->pCont; pDrop->vgId = htonl(pDrop->vgId); return vnodeDrop(pDrop->vgId); } static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { -// SMDAlterStreamMsg *pStream = pCont; +// SAlterStreamMsg *pStream = pCont; // pStream->uid = htobe64(pStream->uid); // pStream->stime = htobe64(pStream->stime); // pStream->vnode = htonl(pStream->vnode); @@ -453,12 +424,12 @@ static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) { } static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { - SMDCfgDnodeMsg *pCfg = pMsg->pCont; + SCfgDnodeMsg *pCfg = pMsg->pCont; return taosCfgDynamicOptions(pCfg->config); } static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { - SMDCreateMnodeMsg *pCfg = pMsg->pCont; + SCreateMnodeMsg *pCfg = pMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); if (pCfg->dnodeId != dnodeGetDnodeId()) { dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); @@ -470,10 +441,10 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; } - dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.nodeNum); - for (int i = 0; i < pCfg->mnodes.nodeNum; ++i) { - pCfg->mnodes.nodeInfos[i].nodeId = htonl(pCfg->mnodes.nodeInfos[i].nodeId); - dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.nodeInfos[i].nodeId, pCfg->mnodes.nodeInfos[i].nodeEp); + dDebug("dnodeId:%d, create mnode msg is received from mnodes, numOfMnodes:%d", pCfg->dnodeId, pCfg->mnodes.mnodeNum); + for (int i = 0; i < pCfg->mnodes.mnodeNum; ++i) { + pCfg->mnodes.mnodeInfos[i].mnodeId = htonl(pCfg->mnodes.mnodeInfos[i].mnodeId); + dDebug("mnode index:%d, mnode:%d:%s", i, pCfg->mnodes.mnodeInfos[i].mnodeId, pCfg->mnodes.mnodeInfos[i].mnodeEp); } dnodeStartMnode(&pCfg->mnodes); @@ -481,34 +452,6 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } -void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) { - if (pEpSet->numOfEps <= 0) { - dError("mnode EP list for peer is changed, but content is invalid, discard it"); - return; - } - - dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse); - for (int i = 0; i < pEpSet->numOfEps; ++i) { - pEpSet->port[i] -= TSDB_PORT_DNODEDNODE; - dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]); - } - - tsDMnodeEpSet = *pEpSet; -} - -void dnodeGetMnodeEpSetForPeer(void *epSetRaw) { - SRpcEpSet *epSet = epSetRaw; - *epSet = tsDMnodeEpSet; - - for (int i=0; inumOfEps; ++i) - epSet->port[i] += TSDB_PORT_DNODEDNODE; -} - -void dnodeGetMnodeEpSetForShell(void *epSetRaw) { - SRpcEpSet *epSet = epSetRaw; - *epSet = tsDMnodeEpSet; -} - static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { if (pMsg->code != TSDB_CODE_SUCCESS) { dError("status rsp is received, error:%s", tstrerror(pMsg->code)); @@ -516,202 +459,24 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { return; } - SDMStatusRsp *pStatusRsp = pMsg->pCont; - SDMMnodeInfos *pMnodes = &pStatusRsp->mnodes; - if (pMnodes->nodeNum <= 0) { - dError("status msg is invalid, num of ips is %d", pMnodes->nodeNum); - taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); - return; - } + SStatusRsp *pStatusRsp = pMsg->pCont; + SMnodeInfos *minfos = &pStatusRsp->mnodes; + dnodeUpdateMInfos(minfos); - SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; - pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); + SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; + pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); pCfg->moduleStatus = htonl(pCfg->moduleStatus); pCfg->dnodeId = htonl(pCfg->dnodeId); - - for (int32_t i = 0; i < pMnodes->nodeNum; ++i) { - SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i]; - pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId); - } + dnodeUpdateCfg(pCfg); vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes); - // will not set mnode in status msg - // dnodeProcessModuleStatus(pCfg->moduleStatus); - dnodeUpdateDnodeCfg(pCfg); + SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess)); + dnodeUpdateEps(pEps); - dnodeUpdateMnodeInfos(pMnodes); taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); } -static bool dnodeCheckMnodeInfos(SDMMnodeInfos *pMnodes) { - if (pMnodes->nodeNum <= 0 || pMnodes->nodeNum > 3) { - dError("invalid mnode infos, num:%d", pMnodes->nodeNum); - return false; - } - - for (int32_t i = 0; i < pMnodes->nodeNum; ++i) { - SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i]; - if (pMnodeInfo->nodeId <= 0 || strlen(pMnodeInfo->nodeEp) <= 5) { - dError("invalid mnode info:%d, nodeId:%d nodeEp:%s", i, pMnodeInfo->nodeId, pMnodeInfo->nodeEp); - return false; - } - } - - return true; -} - -static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { - bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0); - bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0); - if (!(mnodesChanged || mnodesNotInit)) return; - - if (!dnodeCheckMnodeInfos(pMnodes)) return; - - memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)); - dInfo("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); - } - - tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); - } - - dnodeSaveMnodeInfos(); - sdbUpdateAsync(); -} - -static bool dnodeReadMnodeInfos() { - char ipFile[TSDB_FILENAME_LEN*2] = {0}; - - sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); - FILE *fp = fopen(ipFile, "r"); - if (!fp) { - dDebug("failed to read mnodeEpSet.json, file not exist"); - return false; - } - - bool ret = false; - int maxLen = 2000; - char *content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - free(content); - fclose(fp); - dError("failed to read mnodeEpSet.json, content is null"); - return false; - } - - content[len] = 0; - cJSON* root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read mnodeEpSet.json, invalid json format"); - goto PARSE_OVER; - } - - cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); - if (!inUse || inUse->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, inUse not found"); - goto PARSE_OVER; - } - tsDMnodeInfos.inUse = inUse->valueint; - - cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); - if (!nodeNum || nodeNum->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, nodeNum not found"); - goto PARSE_OVER; - } - tsDMnodeInfos.nodeNum = nodeNum->valueint; - - cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); - if (!nodeInfos || nodeInfos->type != cJSON_Array) { - dError("failed to read mnodeEpSet.json, nodeInfos not found"); - goto PARSE_OVER; - } - - int size = cJSON_GetArraySize(nodeInfos); - if (size != tsDMnodeInfos.nodeNum) { - dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); - goto PARSE_OVER; - } - - for (int i = 0; i < size; ++i) { - cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); - if (nodeInfo == NULL) continue; - - cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); - if (!nodeId || nodeId->type != cJSON_Number) { - dError("failed to read mnodeEpSet.json, nodeId not found"); - goto PARSE_OVER; - } - tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; - - cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); - if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - dError("failed to read mnodeEpSet.json, nodeName not found"); - goto PARSE_OVER; - } - strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); - } - - ret = true; - - dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); - } - -PARSE_OVER: - free(content); - cJSON_Delete(root); - fclose(fp); - return ret; -} - -static void dnodeSaveMnodeInfos() { - char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); - FILE *fp = fopen(ipFile, "w"); - if (!fp) return; - - int32_t len = 0; - int32_t maxLen = 2000; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"inUse\": %d,\n", tsDMnodeInfos.inUse); - len += snprintf(content + len, maxLen - len, " \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum); - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp); - if (i < tsDMnodeInfos.nodeNum -1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - free(content); - - dInfo("save mnode epSet successed"); -} - -char *dnodeGetMnodeMasterEp() { - return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp; -} - -void* dnodeGetMnodeInfos() { - return &tsDMnodeInfos; -} - static void dnodeSendStatusMsg(void *handle, void *tmrId) { if (tsDnodeTmr == NULL) { dError("dnode timer is already released"); @@ -724,22 +489,21 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { return; } - int32_t contLen = sizeof(SDMStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - SDMStatusMsg *pStatus = rpcMallocCont(contLen); + int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + SStatusMsg *pStatus = rpcMallocCont(contLen); if (pStatus == NULL) { taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); dError("failed to malloc status message"); return; } - //strcpy(pStatus->dnodeName, tsDnodeName); + dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId); + pStatus->dnodeId = htonl(dnodeGetDnodeId()); pStatus->version = htonl(tsVersion); - pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId); pStatus->lastReboot = htonl(tsRebootTime); pStatus->numOfCores = htons((uint16_t) tsNumOfCores); pStatus->diskAvailable = tsAvailDataDirGB; pStatus->alternativeRole = (uint8_t) tsAlternativeRole; - tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN); tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); // fill cluster cfg parameters @@ -759,7 +523,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); vnodeBuildStatusMsg(pStatus); - contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); + contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); pStatus->openVnodes = htons(pStatus->openVnodes); SRpcMsg rpcMsg = { @@ -769,110 +533,19 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { }; SRpcEpSet epSet; - dnodeGetMnodeEpSetForPeer(&epSet); + dnodeGetEpSetForPeer(&epSet); dnodeSendMsgToDnode(&epSet, &rpcMsg); } -static bool dnodeReadDnodeCfg() { - char dnodeCfgFile[TSDB_FILENAME_LEN*2] = {0}; - - sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir); - - FILE *fp = fopen(dnodeCfgFile, "r"); - if (!fp) { - dDebug("failed to read dnodeCfg.json, file not exist"); - return false; - } - - bool ret = false; - int maxLen = 100; - char *content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - free(content); - fclose(fp); - dError("failed to read dnodeCfg.json, content is null"); - return false; - } - - content[len] = 0; - cJSON* root = cJSON_Parse(content); - if (root == NULL) { - dError("failed to read dnodeCfg.json, invalid json format"); - goto PARSE_CFG_OVER; - } - - cJSON* dnodeId = cJSON_GetObjectItem(root, "dnodeId"); - if (!dnodeId || dnodeId->type != cJSON_Number) { - dError("failed to read dnodeCfg.json, dnodeId not found"); - goto PARSE_CFG_OVER; - } - tsDnodeCfg.dnodeId = dnodeId->valueint; - - cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId"); - if (!clusterId || clusterId->type != cJSON_String) { - dError("failed to read dnodeCfg.json, clusterId not found"); - goto PARSE_CFG_OVER; - } - tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN); - - ret = true; - - dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId); - -PARSE_CFG_OVER: - free(content); - cJSON_Delete(root); - fclose(fp); - return ret; -} - -static void dnodeSaveDnodeCfg() { - char dnodeCfgFile[TSDB_FILENAME_LEN] = {0}; - sprintf(dnodeCfgFile, "%s/dnodeCfg.json", tsDnodeDir); - - FILE *fp = fopen(dnodeCfgFile, "w"); - if (!fp) return; - - int32_t len = 0; - int32_t maxLen = 200; - char * content = calloc(1, maxLen + 1); - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId); - len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId); - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - free(content); - - dInfo("save dnodeId successed"); -} - -void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) { - if (tsDnodeCfg.dnodeId == 0) { - dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId); - tsDnodeCfg.dnodeId = pCfg->dnodeId; - tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN); - dnodeSaveDnodeCfg(); - } -} - -int32_t dnodeGetDnodeId() { - return tsDnodeCfg.dnodeId; -} - void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); SRpcEpSet epSet = {0}; if (forShell) { - dnodeGetMnodeEpSetForShell(&epSet); + dnodeGetEpSetForShell(&epSet); } else { - dnodeGetMnodeEpSetForPeer(&epSet); + dnodeGetEpSetForPeer(&epSet); } dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 46376159c6..bd9500ba51 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -114,6 +114,7 @@ int32_t dnodeInitModules() { } } + dInfo("dnode modules is initialized"); return 0; } @@ -146,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) { } } -bool dnodeStartMnode(void *pMnodes) { - SDMMnodeInfos *mnodes = pMnodes; +bool dnodeStartMnode(SMnodeInfos *minfos) { + SMnodeInfos *mnodes = minfos; if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) { dDebug("mnode module is already started, module status:%d", tsModuleStatus); diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 3bc2f7b48b..c6fc2b9e36 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -28,8 +28,8 @@ #include "dnodeMgmt.h" #include "dnodeVWrite.h" #include "dnodeMPeer.h" +#include "dnodeMInfos.h" -extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); @@ -38,10 +38,10 @@ static void *tsDnodeServerRpc = NULL; static void *tsDnodeClientRpc = NULL; int32_t dnodeInitServer() { - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVnodeWriteQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVnodeWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = dnodeDispatchToVWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = dnodeDispatchToVWriteQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = dnodeDispatchToVWriteQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToMgmtQueue; @@ -72,7 +72,7 @@ int32_t dnodeInitServer() { return -1; } - dInfo("inter-dnodes RPC server is opened"); + dInfo("dnode inter-dnodes RPC server is initialized"); return 0; } @@ -137,7 +137,7 @@ int32_t dnodeInitClient() { return -1; } - dInfo("inter-dnodes rpc client is opened"); + dInfo("dnode inter-dnodes rpc client is initialized"); return 0; } @@ -145,13 +145,13 @@ void dnodeCleanupClient() { if (tsDnodeClientRpc) { rpcClose(tsDnodeClientRpc); tsDnodeClientRpc = NULL; - dInfo("inter-dnodes rpc client is closed"); + dInfo("dnode inter-dnodes rpc client is closed"); } } static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { - dnodeUpdateMnodeEpSetForPeer(pEpSet); + dnodeUpdateEpSetForPeer(pEpSet); } if (dnodeProcessRspMsgFp[pMsg->msgType]) { @@ -173,7 +173,7 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { SRpcEpSet epSet = {0}; - dnodeGetMnodeEpSetForPeer(&epSet); + dnodeGetEpSetForPeer(&epSet); rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp); } diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 4c6c2100e0..33cda60793 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -38,10 +38,10 @@ static int32_t tsDnodeQueryReqNum = 0; static int32_t tsDnodeSubmitReqNum = 0; int32_t dnodeInitShell() { - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVnodeReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVnodeWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; // the following message shall be treated as mnode write dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue; @@ -97,7 +97,7 @@ int32_t dnodeInitShell() { return -1; } - dInfo("shell rpc server is opened"); + dInfo("dnode shell rpc server is initialized"); return 0; } @@ -146,12 +146,12 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; - SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg)); + SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); tstrncpy(pMsg->user, user, sizeof(pMsg->user)); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pMsg; - rpcMsg.contLen = sizeof(SDMAuthMsg); + rpcMsg.contLen = sizeof(SAuthMsg); rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; dDebug("user:%s, send auth msg to mnodes", user); @@ -161,7 +161,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char if (rpcRsp.code != 0) { dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); } else { - SDMAuthRsp *pRsp = rpcRsp.pCont; + SAuthRsp *pRsp = rpcRsp.pCont; dDebug("user:%s, auth msg received from mnodes", user); memcpy(secret, pRsp->secret, TSDB_KEY_LEN); memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); @@ -176,8 +176,8 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid); - int32_t contLen = sizeof(SDMConfigTableMsg); - SDMConfigTableMsg *pMsg = rpcMallocCont(contLen); + int32_t contLen = sizeof(SConfigTableMsg); + SConfigTableMsg *pMsg = rpcMallocCont(contLen); pMsg->dnodeId = htonl(dnodeGetDnodeId()); pMsg->vgId = htonl(vgId); diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 4fdc0b8a73..e973f9901f 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -268,7 +268,7 @@ static void dnodeGetEmail(char* filepath) { return; } - if (taosTRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { + if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } @@ -299,6 +299,7 @@ int32_t dnodeInitTelemetry() { dTrace("failed to create telemetry thread, reason:%s", strerror(errno)); } + dInfo("dnode telemetry is initialized"); return 0; } diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index e61158ef30..abf3cb527d 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -61,7 +61,7 @@ int32_t dnodeInitVnodeRead() { pWorker->workerId = i; } - dInfo("dnode read is opened, min worker:%d max worker:%d", readPool.min, readPool.max); + dInfo("dnode read is initialized, min worker:%d max worker:%d", readPool.min, readPool.max); return 0; } @@ -132,7 +132,7 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { } } -void *dnodeAllocateVnodeRqueue(void *pVnode) { +void *dnodeAllocVReadQueue(void *pVnode) { pthread_mutex_lock(&readPool.mutex); taos_queue queue = taosOpenQueue(); if (queue == NULL) { @@ -167,7 +167,7 @@ void *dnodeAllocateVnodeRqueue(void *pVnode) { return queue; } -void dnodeFreeVnodeRqueue(void *rqueue) { +void dnodeFreeVReadQueue(void *rqueue) { taosCloseQueue(rqueue); // dynamically adjust the number of threads diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index f2740bf6b8..3a820f180c 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -15,74 +15,65 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" -#include "taoserror.h" -#include "tutil.h" +#include "tglobal.h" #include "tqueue.h" -#include "trpc.h" #include "tsdb.h" #include "twal.h" -#include "tdataformat.h" -#include "tglobal.h" #include "tsync.h" #include "vnode.h" -#include "dnodeInt.h" #include "syncInt.h" -#include "dnodeVWrite.h" -#include "dnodeMgmt.h" +#include "dnodeInt.h" typedef struct { - taos_qall qall; - taos_qset qset; // queue set - pthread_t thread; // thread - int32_t workerId; // worker ID + taos_qall qall; + taos_qset qset; // queue set + int32_t workerId; // worker ID + pthread_t thread; // thread } SWriteWorker; typedef struct { - SRspRet rspRet; - int32_t processedCount; - int32_t code; - void *pCont; - int32_t contLen; - SRpcMsg rpcMsg; + SRspRet rspRet; + SRpcMsg rpcMsg; + int32_t processedCount; + int32_t code; + int32_t contLen; + void * pCont; } SWriteMsg; typedef struct { - int32_t max; // max number of workers - int32_t nextId; // from 0 to max-1, cyclic - SWriteWorker *writeWorker; + int32_t max; // max number of workers + int32_t nextId; // from 0 to max-1, cyclic + SWriteWorker *worker; pthread_mutex_t mutex; } SWriteWorkerPool; +static SWriteWorkerPool tsVWriteWP; static void *dnodeProcessWriteQueue(void *param); -static void dnodeHandleIdleWorker(SWriteWorker *pWorker); -SWriteWorkerPool wWorkerPool; +int32_t dnodeInitVWrite() { + tsVWriteWP.max = tsNumOfCores; + tsVWriteWP.worker = (SWriteWorker *)tcalloc(sizeof(SWriteWorker), tsVWriteWP.max); + if (tsVWriteWP.worker == NULL) return -1; + pthread_mutex_init(&tsVWriteWP.mutex, NULL); -int32_t dnodeInitVnodeWrite() { - wWorkerPool.max = tsNumOfCores; - wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max); - if (wWorkerPool.writeWorker == NULL) return -1; - pthread_mutex_init(&wWorkerPool.mutex, NULL); - - for (int32_t i = 0; i < wWorkerPool.max; ++i) { - wWorkerPool.writeWorker[i].workerId = i; + for (int32_t i = 0; i < tsVWriteWP.max; ++i) { + tsVWriteWP.worker[i].workerId = i; } - dInfo("dnode write is opened, max worker %d", wWorkerPool.max); + dInfo("dnode vwrite is initialized, max worker %d", tsVWriteWP.max); return 0; } -void dnodeCleanupVnodeWrite() { - for (int32_t i = 0; i < wWorkerPool.max; ++i) { - SWriteWorker *pWorker = wWorkerPool.writeWorker + i; +void dnodeCleanupVWrite() { + for (int32_t i = 0; i < tsVWriteWP.max; ++i) { + SWriteWorker *pWorker = tsVWriteWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(pWorker->qset); } } - for (int32_t i = 0; i < wWorkerPool.max; ++i) { - SWriteWorker *pWorker = wWorkerPool.writeWorker + i; + for (int32_t i = 0; i < tsVWriteWP.max; ++i) { + SWriteWorker *pWorker = tsVWriteWP.worker + i; if (pWorker->thread) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); @@ -90,13 +81,13 @@ void dnodeCleanupVnodeWrite() { } } - pthread_mutex_destroy(&wWorkerPool.mutex); - free(wWorkerPool.writeWorker); - dInfo("dnode write is closed"); + pthread_mutex_destroy(&tsVWriteWP.mutex); + tfree(tsVWriteWP.worker); + dInfo("dnode vwrite is closed"); } -void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) { - char *pCont = (char *)pMsg->pCont; +void dnodeDispatchToVWriteQueue(SRpcMsg *pMsg) { + char *pCont = pMsg->pCont; if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { SMsgDesc *pDesc = (SMsgDesc *)pCont; @@ -111,7 +102,7 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) { taos_queue queue = vnodeAcquireWqueue(pHead->vgId); if (queue) { // put message into queue - SWriteMsg *pWrite = (SWriteMsg *)taosAllocateQitem(sizeof(SWriteMsg)); + SWriteMsg *pWrite = taosAllocateQitem(sizeof(SWriteMsg)); pWrite->rpcMsg = *pMsg; pWrite->pCont = pCont; pWrite->contLen = pHead->contLen; @@ -130,12 +121,12 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) { } } -void *dnodeAllocateVnodeWqueue(void *pVnode) { - pthread_mutex_lock(&wWorkerPool.mutex); - SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId; +void *dnodeAllocVWriteQueue(void *pVnode) { + pthread_mutex_lock(&tsVWriteWP.mutex); + SWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId; void *queue = taosOpenQueue(); if (queue == NULL) { - pthread_mutex_unlock(&wWorkerPool.mutex); + pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; } @@ -143,7 +134,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { pWorker->qset = taosOpenQset(); if (pWorker->qset == NULL) { taosCloseQueue(queue); - pthread_mutex_unlock(&wWorkerPool.mutex); + pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; } @@ -152,7 +143,7 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { if (pWorker->qall == NULL) { taosCloseQset(pWorker->qset); taosCloseQueue(queue); - pthread_mutex_unlock(&wWorkerPool.mutex); + pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; } pthread_attr_t thAttr; @@ -160,37 +151,35 @@ void *dnodeAllocateVnodeWqueue(void *pVnode) { pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessWriteQueue, pWorker) != 0) { - dError("failed to create thread to process read queue, reason:%s", strerror(errno)); + dError("failed to create thread to process vwrite queue since %s", strerror(errno)); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); taosCloseQueue(queue); queue = NULL; } else { - dDebug("write worker:%d is launched", pWorker->workerId); - wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; + dDebug("dnode vwrite worker:%d is launched", pWorker->workerId); + tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max; } pthread_attr_destroy(&thAttr); } else { taosAddIntoQset(pWorker->qset, queue, pVnode); - wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max; + tsVWriteWP.nextId = (tsVWriteWP.nextId + 1) % tsVWriteWP.max; } - pthread_mutex_unlock(&wWorkerPool.mutex); - dDebug("pVnode:%p, write queue:%p is allocated", pVnode, queue); + pthread_mutex_unlock(&tsVWriteWP.mutex); + dDebug("pVnode:%p, dnode vwrite queue:%p is allocated", pVnode, queue); return queue; } -void dnodeFreeVnodeWqueue(void *wqueue) { +void dnodeFreeVWriteQueue(void *wqueue) { taosCloseQueue(wqueue); - - // dynamically adjust the number of threads } -void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) { - SWriteMsg *pWrite = (SWriteMsg *)param; - if (pWrite == NULL) return; +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code) { + if (param == NULL) return; + SWriteMsg *pWrite = param; if (code < 0) pWrite->code = code; int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); @@ -215,44 +204,45 @@ static void *dnodeProcessWriteQueue(void *param) { SWriteWorker *pWorker = (SWriteWorker *)param; SWriteMsg * pWrite; SWalHead * pHead; - int32_t numOfMsgs; - int type; - void * pVnode, *item; SRspRet * pRspRet; + void * pVnode; + void * pItem; + int32_t numOfMsgs; + int32_t qtype; - dDebug("write worker:%d is running", pWorker->workerId); + dDebug("dnode vwrite worker:%d is running", pWorker->workerId); while (1) { numOfMsgs = taosReadAllQitemsFromQset(pWorker->qset, pWorker->qall, &pVnode); if (numOfMsgs == 0) { - dDebug("qset:%p, dnode write got no message from qset, exiting", pWorker->qset); + dDebug("qset:%p, dnode vwrite got no message from qset, exiting", pWorker->qset); break; } for (int32_t i = 0; i < numOfMsgs; ++i) { pWrite = NULL; pRspRet = NULL; - taosGetQitem(pWorker->qall, &type, &item); - if (type == TAOS_QTYPE_RPC) { - pWrite = (SWriteMsg *)item; + taosGetQitem(pWorker->qall, &qtype, &pItem); + if (qtype == TAOS_QTYPE_RPC) { + pWrite = pItem; pRspRet = &pWrite->rspRet; - pHead = (SWalHead *)(pWrite->pCont - sizeof(SWalHead)); + pHead = (SWalHead *)((char *)pWrite->pCont - sizeof(SWalHead)); pHead->msgType = pWrite->rpcMsg.msgType; pHead->version = 0; pHead->len = pWrite->contLen; dDebug("%p, rpc msg:%s will be processed in vwrite queue", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); - } else if (type == TAOS_QTYPE_CQ) { - pHead = (SWalHead *)((char*)item + sizeof(SSyncHead)); + } else if (qtype == TAOS_QTYPE_CQ) { + pHead = (SWalHead *)((char *)pItem + sizeof(SSyncHead)); dTrace("%p, CQ wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], pHead->version); } else { - pHead = (SWalHead *)item; + pHead = pItem; dTrace("%p, wal msg:%s will be processed in vwrite queue, version:%" PRIu64, pHead, taosMsg[pHead->msgType], pHead->version); } - int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet); + int32_t code = vnodeProcessWrite(pVnode, qtype, pHead, pRspRet); dTrace("%p, msg:%s is processed in vwrite queue, version:%" PRIu64 ", result:%s", pHead, taosMsg[pHead->msgType], pHead->version, tstrerror(code)); @@ -267,17 +257,17 @@ static void *dnodeProcessWriteQueue(void *param) { // browse all items, and process them one by one taosResetQitems(pWorker->qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(pWorker->qall, &type, &item); - if (type == TAOS_QTYPE_RPC) { - pWrite = (SWriteMsg *)item; - dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code); - } else if (type == TAOS_QTYPE_FWD) { - pHead = (SWalHead *)item; + taosGetQitem(pWorker->qall, &qtype, &pItem); + if (qtype == TAOS_QTYPE_RPC) { + pWrite = pItem; + dnodeSendRpcVWriteRsp(pVnode, pItem, pWrite->rpcMsg.code); + } else if (qtype == TAOS_QTYPE_FWD) { + pHead = pItem; vnodeConfirmForward(pVnode, pHead->version, 0); - taosFreeQitem(item); + taosFreeQitem(pItem); vnodeRelease(pVnode); } else { - taosFreeQitem(item); + taosFreeQitem(pItem); vnodeRelease(pVnode); } } @@ -285,19 +275,3 @@ static void *dnodeProcessWriteQueue(void *param) { return NULL; } - -UNUSED_FUNC -static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { - int32_t num = taosGetQueueNumber(pWorker->qset); - - if (num > 0) { - usleep(30000); - sched_yield(); - } else { - taosFreeQall(pWorker->qall); - taosCloseQset(pWorker->qset); - pWorker->qset = NULL; - dDebug("write worker:%d is released", pWorker->workerId); - pthread_exit(NULL); - } -} diff --git a/src/inc/dnode.h b/src/inc/dnode.h index e84545be17..b4973cc672 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "trpc.h" +#include "taosmsg.h" typedef struct { int32_t queryReqNum; @@ -38,12 +39,13 @@ SDnodeRunStatus dnodeGetRunStatus(); SDnodeStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); -char * dnodeGetMnodeMasterEp(); -void dnodeGetMnodeEpSetForPeer(void *epSet); -void dnodeGetMnodeEpSetForShell(void *epSet); -void * dnodeGetMnodeInfos(); +bool dnodeIsMasterEp(char *ep); +void dnodeGetEpSetForPeer(SRpcEpSet *epSet); +void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); -bool dnodeStartMnode(void *pModes); +void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); +bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); +bool dnodeStartMnode(SMnodeInfos *minfos); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); @@ -51,11 +53,11 @@ void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid); -void *dnodeAllocateVnodeWqueue(void *pVnode); -void dnodeFreeVnodeWqueue(void *queue); -void *dnodeAllocateVnodeRqueue(void *pVnode); -void dnodeFreeVnodeRqueue(void *rqueue); -void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code); +void *dnodeAllocVWriteQueue(void *pVnode); +void dnodeFreeVWriteQueue(void *wqueue); +void *dnodeAllocVReadQueue(void *pVnode); +void dnodeFreeVReadQueue(void *rqueue); +void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); int32_t dnodeAllocateMnodePqueue(); void dnodeFreeMnodePqueue(); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index ddc72eab14..8139c9ab17 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -295,7 +295,7 @@ typedef struct { SSchema schema[]; // tagVal is padded after schema // char tagVal[]; -} SCMAlterTableMsg; +} SAlterTableMsg; typedef struct { SMsgHead head; @@ -312,12 +312,12 @@ typedef struct { } SUpdateTableTagValMsg; typedef struct { - char clientVersion[TSDB_VERSION_LEN]; - char msgVersion[TSDB_VERSION_LEN]; - char db[TSDB_TABLE_FNAME_LEN]; - char appName[TSDB_APPNAME_LEN]; + char clientVersion[TSDB_VERSION_LEN]; + char msgVersion[TSDB_VERSION_LEN]; + char db[TSDB_TABLE_FNAME_LEN]; + char appName[TSDB_APPNAME_LEN]; int32_t pid; -} SCMConnectMsg; +} SConnectMsg; typedef struct { char acctId[TSDB_ACCT_LEN]; @@ -328,7 +328,7 @@ typedef struct { int8_t reserved2; int32_t connId; SRpcEpSet epSet; -} SCMConnectRsp; +} SConnectRsp; typedef struct { int32_t maxUsers; @@ -348,18 +348,18 @@ typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_KEY_LEN]; SAcctCfg cfg; -} SCMCreateAcctMsg, SCMAlterAcctMsg; +} SCreateAcctMsg, SAlterAcctMsg; typedef struct { char user[TSDB_USER_LEN]; -} SCMDropUserMsg, SCMDropAcctMsg; +} SDropUserMsg, SDropAcctMsg; typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_KEY_LEN]; int8_t privilege; int8_t flag; -} SCMCreateUserMsg, SCMAlterUserMsg; +} SCreateUserMsg, SAlterUserMsg; typedef struct { int32_t contLen; @@ -374,11 +374,11 @@ typedef struct { int32_t vgId; uint64_t uid; char tableId[TSDB_TABLE_FNAME_LEN]; -} SMDDropSTableMsg; +} SDropSTableMsg; typedef struct { int32_t vgId; -} SMDDropVnodeMsg; +} SDropVnodeMsg; typedef struct SColIndex { int16_t colId; // column id @@ -544,12 +544,12 @@ typedef struct { int8_t replications; int8_t quorum; int8_t ignoreExist; -} SCMCreateDbMsg, SCMAlterDbMsg; +} SCreateDbMsg, SAlterDbMsg; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; uint8_t ignoreNotExists; -} SCMDropDbMsg, SCMUseDbMsg; +} SDropDbMsg, SUseDbMsg; // IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed // TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE @@ -564,7 +564,7 @@ typedef struct { typedef struct { int32_t vgId; int8_t accessState; -} SDMVgroupAccess; +} SVgroupAccess; typedef struct { int32_t dnodeId; @@ -572,18 +572,29 @@ typedef struct { uint32_t numOfVnodes; char clusterId[TSDB_CLUSTER_ID_LEN]; char reserved[16]; -} SDMDnodeCfg; +} SDnodeCfg; typedef struct { - int32_t nodeId; - char nodeEp[TSDB_EP_LEN]; -} SDMMnodeInfo; + int32_t dnodeId; + uint16_t dnodePort; + char dnodeFqdn[TSDB_FQDN_LEN]; +} SDnodeEp; typedef struct { - int8_t inUse; - int8_t nodeNum; - SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA]; -} SDMMnodeInfos; + int32_t dnodeNum; + SDnodeEp dnodeEps[]; +} SDnodeEps; + +typedef struct { + int32_t mnodeId; + char mnodeEp[TSDB_EP_LEN]; +} SMnodeInfo; + +typedef struct { + int8_t inUse; + int8_t mnodeNum; + SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA]; +} SMnodeInfos; typedef struct { int32_t numOfMnodes; // tsNumOfMnodes @@ -615,13 +626,13 @@ typedef struct { uint8_t reserve2[15]; SClusterCfg clusterCfg; SVnodeLoad load[]; -} SDMStatusMsg; +} SStatusMsg; typedef struct { - SDMMnodeInfos mnodes; - SDMDnodeCfg dnodeCfg; - SDMVgroupAccess vgAccess[]; -} SDMStatusRsp; + SMnodeInfos mnodes; + SDnodeCfg dnodeCfg; + SVgroupAccess vgAccess[]; +} SStatusRsp; typedef struct { uint32_t vgId; @@ -644,54 +655,54 @@ typedef struct { int8_t wals; int8_t quorum; int8_t reserved[16]; -} SMDVnodeCfg; +} SVnodeCfg; typedef struct { int32_t nodeId; char nodeEp[TSDB_EP_LEN]; -} SMDVnodeDesc; +} SVnodeDesc; typedef struct { - char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; - SMDVnodeCfg cfg; - SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; -} SMDCreateVnodeMsg, SMDAlterVnodeMsg; + char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; + SVnodeCfg cfg; + SVnodeDesc nodes[TSDB_MAX_REPLICA]; +} SCreateVnodeMsg, SAlterVnodeMsg; typedef struct { char tableId[TSDB_TABLE_FNAME_LEN]; int16_t createFlag; char tags[]; -} SCMTableInfoMsg; +} STableInfoMsg; typedef struct { int32_t numOfTables; char tableIds[]; -} SCMMultiTableInfoMsg; +} SMultiTableInfoMsg; -typedef struct SCMSTableVgroupMsg { +typedef struct SSTableVgroupMsg { int32_t numOfTables; -} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; +} SSTableVgroupMsg, SSTableVgroupRspMsg; typedef struct { int32_t vgId; int8_t numOfEps; SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCMVgroupInfo; +} SVgroupInfo; typedef struct { int32_t vgId; int8_t numOfEps; SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; -} SCMVgroupMsg; +} SVgroupMsg; typedef struct { int32_t numOfVgroups; - SCMVgroupInfo vgroups[]; + SVgroupInfo vgroups[]; } SVgroupsInfo; typedef struct { int32_t numOfVgroups; - SCMVgroupMsg vgroups[]; + SVgroupMsg vgroups[]; } SVgroupsMsg; typedef struct STableMetaMsg { @@ -706,7 +717,7 @@ typedef struct STableMetaMsg { int16_t tversion; int32_t tid; uint64_t uid; - SCMVgroupMsg vgroup; + SVgroupMsg vgroup; SSchema schema[]; } STableMetaMsg; @@ -732,38 +743,38 @@ typedef struct { char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; uint16_t payloadLen; char payload[]; -} SCMShowMsg; +} SShowMsg; -typedef struct SCMShowRsp { +typedef struct SShowRsp { uint64_t qhandle; STableMetaMsg tableMeta; -} SCMShowRsp; +} SShowRsp; typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port -} SCMCreateDnodeMsg, SCMDropDnodeMsg; + char ep[TSDB_EP_LEN]; // end point, hostname:port +} SCreateDnodeMsg, SDropDnodeMsg; typedef struct { int32_t dnodeId; char dnodeEp[TSDB_EP_LEN]; // end point, hostname:port - SDMMnodeInfos mnodes; -} SMDCreateMnodeMsg; + SMnodeInfos mnodes; +} SCreateMnodeMsg; typedef struct { int32_t dnodeId; int32_t vgId; int32_t tid; -} SDMConfigTableMsg; +} SConfigTableMsg; typedef struct { uint32_t dnodeId; int32_t vgId; -} SDMConfigVnodeMsg; +} SConfigVnodeMsg; typedef struct { char ep[TSDB_EP_LEN]; // end point, hostname:port char config[64]; -} SMDCfgDnodeMsg, SCMCfgDnodeMsg; +} SCfgDnodeMsg; typedef struct { char sql[TSDB_SHOW_SQL_LEN]; @@ -792,7 +803,7 @@ typedef struct { int32_t numOfStreams; char appName[TSDB_APPNAME_LEN]; char pData[]; -} SCMHeartBeatMsg; +} SHeartBeatMsg; typedef struct { uint32_t queryId; @@ -802,11 +813,11 @@ typedef struct { uint32_t connId; int8_t killConnection; SRpcEpSet epSet; -} SCMHeartBeatRsp; +} SHeartBeatRsp; typedef struct { char queryId[TSDB_KILL_MSG_LEN + 1]; -} SCMKillQueryMsg, SCMKillStreamMsg, SCMKillConnMsg; +} SKillQueryMsg, SKillStreamMsg, SKillConnMsg; typedef struct { int32_t vnode; @@ -815,7 +826,7 @@ typedef struct { uint64_t stime; // stream starting time int32_t status; char tableId[TSDB_TABLE_FNAME_LEN]; -} SMDAlterStreamMsg; +} SAlterStreamMsg; typedef struct { char user[TSDB_USER_LEN]; @@ -823,7 +834,7 @@ typedef struct { char encrypt; char secret[TSDB_KEY_LEN]; char ckey[TSDB_KEY_LEN]; -} SDMAuthMsg, SDMAuthRsp; +} SAuthMsg, SAuthRsp; #pragma pack(pop) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index ca0f70d104..671adefab8 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -68,7 +68,7 @@ typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uin // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int (*FGetWalInfo)(void *ahandle, char *name, uint32_t *index); +typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); // when a forward pkt is received, call this to handle data typedef int (*FWriteToCache)(void *ahandle, void *pHead, int type); diff --git a/src/inc/twal.h b/src/inc/twal.h index 1ce7b132b0..3a229ed835 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -19,9 +19,11 @@ extern "C" { #endif -#define TAOS_WAL_NOLOG 0 -#define TAOS_WAL_WRITE 1 -#define TAOS_WAL_FSYNC 2 +typedef enum { + TAOS_WAL_NOLOG = 0, + TAOS_WAL_WRITE = 1, + TAOS_WAL_FSYNC = 2 +} EWalType; typedef struct { int8_t msgType; @@ -34,8 +36,9 @@ typedef struct { } SWalHead; typedef struct { - int8_t walLevel; // wal level + int32_t vgId; int32_t fsyncPeriod; // millisecond + int8_t walLevel; // wal level int8_t wals; // number of WAL files; int8_t keep; // keep the wal file when closed } SWalCfg; @@ -43,14 +46,18 @@ typedef struct { typedef void* twalh; // WAL HANDLE typedef int (*FWalWrite)(void *ahandle, void *pHead, int type); -twalh walOpen(const char *path, const SWalCfg *pCfg); -int walAlter(twalh pWal, const SWalCfg *pCfg); +int32_t walInit(); +void walCleanUp(); + +twalh walOpen(char *path, SWalCfg *pCfg); +int32_t walAlter(twalh pWal, SWalCfg *pCfg); +void walStop(twalh); void walClose(twalh); -int walRenew(twalh); -int walWrite(twalh, SWalHead *); +int32_t walRenew(twalh); +int32_t walWrite(twalh, SWalHead *); void walFsync(twalh); -int walRestore(twalh, void *pVnode, FWalWrite writeFp); -int walGetWalFile(twalh, char *name, uint32_t *index); +int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); +int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); int64_t walGetVersion(twalh); #ifdef __cplusplus diff --git a/src/inc/vnode.h b/src/inc/vnode.h index fdce4d6279..8c387065cc 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -43,10 +43,10 @@ typedef struct { extern char *vnodeStatus[]; -int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg); +int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId, char *rootDir); -int32_t vnodeAlter(void *pVnode, SMDCreateVnodeMsg *pVnodeCfg); +int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); void* vnodeAcquire(int32_t vgId); // add refcount @@ -60,7 +60,7 @@ int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); void vnodeConfirmForward(void *param, uint64_t version, int32_t code); -void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes); +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); int32_t vnodeInitResources(); void vnodeCleanupResources(); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 748b7e7929..9b166f9351 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -244,7 +244,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { } *p++ = c; - if (c == ';') { + if (c == ';' && quote == 0) { c = *p; *p = 0; if (shellRunSingleCommand(con, cmd) < 0) { diff --git a/src/mnode/inc/mnodeProfile.h b/src/mnode/inc/mnodeProfile.h index 1e5b1c0f9c..ee57c5da1f 100644 --- a/src/mnode/inc/mnodeProfile.h +++ b/src/mnode/inc/mnodeProfile.h @@ -45,7 +45,7 @@ void mnodeCleanupProfile(); SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port, int32_t pid, const char* app); SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port); void mnodeReleaseConn(SConnObj *pConn); -int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg); +int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8d7c267ab7..748b856ba4 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -41,7 +41,7 @@ static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg); +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg); static int32_t mnodeDropDb(SMnodeMsg *newMsg); static int32_t mnodeSetDbDropping(SDbObj *pDb); static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -352,7 +352,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { return code; } -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) { +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); if (code != 0) return code; @@ -805,7 +805,7 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) { } static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) { - SCMCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; pCreate->maxTables = htonl(pCreate->maxTables); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); @@ -830,7 +830,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) { return code; } -static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { +static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { SDbCfg newCfg = pDb->cfg; int32_t maxTables = htonl(pAlter->maxTables); int32_t cacheBlockSize = htonl(pAlter->cacheBlockSize); @@ -977,7 +977,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { +static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter); if (terrno != TSDB_CODE_SUCCESS) { return terrno; @@ -1009,7 +1009,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter, void *pMsg) { } static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) { - SCMAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; + SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db); @@ -1060,7 +1060,7 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) { } static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { - SCMDropDbMsg *pDrop = pMsg->rpcMsg.pCont; + SDropDbMsg *pDrop = pMsg->rpcMsg.pCont; mDebug("db:%s, drop db msg is received from thandle:%p", pDrop->db, pMsg->rpcMsg.handle); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 1cd861e223..bf8647c193 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,11 +39,15 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; -static void *tsDnodeSdb = NULL; +static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; extern void * tsVgroupSdb; +static SDnodeEps*tsDnodeEps; +static int32_t tsDnodeEpsSize; +static pthread_mutex_t tsDnodeEpsMutex; + static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg); static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); @@ -59,6 +63,7 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole); +static void mnodeUpdateDnodeEps(); static char* offlineReason[] = { "", @@ -95,6 +100,9 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) { pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } + dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); + mnodeUpdateDnodeEps(); + mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -107,6 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) { #endif mnodeDropMnodeLocal(pDnode->dnodeId); balanceAsyncNotify(); + mnodeUpdateDnodeEps(); mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId); return TSDB_CODE_SUCCESS; @@ -121,6 +130,7 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) { } mnodeDecDnodeRef(pDnode); + mnodeUpdateDnodeEps(); return TSDB_CODE_SUCCESS; } @@ -152,12 +162,14 @@ static int32_t mnodeDnodeActionRestored() { } } + mnodeUpdateDnodeEps(); return TSDB_CODE_SUCCESS; } int32_t mnodeInitDnodes() { SDnodeObj tObj; tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + pthread_mutex_init(&tsDnodeEpsMutex, NULL); SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_DNODE, @@ -201,6 +213,9 @@ int32_t mnodeInitDnodes() { void mnodeCleanupDnodes() { sdbCloseTable(tsDnodeSdb); + pthread_mutex_destroy(&tsDnodeEpsMutex); + free(tsDnodeEps); + tsDnodeEps = NULL; tsDnodeSdb = NULL; } @@ -299,7 +314,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_NO_RIGHTS; } - SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; + SCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; if (pCmCfgDnode->ep[0] == 0) { tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN); } @@ -334,7 +349,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { mnodeDecDnodeRef(pDnode); return code; } else { - SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); + SCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); strcpy(pMdCfgDnode->config, pCmCfgDnode->config); @@ -343,7 +358,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { .code = 0, .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, .pCont = pMdCfgDnode, - .contLen = sizeof(SMDCfgDnodeMsg) + .contLen = sizeof(SCfgDnodeMsg) }; mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); @@ -418,9 +433,54 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { return 0; } +static int32_t mnodeGetDnodeEpsSize() { + pthread_mutex_lock(&tsDnodeEpsMutex); + int32_t size = tsDnodeEpsSize; + pthread_mutex_unlock(&tsDnodeEpsMutex); + return size; +} + +static void mnodeGetDnodeEpsData(SDnodeEps *pEps, int32_t epsSize) { + pthread_mutex_lock(&tsDnodeEpsMutex); + if (epsSize == tsDnodeEpsSize) { + memcpy(pEps, tsDnodeEps, tsDnodeEpsSize); + } + pthread_mutex_unlock(&tsDnodeEpsMutex); +} + +static void mnodeUpdateDnodeEps() { + pthread_mutex_lock(&tsDnodeEpsMutex); + + int32_t totalDnodes = mnodeGetDnodesNum(); + tsDnodeEpsSize = sizeof(SDnodeEps) + totalDnodes * sizeof(SDnodeEp); + free(tsDnodeEps); + tsDnodeEps = calloc(1, tsDnodeEpsSize); + tsDnodeEps->dnodeNum = htonl(totalDnodes); + + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + int32_t dnodesNum = 0; + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + if (dnodesNum >= totalDnodes) break; + + SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum]; + dnodesNum++; + pEp->dnodeId = htonl(pDnode->dnodeId); + pEp->dnodePort = htons(pDnode->dnodePort); + tstrncpy(pEp->dnodeFqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + mnodeDecDnodeRef(pDnode); + } + + sdbFreeIter(pIter); + pthread_mutex_unlock(&tsDnodeEpsMutex); +} + static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = NULL; - SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont; + SStatusMsg *pStatus = pMsg->rpcMsg.pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->moduleStatus = htonl(pStatus->moduleStatus); pStatus->lastReboot = htonl(pStatus->lastReboot); @@ -477,8 +537,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } int32_t openVnodes = htons(pStatus->openVnodes); - int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess); - SDMStatusRsp *pRsp = rpcMallocCont(contLen); + int32_t epsSize = mnodeGetDnodeEpsSize(); + int32_t vgAccessSize = openVnodes * sizeof(SVgroupAccess); + int32_t contLen = sizeof(SStatusRsp) + vgAccessSize + epsSize; + + SStatusRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mnodeDecDnodeRef(pDnode); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -488,8 +551,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt); pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes); tstrncpy(pRsp->dnodeCfg.clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN); - SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp)); - + SVgroupAccess *pAccess = (SVgroupAccess *)((char *)pRsp + sizeof(SStatusRsp)); + for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); @@ -539,6 +602,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { mnodeDecDnodeRef(pDnode); + SDnodeEps *pEps = (SDnodeEps *)((char *)pRsp + sizeof(SStatusRsp) + vgAccessSize); + mnodeGetDnodeEpsData(pEps, epsSize); + pMsg->rpcRsp.len = contLen; pMsg->rpcRsp.rsp = pRsp; @@ -645,7 +711,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { } static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { - SCMCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) { return TSDB_CODE_MND_NO_RIGHTS; @@ -655,7 +721,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg) { - SCMDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; if (strcmp(pMsg->pUser->user, TSDB_DEFAULT_USER) != 0) { return TSDB_CODE_MND_NO_RIGHTS; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 89b2f50b73..5a1825a2a3 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -38,7 +38,7 @@ static void * tsMnodeSdb = NULL; static int32_t tsMnodeUpdateSize = 0; static SRpcEpSet tsMnodeEpSetForShell; static SRpcEpSet tsMnodeEpSetForPeer; -static SDMMnodeInfos tsMnodeInfos; +static SMnodeInfos tsMnodeInfos; static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -70,8 +70,9 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) { pDnode->isMgmt = true; mnodeDecDnodeRef(pDnode); - - mInfo("mnode:%d, fqdn:%s ep:%s port:%d, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); + + mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, + pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -202,7 +203,7 @@ void mnodeUpdateMnodeEpSet() { memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet)); memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet)); - memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos)); + memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos)); int32_t index = 0; void * pIter = NULL; @@ -221,8 +222,8 @@ void mnodeUpdateMnodeEpSet() { tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index])); - tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId); - strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp); + tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId); + strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { tsMnodeEpSetForShell.inUse = index; @@ -238,7 +239,7 @@ void mnodeUpdateMnodeEpSet() { mnodeDecMnodeRef(pMnode); } - tsMnodeInfos.nodeNum = index; + tsMnodeInfos.mnodeNum = index; tsMnodeEpSetForShell.numOfEps = index; tsMnodeEpSetForPeer.numOfEps = index; @@ -260,19 +261,19 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { } char* mnodeGetMnodeMasterEp() { - return tsMnodeInfos.nodeInfos[tsMnodeInfos.inUse].nodeEp; + return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp; } void mnodeGetMnodeInfos(void *mnodeInfos) { mnodeMnodeRdLock(); - *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos; + *(SMnodeInfos *)mnodeInfos = tsMnodeInfos; mnodeMnodeUnLock(); } static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp); - SMDCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SMDCreateMnodeMsg)); + SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg)); if (pCreate == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } else { @@ -280,21 +281,21 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); pCreate->mnodes = tsMnodeInfos; bool found = false; - for (int i = 0; i < pCreate->mnodes.nodeNum; ++i) { - if (pCreate->mnodes.nodeInfos[i].nodeId == htonl(dnodeId)) { + for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) { + if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) { found = true; } } if (!found) { - pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeId = htonl(dnodeId); - tstrncpy(pCreate->mnodes.nodeInfos[pCreate->mnodes.nodeNum].nodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); - pCreate->mnodes.nodeNum++; + pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeId = htonl(dnodeId); + tstrncpy(pCreate->mnodes.mnodeInfos[pCreate->mnodes.mnodeNum].mnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); + pCreate->mnodes.mnodeNum++; } } SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pCreate; - rpcMsg.contLen = sizeof(SMDCreateMnodeMsg); + rpcMsg.contLen = sizeof(SCreateMnodeMsg); rpcMsg.msgType = TSDB_MSG_TYPE_MD_CREATE_MNODE; SRpcMsg rpcRsp = {0}; diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index c29d1ec0b7..9831a95c5f 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -280,7 +280,7 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi } // not thread safe, need optimized -int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SCMHeartBeatMsg *pHBMsg) { +int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { pConn->numOfQueries = htonl(pHBMsg->numOfQueries); if (pConn->numOfQueries > 0) { if (pConn->pQueries == NULL) { @@ -561,7 +561,7 @@ static int32_t mnodeProcessKillQueryMsg(SMnodeMsg *pMsg) { SUserObj *pUser = pMsg->pUser; if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; - SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; mInfo("kill query msg is received, queryId:%s", pKill->queryId); const char delim = ':'; @@ -592,7 +592,7 @@ static int32_t mnodeProcessKillStreamMsg(SMnodeMsg *pMsg) { SUserObj *pUser = pMsg->pUser; if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; - SCMKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; mInfo("kill stream msg is received, streamId:%s", pKill->queryId); const char delim = ':'; @@ -623,7 +623,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) { SUserObj *pUser = pMsg->pUser; if (strcmp(pUser->user, TSDB_DEFAULT_USER) != 0) return TSDB_CODE_MND_NO_RIGHTS; - SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont; + SKillConnMsg *pKill = pMsg->rpcMsg.pCont; int32_t connId = atoi(pKill->queryId); SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); if (pConn == NULL) { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 8c61c61a10..8fb0b33060 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) { } static int32_t sdbInitWal() { - SWalCfg walCfg = {.walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0}; char temp[TSDB_FILENAME_LEN]; sprintf(temp, "%s/wal", tsMnodeDir); tsSdbObj.wal = walOpen(temp, &walCfg); @@ -237,8 +237,8 @@ static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint3 return 0; } -static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) { - return walGetWalFile(tsSdbObj.wal, name, index); +static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { + return walGetWalFile(tsSdbObj.wal, fileName, fileId); } static void sdbNotifyRole(void *ahandle, int8_t role) { @@ -312,7 +312,7 @@ void sdbUpdateAsync() { } void sdbUpdateSync(void *pMnodes) { - SDMMnodeInfos *mnodes = pMnodes; + SMnodeInfos *mnodes = pMnodes; if (!mnodeIsRunning()) { mDebug("mnode not start yet, update sync config later"); return; @@ -346,10 +346,10 @@ void sdbUpdateSync(void *pMnodes) { syncCfg.replica = index; mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { - for (index = 0; index < mnodes->nodeNum; ++index) { - SDMMnodeInfo *node = &mnodes->nodeInfos[index]; - syncCfg.nodeInfo[index].nodeId = node->nodeId; - taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); + for (index = 0; index < mnodes->mnodeNum; ++index) { + SMnodeInfo *node = &mnodes->mnodeInfos[index]; + syncCfg.nodeInfo[index].nodeId = node->mnodeId; + taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC; } syncCfg.replica = index; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index d759352f71..2c9d404e1d 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -110,7 +110,7 @@ static char *mnodeGetShowType(int32_t showType) { } static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { - SCMShowMsg *pShowMsg = pMsg->rpcMsg.pCont; + SShowMsg *pShowMsg = pMsg->rpcMsg.pCont; if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) { return TSDB_CODE_MND_INVALID_MSG_TYPE; } @@ -132,8 +132,8 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; - SCMShowRsp *pShowRsp = rpcMallocCont(size); + int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; + SShowRsp *pShowRsp = rpcMallocCont(size); if (pShowRsp == NULL) { mnodeReleaseShowObj(pShow, true); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -146,7 +146,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { if (code == TSDB_CODE_SUCCESS) { pMsg->rpcRsp.rsp = pShowRsp; - pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns; + pMsg->rpcRsp.len = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; mnodeReleaseShowObj(pShow, false); return TSDB_CODE_SUCCESS; } else { @@ -232,12 +232,12 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { - SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp)); + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp)); if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; + SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { rpcFreeCont(pRsp); return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code @@ -281,15 +281,15 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { mnodeGetMnodeEpSetForShell(&pRsp->epSet); pMsg->rpcRsp.rsp = pRsp; - pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp); + pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); mnodeReleaseConn(pConn); return TSDB_CODE_SUCCESS; } static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { - SCMConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; - SCMConnectRsp *pConnectRsp = NULL; + SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; + SConnectRsp *pConnectRsp = NULL; int32_t code = TSDB_CODE_SUCCESS; SRpcConnInfo connInfo = {0}; @@ -325,7 +325,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { mnodeDecDbRef(pDb); } - pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp)); + pConnectRsp = rpcMallocCont(sizeof(SConnectRsp)); if (pConnectRsp == NULL) { code = TSDB_CODE_MND_OUT_OF_MEMORY; goto connect_over; @@ -354,14 +354,14 @@ connect_over: } else { mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); pMsg->rpcRsp.rsp = pConnectRsp; - pMsg->rpcRsp.len = sizeof(SCMConnectRsp); + pMsg->rpcRsp.len = sizeof(SConnectRsp); } return code; } static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { - SCMUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; + SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; int32_t code = TSDB_CODE_SUCCESS; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 82a062169a..28dc2c6cff 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -778,7 +778,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { - SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->createFlag = htons(pInfo->createFlag); mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); @@ -915,8 +915,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); if (pVgroup == NULL) break; - SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg)); - pDrop->contLen = htonl(sizeof(SMDDropSTableMsg)); + SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg)); + pDrop->contLen = htonl(sizeof(SDropSTableMsg)); pDrop->vgId = htonl(pVgroup->vgId); pDrop->uid = htobe64(pStable->uid); mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); @@ -924,7 +924,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, pVgroup->vgId); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); - SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; + SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } @@ -1472,31 +1472,31 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { } static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { - SCMSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont; + SSTableVgroupMsg *pInfo = pMsg->rpcMsg.pCont; int32_t numOfTable = htonl(pInfo->numOfTables); // reserve space - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg); + int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; + char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { - contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg)); - } - + contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg)); + } + mnodeDecTableRef(pTable); } - SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); + SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } pRsp->numOfTables = 0; - char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg); + char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; + char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); @@ -1548,7 +1548,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { pVgroupMsg->numOfVgroups = htonl(vgSize); // one table is done, try the next table - msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg); + msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SVgroupMsg); pRsp->numOfTables++; } } @@ -2146,7 +2146,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { } static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { - SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; STagData *pTags = (STagData *)pInfo->tags; int32_t tagLen = htonl(pTags->dataLen); if (pTags->name[0] == 0) { @@ -2307,7 +2307,7 @@ static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; #if 0 - SDMConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; + SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); pCfg->sid = htonl(pCfg->sid); @@ -2469,7 +2469,7 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { } static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { - SCMMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->numOfTables = htonl(pInfo->numOfTables); int32_t totalMallocLen = 4 * 1024 * 1024; // first malloc 4 MB, subsequent reallocation as twice @@ -2705,7 +2705,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows } static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { - SCMAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; + SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId, pMsg->rpcMsg.handle); diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 779e252548..8e9d2908c5 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -414,7 +414,7 @@ static int32_t mnodeProcessCreateUserMsg(SMnodeMsg *pMsg) { SUserObj *pOperUser = pMsg->pUser; if (pOperUser->superAuth) { - SCMCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; return mnodeCreateUser(pOperUser->pAcct, pCreate->user, pCreate->pass, pMsg); } else { mError("user:%s, no rights to create user", pOperUser->user); @@ -426,7 +426,7 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg) { int32_t code; SUserObj *pOperUser = pMsg->pUser; - SCMAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; + SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; SUserObj *pUser = mnodeGetUser(pAlter->user); if (pUser == NULL) { return TSDB_CODE_MND_INVALID_USER; @@ -514,7 +514,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { int32_t code; SUserObj *pOperUser = pMsg->pUser; - SCMDropUserMsg *pDrop = pMsg->rpcMsg.pCont; + SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; SUserObj *pUser = mnodeGetUser(pDrop->user); if (pUser == NULL) { return TSDB_CODE_MND_INVALID_USER; @@ -604,11 +604,11 @@ int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, cha } static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { - SDMAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont; - SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp)); + SAuthMsg *pAuthMsg = pMsg->rpcMsg.pCont; + SAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SAuthRsp)); pMsg->rpcRsp.rsp = pAuthRsp; - pMsg->rpcRsp.len = sizeof(SDMAuthRsp); + pMsg->rpcRsp.len = sizeof(SAuthRsp); return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index df703241c2..1d1f21c3d7 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -818,11 +818,11 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { } } -static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { +static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; - SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); + SCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SCreateVnodeMsg)); if (pVnode == NULL) return NULL; strcpy(pVnode->db, pVgroup->dbName); @@ -830,7 +830,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { //TODO: dynamic alloc tables in tsdb maxTables = MAX(10000, tsMaxTablePerVnode); - SMDVnodeCfg *pCfg = &pVnode->cfg; + SVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); pCfg->cfgVersion = htonl(pDb->cfgVersion); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); @@ -851,7 +851,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; - SMDVnodeDesc *pNodes = pVnode->nodes; + SVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode; if (pDnode != NULL) { @@ -886,11 +886,11 @@ SRpcEpSet mnodeGetEpSetFromIp(char *ep) { } static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) { - SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); + SAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = NULL, .pCont = pAlter, - .contLen = pAlter ? sizeof(SMDAlterVnodeMsg) : 0, + .contLen = pAlter ? sizeof(SAlterVnodeMsg) : 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE }; @@ -909,11 +909,11 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { } static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) { - SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); + SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = ahandle, .pCont = pCreate, - .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0, + .contLen = pCreate ? sizeof(SCreateVnodeMsg) : 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE }; @@ -982,8 +982,8 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { } } -static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { - SMDDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SMDDropVnodeMsg)); +static SDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { + SDropVnodeMsg *pDrop = rpcMallocCont(sizeof(SDropVnodeMsg)); if (pDrop == NULL) return NULL; pDrop->vgId = htonl(vgId); @@ -991,11 +991,11 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { } void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) { - SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); + SDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { .ahandle = ahandle, .pCont = pDrop, - .contLen = pDrop ? sizeof(SMDDropVnodeMsg) : 0, + .contLen = pDrop ? sizeof(SDropVnodeMsg) : 0, .code = 0, .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE }; @@ -1044,7 +1044,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { } static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { - SDMConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont; + SConfigVnodeMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 86e16db8b1..9720499004 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -52,9 +52,10 @@ extern "C" { #include "osWindows.h" #endif +#include "osDef.h" +#include "osAlloc.h" #include "osAtomic.h" #include "osCommon.h" -#include "osDef.h" #include "osDir.h" #include "osFile.h" #include "osLz4.h" diff --git a/src/os/inc/osAlloc.h b/src/os/inc/osAlloc.h new file mode 100644 index 0000000000..2d97017480 --- /dev/null +++ b/src/os/inc/osAlloc.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_ALLOC_H +#define TDENGINE_OS_ALLOC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_FUNC_ALLOC + #define tmalloc(size) malloc(size) + #define tcalloc(nmemb, size) calloc(nmemb, size) + #define trealloc(p, size) realloc(p, size) + #define tmemalign(alignment, size) malloc(size) + #define tfree(p) free(p) + #define tmemzero(p, size) memset(p, 0, size) +#else + void *tmalloc(int32_t size); + void *tcalloc(int32_t nmemb, int32_t size); + void *trealloc(void *p, int32_t size); + void *tmemalign(int32_t alignment, int32_t size); + void tfree(void *p); + void tmemzero(void *p, int32_t size); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index c1a950fbe6..7bb844831e 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -72,8 +72,6 @@ extern "C" { #include #define TAOS_OS_FUNC_FILE_SENDIFLE - #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size) - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) #define TAOS_OS_FUNC_SEMPHONE #define tsem_t dispatch_semaphore_t diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index dc19c8177c..62e44d8eb0 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,46 +20,52 @@ extern "C" { #endif -ssize_t taosTReadImp(int fd, void *buf, size_t count); -ssize_t taosTWriteImp(int fd, void *buf, size_t count); +#define tread(fd, buf, count) read(fd, buf, count) +#define twrite(fd, buf, count) write(fd, buf, count) +#define tlseek(fd, offset, whence) lseek(fd, offset, whence) +#define tclose(fd) \ + { \ + if (FD_VALID(fd)) { \ + close(fd); \ + fd = FD_INITIALIZER; \ + } \ + } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); -int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count); +int64_t taosReadImp(int32_t fd, void *buf, int64_t count); +int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); +int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); +int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath); -#ifndef TAOS_OS_FUNC_FILE_SENDIFLE - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) - #define taosFSendFile(outfile, infile, offset, count) taosTSendFileImp(fileno(outfile), fileno(infile), offset, size) -#endif +#define taosRead(fd, buf, count) taosReadImp(fd, buf, count) +#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count) +#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence) +#define taosClose(x) tclose(x) -#define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count) -#define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count) -#define taosLSeek(fd, offset, whence) lseek(fd, offset, whence) +// TAOS_OS_FUNC_FILE_SENDIFLE +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL - void taosSetRandomFileFailFactor(int factor); + void taosSetRandomFileFailFactor(int32_t factor); void taosSetRandomFileFailOutput(const char *path); #ifdef TAOS_RANDOM_FILE_FAIL_TEST - ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); - #undef taosTRead - #undef taosTWrite + int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line); + int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line); + int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line); + #undef taosRead + #undef taosWrite #undef taosLSeek - #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) #endif #endif -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); - // TAOS_OS_FUNC_FILE_GETTMPFILEPATH void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath); -#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE - #define taosFtruncate ftruncate -#endif - +// TAOS_OS_FUNC_FILE_FTRUNCATE +int32_t taosFtruncate(int32_t fd, int64_t length); #ifdef __cplusplus } #endif diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 0ab3ff0fca..cbfdedef48 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,21 +33,19 @@ extern "C" { x = FD_INITIALIZER; \ } \ } - typedef int SOCKET; + typedef int32_t SOCKET; #endif #ifndef TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME -1 #endif -#define taosClose(x) taosCloseSocket(x) - #ifdef TAOS_RANDOM_NETWORK_FAIL #ifdef TAOS_RANDOM_NETWORK_FAIL_TEST - ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); - ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); - ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); - ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); + int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags); + int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, const struct sockaddr *dest_addr, socklen_t addrlen); + int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count); + int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count); #undef taosSend #undef taosSendto #undef taosReadSocket @@ -60,14 +58,14 @@ extern "C" { #endif // TAOS_OS_FUNC_SOCKET -int taosSetNonblocking(SOCKET sock, int on); -void taosBlockSIGPIPE(); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); +void taosBlockSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen); +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); // TAOS_OS_FUNC_SOCKET_INET -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #ifdef __cplusplus diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index dc1da35037..36e30528bf 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -62,11 +62,8 @@ extern "C" { #define TAOS_OS_FUNC_FILE_ISDIR #define TAOS_OS_FUNC_FILE_ISLNK #define TAOS_OS_FUNC_FILE_SENDIFLE - #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size) - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH -#define TAOS_OS_FUNC_FILE_FTRUNCATE - extern int taosFtruncate(int fd, int64_t length); +#define TAOS_OS_FUNC_FILE_FTRUNCATE #define TAOS_OS_FUNC_MATH #define SWAP(a, b, c) \ @@ -139,7 +136,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); #define in_addr_t unsigned long #define socklen_t int #define htobe64 htonll -#define twrite write #define getpid _getpid struct tm *localtime_r(const time_t *timep, struct tm *result); diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index 66bdb5b939..dacf4db741 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -19,21 +19,19 @@ #define _SEND_FILE_STEP_ 1000 -int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { +int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; + int writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { + } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); return (int)(writeLen + rlen); - } - else { + } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); writeLen += _SEND_FILE_STEP_; } @@ -44,8 +42,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou size_t rlen = fread(buffer, 1, remain, in_file); if (rlen <= 0) { return writeLen; - } - else { + } else { fwrite(buffer, 1, remain, out_file); writeLen += remain; } @@ -54,7 +51,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou return writeLen; } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - uError("not implemented yet"); +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { + uError("taosSendFile not implemented yet"); return -1; } \ No newline at end of file diff --git a/src/os/src/detail/osAlloc.c b/src/os/src/detail/osAlloc.c new file mode 100644 index 0000000000..4ca35793e7 --- /dev/null +++ b/src/os/src/detail/osAlloc.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tulog.h" +#include "osAlloc.h" + +#define TSDB_HAVE_MEMALIGN +#ifdef TAOS_OS_FUNC_ALLOC + +void *tmalloc(int32_t size) { + void *p = malloc(size); + if (p == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to malloc memory, size:%d reason:%s", size, strerror(errno)); + } + + return p; +} + +void *tcalloc(int32_t nmemb, int32_t size) { + void *p = calloc(nmemb, size); + if (p == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to calloc memory, nmemb:%d size:%d reason:%s", nmemb, size, strerror(errno)); + } + + return p; +} + +void *trealloc(void *p, int32_t size) { + p = realloc(p, size); + if (p == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to realloc memory, size:%d reason:%s", size, strerror(errno)); + } + + return p; +} + +void tfree(void *p) { free(p); } + +void tmemzero(void *p, int32_t size) { memset(p, 0, size); } + +#ifdef TSDB_HAVE_MEMALIGN + +void *tmemalign(int32_t alignment, int32_t size) { + void *p; + + int err = posix_memalign(&p, alignment, size); + if (err) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to memalign memory, alignment:%d size:%d reason:%s", alignment, size, strerror(err)); + p = NULL; + } + + return p; +} + +#else + +void *tmemalign(int32_t alignment, int32_t size) { return tmalloc(size); } + +#endif +#endif \ No newline at end of file diff --git a/src/os/src/detail/osFail.c b/src/os/src/detail/osFail.c index e0eb200851..a99bcd01db 100644 --- a/src/os/src/detail/osFail.c +++ b/src/os/src/detail/osFail.c @@ -20,7 +20,7 @@ #ifdef TAOS_RANDOM_NETWORK_FAIL -ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) { +int64_t taosSendRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = ECONNRESET; return -1; @@ -29,8 +29,8 @@ ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) { return send(sockfd, buf, len, flags); } -ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, - socklen_t addrlen) { +int64_t taosSendToRandomFail(int32_t sockfd, const void *buf, size_t len, int32_t flags, + const struct sockaddr *dest_addr, socklen_t addrlen) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = ECONNRESET; return -1; @@ -39,7 +39,7 @@ ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, return sendto(sockfd, buf, len, flags, dest_addr, addrlen); } -ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) { +int64_t taosReadSocketRandomFail(int32_t fd, void *buf, size_t count) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = ECONNRESET; return -1; @@ -48,7 +48,7 @@ ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) { return read(fd, buf, count); } -ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) { +int64_t taosWriteSocketRandomFail(int32_t fd, const void *buf, size_t count) { if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) { errno = EINTR; return -1; @@ -61,10 +61,10 @@ ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) { #ifdef TAOS_RANDOM_FILE_FAIL -static int random_file_fail_factor = 20; +static int32_t random_file_fail_factor = 20; static FILE *fpRandomFileFailOutput = NULL; -void taosSetRandomFileFailFactor(int factor) { +void taosSetRandomFileFailFactor(int32_t factor) { random_file_fail_factor = factor; } @@ -77,7 +77,7 @@ static void close_random_file_fail_output() { } } -static void random_file_fail_output_sig(int sig) { +static void random_file_fail_output_sig(int32_t sig) { fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig); struct sigaction act = {0}; @@ -105,7 +105,7 @@ void taosSetRandomFileFailOutput(const char *path) { sigaction(SIGILL, &act, NULL); } -ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) { +int64_t taosReadFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) { if (random_file_fail_factor > 0) { if (rand() % random_file_fail_factor == 0) { errno = EIO; @@ -113,10 +113,10 @@ ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file } } - return taosTReadImp(fd, buf, count); + return taosReadImp(fd, buf, count); } -ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) { +int64_t taosWriteFileRandomFail(int32_t fd, void *buf, int32_t count, const char *file, uint32_t line) { if (random_file_fail_factor > 0) { if (rand() % random_file_fail_factor == 0) { errno = EIO; @@ -124,10 +124,10 @@ ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *fil } } - return taosTWriteImp(fd, buf, count); + return taosWriteImp(fd, buf, count); } -off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) { +int64_t taosLSeekRandomFail(int32_t fd, int64_t offset, int32_t whence, const char *file, uint32_t line) { if (random_file_fail_factor > 0) { if (rand() % random_file_fail_factor == 0) { errno = EIO; @@ -135,7 +135,7 @@ off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, ui } } - return lseek(fd, offset, whence); + return taosLSeekImp(fd, offset, whence); } #endif //TAOS_RANDOM_FILE_FAIL diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 8f055dd812..6eb4515f30 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -15,16 +15,22 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "tglobal.h" #ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH + void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; - char *tmpDir = "/tmp/"; + int32_t len = strlen(tsTempDir); + memcpy(tmpPath, tsTempDir, len); - strcpy(tmpPath, tmpDir); - strcat(tmpPath, tdengineTmpFileNamePrefix); + if (tmpPath[len - 1] != '/') { + tmpPath[len++] = '/'; + } + + strcpy(tmpPath + len, tdengineTmpFileNamePrefix); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); @@ -34,10 +40,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } + #endif -// rename file name -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { +int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath) { int32_t ts = taosGetTimestampSec(); char fname[PATH_MAX] = {0}; // max file name length must be less than 255 @@ -46,12 +52,13 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP if (delimiterPos == NULL) return -1; int32_t fileNameLen = 0; - if (suffix) + if (suffix) { fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix); - else + } else { fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts); + } - size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1); + int32_t len = (int32_t)((delimiterPos - fullPath) + fileNameLen + 1); if (*dstPath == NULL) { *dstPath = calloc(1, len + 1); if (*dstPath == NULL) return -1; @@ -64,9 +71,9 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP return rename(fullPath, *dstPath); } -ssize_t taosTReadImp(int fd, void *buf, size_t count) { - size_t leftbytes = count; - ssize_t readbytes; +int64_t taosReadImp(int32_t fd, void *buf, int64_t count) { + int64_t leftbytes = count; + int64_t readbytes; char * tbuf = (char *)buf; while (leftbytes > 0) { @@ -78,19 +85,19 @@ ssize_t taosTReadImp(int fd, void *buf, size_t count) { return -1; } } else if (readbytes == 0) { - return (ssize_t)(count - leftbytes); + return (int64_t)(count - leftbytes); } leftbytes -= readbytes; tbuf += readbytes; } - return (ssize_t)count; + return count; } -ssize_t taosTWriteImp(int fd, void *buf, size_t n) { - size_t nleft = n; - ssize_t nwritten = 0; +int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) { + int64_t nleft = n; + int64_t nwritten = 0; char * tbuf = (char *)buf; while (nleft > 0) { @@ -105,13 +112,18 @@ ssize_t taosTWriteImp(int fd, void *buf, size_t n) { tbuf += nwritten; } - return (ssize_t)n; + return n; +} + +int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { + return (int64_t)tlseek(fd, (long)offset, whence); } #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - size_t leftbytes = size; - ssize_t sentbytes; + +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { + int64_t leftbytes = size; + int64_t sentbytes; while (leftbytes > 0) { /* @@ -126,7 +138,7 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { return -1; } } else if (sentbytes == 0) { - return (ssize_t)(size - leftbytes); + return (int64_t)(size - leftbytes); } leftbytes -= sentbytes; @@ -134,4 +146,17 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { return size; } + +int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { + return taosSendFile(fileno(outfile), fileno(infile), offset, size); +} + +#endif + +#ifndef TAOS_OS_FUNC_FILE_FTRUNCATE + +int32_t taosFtruncate(int32_t fd, int64_t length) { + return ftruncate(fd, length); +} + #endif \ No newline at end of file diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 8a51c389e9..c7c9d77427 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -19,8 +19,8 @@ #ifndef TAOS_OS_FUNC_SOCKET -int taosSetNonblocking(SOCKET sock, int on) { - int flags = 0; +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { + int32_t flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); return 1; @@ -43,7 +43,7 @@ void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { uError("failed to block SIGPIPE"); } @@ -53,7 +53,7 @@ void taosBlockSIGPIPE() { #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 8110a19490..19351eb7c9 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -46,5 +46,16 @@ void osInit() { strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); strcpy(tsOsName, "Windows"); + + const char *tmpDir = getenv("tmp"); + if (tmpDir != NULL) { + tmpDir = getenv("temp"); + } + if (tmpDir != NULL) { + strcpy(tsTempDir, tmpDir); + } else { + strcpy(tsTempDir, "C:\\Windows\\Temp"); + } + taosWinSocketInit(); } diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 5549c078a5..734ed9916d 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -16,17 +16,20 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" +#include "tglobal.h" void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; - char *tmpDir = getenv("tmp"); - if (tmpDir == NULL) { - tmpDir = ""; + int32_t len = (int32_t)strlen(tsTempDir); + memcpy(tmpPath, tsTempDir, len); + + if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { + tmpPath[len++] = '\\'; } - - strcpy(tmpPath, tmpDir); + + strcpy(tmpPath + len, tdengineTmpFileNamePrefix); strcat(tmpPath, tdengineTmpFileNamePrefix); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { strcat(tmpPath, fileNamePrefix); @@ -40,19 +43,19 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { #define _SEND_FILE_STEP_ 1000 -int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { +int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; + int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); + return (int64_t)(writeLen + rlen); } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); @@ -60,7 +63,7 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou } } - int remain = count - writeLen; + int64_t remain = count - writeLen; if (remain > 0) { size_t rlen = fread(buffer, 1, remain, in_file); if (rlen <= 0) { @@ -75,12 +78,12 @@ int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t cou return writeLen; } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - uError("taosTSendFileImp no implemented yet"); +int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { + uError("taosSendFile no implemented yet"); return 0; } -int taosFtruncate(int fd, int64_t length) { +int32_t taosFtruncate(int32_t fd, int64_t length) { uError("taosFtruncate no implemented yet"); return 0; } \ No newline at end of file diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index da9242d6a3..3b091b2699 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -34,7 +34,7 @@ void taosWinSocketInit() { } } -int taosSetNonblocking(SOCKET sock, int on) { +int32_t taosSetNonblocking(SOCKET sock, int32_t on) { u_long mode; if (on) { mode = 1; @@ -48,7 +48,7 @@ int taosSetNonblocking(SOCKET sock, int on) { void taosBlockSIGPIPE() {} -int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) { +int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -72,7 +72,7 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op uint32_t taosInetAddr(char *ipAddr) { uint32_t value; - int ret = inet_pton(AF_INET, ipAddr, &value); + int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { return INADDR_NONE; } else { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 2514da7231..691d2aed9c 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -803,7 +803,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - ssize_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); + int64_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); if (rc == -1) { // tscError("failed to merge tsBuf from:%s to %s, reason:%s\n", pSrcBuf->path, pDestBuf->path, strerror(errno)); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 1dfd87ff3a..d01c34c810 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -341,7 +341,7 @@ void *rpcMallocCont(int contLen) { tError("failed to malloc msg, size:%d", size); return NULL; } else { - tTrace("malloc mem: %p", start); + tTrace("malloc mem:%p size:%d", start, size); } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); @@ -557,10 +557,7 @@ void rpcCancelRequest(void *handle) { int code = taosAcquireRef(tsRpcRefId, pContext); if (code < 0) return; - if (pContext->pConn) { - tDebug("%s, app tries to cancel request", pContext->pConn->info); - rpcCloseConn(pContext->pConn); - } + rpcCloseConn(pContext->pConn); taosReleaseRef(tsRpcRefId, pContext); } @@ -655,6 +652,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { static void rpcCloseConn(void *thandle) { SRpcConn *pConn = (SRpcConn *)thandle; + if (pConn == NULL) return; rpcLockConn(pConn); @@ -1026,6 +1024,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { if (pConn->outType) { SRpcReqContext *pContext = pConn->pContext; pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pContext->pConn = NULL; pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); } @@ -1135,6 +1134,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // it's a response rpcMsg.handle = pContext; rpcMsg.ahandle = pContext->ahandle; + pContext->pConn = NULL; // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { @@ -1370,6 +1370,7 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pConn->pContext->pConn = NULL; pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pConn->pContext, pRpc->tmrCtrl); rpcReleaseConn(pConn); @@ -1478,7 +1479,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { pNewHead->msgLen = rpcMsgLenFromCont(origLen); rpcFreeMsg(pHead); // free the compressed message buffer pHead = pNewHead; - tTrace("decomp malloc mem: %p", temp); + tTrace("decomp malloc mem:%p", temp); } else { tError("failed to allocate memory to decompress msg, contLen:%d", contLen); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index bc8d360d39..c6de398608 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -437,12 +437,13 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { } msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); - buffer = malloc(msgLen + tsRpcOverhead); + int32_t size = msgLen + tsRpcOverhead; + buffer = malloc(size); if (NULL == buffer) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("TCP malloc mem: %p", buffer); + tTrace("TCP malloc mem:%p size:%d", buffer, size); } msg = buffer + tsRpcOverhead; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 6f65304661..9bac2b1c19 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -209,12 +209,13 @@ static void *taosRecvUdpData(void *param) { continue; } - char *tmsg = malloc(dataLen + tsRpcOverhead); + int32_t size = dataLen + tsRpcOverhead; + char *tmsg = malloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; } else { - tTrace("UDP malloc mem: %p", tmsg); + tTrace("UDP malloc mem:%p size:%d", tmsg, size); } tmsg += tsRpcOverhead; // overhead for SRpcReqContext diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 60625d75ec..0fe189db7a 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -149,7 +149,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { int sfd = open(name, O_RDONLY); if (sfd < 0) break; - ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); + ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) break; @@ -287,7 +287,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, return -1; } -static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { +static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; int code = -1; char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file @@ -377,7 +377,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { int32_t size; struct stat fstat; int code = -1; - uint32_t index = 0; + int64_t index = 0; while (1) { // retrieve wal info @@ -406,7 +406,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { int sfd = open(fname, O_RDONLY); if (sfd < 0) break; - code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); + code = taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); if (code < 0) break; diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 380b971fa8..0cf752da97 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex return magic; } -int getWalInfo(void *ahandle, char *name, uint32_t *index) { +int getWalInfo(void *ahandle, char *name, int64_t *index) { struct stat fstat; char aname[280]; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 5633e97cdf..3a5416dd30 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -13,10 +13,8 @@ * along with this program. If not, see . */ #define _DEFAULT_SOURCE -#include - #define TAOS_RANDOM_FILE_FAIL_TEST - +#include #include "os.h" #include "talgo.h" #include "tchecksum.h" @@ -428,7 +426,7 @@ int tsdbUpdateFileHeader(SFile *pFile) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -493,7 +491,7 @@ int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) { return -1; } - if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE, strerror(errno)); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a1e6376304..f714aadc36 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -579,7 +579,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -620,7 +620,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 357093bd9e..9940ec40ef 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -14,9 +14,7 @@ */ #define _DEFAULT_SOURCE - #define TAOS_RANDOM_FILE_FAIL_TEST - #include "os.h" #include "talgo.h" #include "tchecksum.h" @@ -335,7 +333,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { return -1; } - if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { + if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -380,7 +378,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { + if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -432,7 +430,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(offset == pFile->info.size); - if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { + if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -454,7 +452,7 @@ int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffe return -1; } - if (taosTRead(pFile->fd, buffer, len) < len) { + if (taosRead(pFile->fd, buffer, len) < len) { tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len, strerror(errno)); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; @@ -551,7 +549,7 @@ int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) { return -1; } - if (taosTRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { + if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -608,7 +606,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { return -1; } - if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { + if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -823,7 +821,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa sizeof(TSCKSUM)); // Write the whole block to file - if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) { + if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -1210,7 +1208,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl return -1; } - if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { + if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -1325,7 +1323,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { + if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index f14e8dbb35..391cc44acc 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,21 +20,21 @@ extern "C" { #endif -int taosReadn(SOCKET sock, char *buffer, int len); -int taosWriteMsg(SOCKET fd, void *ptr, int nbytes); -int taosReadMsg(SOCKET fd, void *ptr, int nbytes); -int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes); -int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); -int taosSetNonblocking(SOCKET sock, int on); +int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); +int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); +int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); +int32_t taosSetNonblocking(SOCKET sock, int32_t on); -SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); -SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int taosKeepTcpAlive(SOCKET sockFd); +SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int32_t taosKeepTcpAlive(SOCKET sockFd); -int taosGetFqdn(char *); +int32_t taosGetFqdn(char *); uint32_t taosGetIpFromFqdn(const char *); -void tinet_ntoa(char *ipstr, unsigned int ip); +void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); #ifdef __cplusplus diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 0ec55841a0..f449bfb68b 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -355,7 +355,11 @@ bool taosReadGlobalCfg() { fclose(fp); taosTFree(line); - + + if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { + taosSetAllDebugFlag(); + } + return true; } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 6ba1d87d92..0806c29ff8 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -14,9 +14,7 @@ */ #define _DEFAULT_SOURCE - #define TAOS_RANDOM_FILE_FAIL_TEST - #include "os.h" #include "hash.h" #include "taoserror.h" @@ -188,7 +186,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) { goto _err; } - if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { + if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -248,13 +246,13 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe ASSERT(tlen == POINTER_DISTANCE(pBuf, buf)); ASSERT(tlen == sizeof(SKVRecord)); - if (taosTWrite(pStore->fd, buf, tlen) < tlen) { + if (taosWrite(pStore->fd, buf, tlen) < tlen) { uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosTWrite(pStore->fd, cont, contLen) < contLen) { + if (taosWrite(pStore->fd, cont, contLen) < contLen) { uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno)); return -1; } @@ -292,7 +290,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { void *pBuf = buf; tdEncodeKVRecord(&pBuf, &rInfo); - if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { + if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -339,7 +337,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { int fd = open(fname, O_RDONLY); if (fd < 0) goto _err; - if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; + if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err; void *pBuf = (void *)buf; @@ -368,7 +366,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t return -1; } - if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { + if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -402,7 +400,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); - if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { + if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -535,7 +533,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); while (true) { - ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord)); + int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; if (tsize < sizeof(SKVRecord)) { uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, @@ -598,7 +596,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - if (taosTRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { + if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 09b0933fd6..0ad7917b39 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -336,11 +336,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { lseek(tsLogObj.logHandle->fd, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); return 0; } @@ -390,7 +390,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosTWrite(tsLogObj.logHandle->fd, buffer, len); + taosWrite(tsLogObj.logHandle->fd, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -400,7 +400,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { @@ -419,7 +419,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -427,9 +427,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); - - return; + taosWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { @@ -467,7 +465,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosTWrite(tsLogObj.logHandle->fd, buffer, len); + taosWrite(tsLogObj.logHandle->fd, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -477,7 +475,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWrite(1, buffer, (uint32_t)len); } #if 0 @@ -606,7 +604,7 @@ static void *taosAsyncOutputLog(void *param) { while (1) { log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT); if (log_size) { - taosTWrite(tLogBuff->fd, tempBuffer, log_size); + taosWrite(tLogBuff->fd, tempBuffer, log_size); LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff); } else { break; diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 4f05277a84..9536f6fb70 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -265,7 +265,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...) buffer[len] = 0; if (pNote->taosNoteFd >= 0) { - taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len); + taosWrite(pNote->taosNoteFd, buffer, (unsigned int)len); if (pNote->taosNoteMaxLines > 0) { pNote->taosNoteLines++; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 4cf73e6dff..1003bc6178 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -18,7 +18,7 @@ #include "tsocket.h" #include "taoserror.h" -int taosGetFqdn(char *fqdn) { +int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; if (gethostname(hostname, 1023) == -1) { @@ -26,10 +26,10 @@ int taosGetFqdn(char *fqdn) { return -1; } - struct addrinfo hints = {0}; + struct addrinfo hints = {0}; struct addrinfo *result = NULL; hints.ai_flags = AI_CANONNAME; - int ret = getaddrinfo(hostname, NULL, &hints, &result); + int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; @@ -49,10 +49,10 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr *sa = result->ai_addr; - struct sockaddr_in *si = (struct sockaddr_in*)sa; - struct in_addr ia = si->sin_addr; - uint32_t ip = ia.s_addr; + struct sockaddr * sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in *)sa; + struct in_addr ia = si->sin_addr; + uint32_t ip = ia.s_addr; freeaddrinfo(result); return ip; } else { @@ -70,7 +70,7 @@ uint32_t taosGetIpFromFqdn(const char *fqdn) { } } -// Function converting an IP address string to an unsigned int. +// Function converting an IP address string to an uint32_t. uint32_t ip2uint(const char *const ip_addr) { char ip_addr_cpy[20]; char ip[5]; @@ -81,7 +81,7 @@ uint32_t ip2uint(const char *const ip_addr) { s_start = ip_addr_cpy; s_end = ip_addr_cpy; - int k; + int32_t k; for (k = 0; *s_start != '\0'; s_start = s_end) { for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { @@ -95,17 +95,17 @@ uint32_t ip2uint(const char *const ip_addr) { ip[k] = '\0'; - return *((unsigned int *)ip); + return *((uint32_t *)ip); } -int taosWriteMsg(SOCKET fd, void *buf, int nbytes) { - int nleft, nwritten; - char *ptr = (char *)buf; +int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { + int32_t nleft, nwritten; + char * ptr = (char *)buf; nleft = nbytes; while (nleft > 0) { - nwritten = (int)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); + nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); if (nwritten <= 0) { if (errno == EINTR) continue; @@ -120,16 +120,16 @@ int taosWriteMsg(SOCKET fd, void *buf, int nbytes) { return (nbytes - nleft); } -int taosReadMsg(SOCKET fd, void *buf, int nbytes) { - int nleft, nread; - char *ptr = (char *)buf; +int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { + int32_t nleft, nread; + char * ptr = (char *)buf; nleft = nbytes; if (fd < 0) return -1; while (nleft > 0) { - nread = (int)taosReadSocket(fd, ptr, (size_t)nleft); + nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft); if (nread == 0) { break; } else if (nread < 0) { @@ -147,11 +147,11 @@ int taosReadMsg(SOCKET fd, void *buf, int nbytes) { return (nbytes - nleft); } -int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { +int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) { taosSetNonblocking(fd, 1); - int nleft, nwritten, nready; - fd_set fset; + int32_t nleft, nwritten, nready; + fd_set fset; struct timeval tv; nleft = nbytes; @@ -160,7 +160,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { tv.tv_usec = 0; FD_ZERO(&fset); FD_SET(fd, &fset); - if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { errno = ETIMEDOUT; uError("fd %d timeout, no enough space to write", fd); break; @@ -172,7 +172,7 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { return -1; } - nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); + nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL); if (nwritten <= 0) { if (errno == EAGAIN || errno == EINTR) continue; @@ -189,10 +189,10 @@ int taosNonblockwrite(SOCKET fd, char *ptr, int nbytes) { return (nbytes - nleft); } -int taosReadn(SOCKET fd, char *ptr, int nbytes) { - int nread, nready, nleft = nbytes; +int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { + int32_t nread, nready, nleft = nbytes; - fd_set fset; + fd_set fset; struct timeval tv; while (nleft > 0) { @@ -200,7 +200,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) { tv.tv_usec = 0; FD_ZERO(&fset); FD_SET(fd, &fset); - if ((nready = select((int)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { + if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) { errno = ETIMEDOUT; uError("fd %d timeout\n", fd); break; @@ -210,7 +210,7 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) { return -1; } - if ((nread = (int)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { + if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) { if (errno == EINTR) continue; uError("read error, %d (%s)", errno, strerror(errno)); return -1; @@ -229,8 +229,8 @@ int taosReadn(SOCKET fd, char *ptr, int nbytes) { SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; - int bufSize = 1024000; + SOCKET sockFd; + int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -239,7 +239,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -268,9 +268,9 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { } SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; + SOCKET sockFd = 0; + int32_t ret; struct sockaddr_in serverAddr, clientAddr; - int ret; sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); @@ -281,7 +281,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie } /* set REUSEADDR option, so the portnumber can be re-used */ - int reuse = 1; + int32_t reuse = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); taosCloseSocket(sockFd); @@ -296,8 +296,8 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie /* bind socket to client address */ if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", - clientIp, destIp, destPort, strerror(errno)); + uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort, + strerror(errno)); taosCloseSocket(sockFd); return -1; } @@ -311,7 +311,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); if (ret != 0) { - //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + // uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); taosCloseSocket(sockFd); sockFd = -1; } else { @@ -321,36 +321,36 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie return sockFd; } -int taosKeepTcpAlive(SOCKET sockFd) { - int alive = 1; +int32_t taosKeepTcpAlive(SOCKET sockFd) { + int32_t alive = 1; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) { uError("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int probes = 3; + int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int alivetime = 10; + int32_t alivetime = 10; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) { uError("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int interval = 3; + int32_t interval = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) { uError("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); return -1; } - int nodelay = 1; + int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { uError("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno)); taosCloseSocket(sockFd); @@ -371,8 +371,8 @@ int taosKeepTcpAlive(SOCKET sockFd) { SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; - SOCKET sockFd; - int reuse; + SOCKET sockFd; + int32_t reuse; uDebug("open tcp server socket:0x%x:%hu", ip, port); @@ -381,7 +381,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -417,38 +417,38 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { return sockFd; } -void tinet_ntoa(char *ipstr, unsigned int ip) { +void tinet_ntoa(char *ipstr, uint32_t ip) { sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24); } #define COPY_SIZE 32768 // sendfile shall be used -int taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { +int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { int64_t leftLen; - int readLen, writeLen; + int32_t readLen, writeLen; char temp[COPY_SIZE]; leftLen = len; while (leftLen > 0) { if (leftLen < COPY_SIZE) - readLen = (int)leftLen; + readLen = (int32_t)leftLen; else readLen = COPY_SIZE; // 4K - int retLen = taosReadMsg(sfd, temp, (int)readLen); + int32_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); if (readLen != retLen) { - uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, leftLen, - strerror(errno)); + uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, + leftLen, strerror(errno)); return -1; } writeLen = taosWriteMsg(dfd, temp, readLen); if (readLen != writeLen) { - uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, len, leftLen, - strerror(errno)); + uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, + len, leftLen, strerror(errno)); return -1; } diff --git a/src/wal/inc/walMgmt.h b/src/vnode/inc/vnodeCfg.h similarity index 82% rename from src/wal/inc/walMgmt.h rename to src/vnode/inc/vnodeCfg.h index a23c7f8ec3..c5887fef5d 100644 --- a/src/wal/inc/walMgmt.h +++ b/src/vnode/inc/vnodeCfg.h @@ -13,13 +13,16 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_WAL_MGMT_H -#define TDENGINE_WAL_MGMT_H +#ifndef TDENGINE_VNODE_CFG_H +#define TDENGINE_VNODE_CFG_H #ifdef __cplusplus extern "C" { #endif +int32_t vnodeReadCfg(SVnodeObj *pVnode); +int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg); + #ifdef __cplusplus } #endif diff --git a/src/vnode/inc/vnodeVersion.h b/src/vnode/inc/vnodeVersion.h new file mode 100644 index 0000000000..1d086cb21f --- /dev/null +++ b/src/vnode/inc/vnodeVersion.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_VERSION_H +#define TDENGINE_VNODE_VERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t vnodeReadVersion(SVnodeObj *pVnode); +int32_t vnodeSaveVersion(SVnodeObj *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c new file mode 100644 index 0000000000..f0040f8cdf --- /dev/null +++ b/src/vnode/src/vnodeCfg.c @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "taoserror.h" +#include "cJSON.h" +#include "tglobal.h" +#include "tsdb.h" +#include "dnode.h" +#include "vnodeInt.h" +#include "vnodeVersion.h" +#include "vnodeCfg.h" + +static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { + strcpy(pVnode->db, vnodeMsg->db); + pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion; + pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize; + pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks; + pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile; + pVnode->tsdbCfg.keep = vnodeMsg->cfg.daysToKeep; + pVnode->tsdbCfg.keep1 = vnodeMsg->cfg.daysToKeep1; + pVnode->tsdbCfg.keep2 = vnodeMsg->cfg.daysToKeep2; + pVnode->tsdbCfg.minRowsPerFileBlock = vnodeMsg->cfg.minRowsPerFileBlock; + pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; + pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; + pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; + pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; + pVnode->walCfg.wals = vnodeMsg->cfg.wals; + pVnode->walCfg.keep = 0; + pVnode->syncCfg.replica = vnodeMsg->cfg.replications; + pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; + + for (int i = 0; i < pVnode->syncCfg.replica; ++i) { + SVnodeDesc *node = &vnodeMsg->nodes[i]; + pVnode->syncCfg.nodeInfo[i].nodeId = node->nodeId; + taosGetFqdnPortFromEp(node->nodeEp, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort); + pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC; + } + + vInfo("vgId:%d, load vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica); + for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) { + SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i]; + vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort); + } +} + +int32_t vnodeReadCfg(SVnodeObj *pVnode) { + int32_t ret = TSDB_CODE_VND_APP_ERROR; + int32_t len = 0; + int maxLen = 1000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + bool nodeChanged = false; + SCreateVnodeMsg vnodeMsg; + + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId); + + vnodeMsg.cfg.vgId = pVnode->vgId; + + fp = fopen(file, "r"); + if (!fp) { + vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId, file, strerror(errno)); + ret = TAOS_SYSTEM_ERROR(errno); + goto PARSE_VCFG_ERROR; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + cJSON *db = cJSON_GetObjectItem(root, "db"); + if (!db || db->type != cJSON_String || db->valuestring == NULL) { + vError("vgId:%d, failed to read %s, db not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + strcpy(vnodeMsg.db, db->valuestring); + + cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!cfgVersion || cfgVersion->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.cfgVersion = cfgVersion->valueint; + + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); + if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint; + + cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); + if (!totalBlocks || totalBlocks->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.totalBlocks = totalBlocks->valueint; + + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + if (!daysPerFile || daysPerFile->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysPerFile = daysPerFile->valueint; + + cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); + if (!daysToKeep || daysToKeep->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysToKeep = daysToKeep->valueint; + + cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); + if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint; + + cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); + if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint; + + cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); + if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; + + cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); + if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; + + cJSON *precision = cJSON_GetObjectItem(root, "precision"); + if (!precision || precision->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, precision not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.precision = (int8_t)precision->valueint; + + cJSON *compression = cJSON_GetObjectItem(root, "compression"); + if (!compression || compression->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, compression not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.compression = (int8_t)compression->valueint; + + cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel"); + if (!walLevel || walLevel->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, walLevel not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.walLevel = (int8_t)walLevel->valueint; + + cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync"); + if (!walLevel || walLevel->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint; + + cJSON *wals = cJSON_GetObjectItem(root, "wals"); + if (!wals || wals->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, wals not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.wals = (int8_t)wals->valueint; + + cJSON *replica = cJSON_GetObjectItem(root, "replica"); + if (!replica || replica->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.replications = (int8_t)replica->valueint; + + cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); + if (!quorum || quorum->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, quorum not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); + if (!nodeInfos || nodeInfos->type != cJSON_Array) { + vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + int size = cJSON_GetArraySize(nodeInfos); + if (size != vnodeMsg.cfg.replications) { + vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + + for (int i = 0; i < size; ++i) { + cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); + if (nodeInfo == NULL) continue; + SVnodeDesc *node = &vnodeMsg.nodes[i]; + + cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); + if (!nodeId || nodeId->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + node->nodeId = nodeId->valueint; + + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { + vError("vgId:%d, failed to read %s, nodeFqdn not found", pVnode->vgId, file); + goto PARSE_VCFG_ERROR; + } + tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + if (!nodeChanged) { + nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp); + } + } + + ret = TSDB_CODE_SUCCESS; + +PARSE_VCFG_ERROR: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + if (nodeChanged) { + vnodeWriteCfg(&vnodeMsg); + } + + if (ret == TSDB_CODE_SUCCESS) { + vnodeLoadCfg(pVnode, &vnodeMsg); + } + + terrno = 0; + return ret; +} + +int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/config.json", tsVnodeDir, pMsg->cfg.vgId); + + FILE *fp = fopen(file, "w"); + if (!fp) { + vError("vgId:%d, failed to write %s error:%s", pMsg->cfg.vgId, file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; + } + + int32_t len = 0; + int32_t maxLen = 1000; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion); + len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize); + len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks); + len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile); + len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pMsg->cfg.daysToKeep); + len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pMsg->cfg.daysToKeep1); + len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pMsg->cfg.daysToKeep2); + len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pMsg->cfg.minRowsPerFileBlock); + len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pMsg->cfg.maxRowsPerFileBlock); + len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pMsg->cfg.precision); + len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression); + len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel); + len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications); + len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); + len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); + for (int32_t i = 0; i < pMsg->cfg.replications; i++) { + SVnodeDesc *node = &pMsg->nodes[i]; + dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL); + len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp); + if (i < pMsg->cfg.replications - 1) { + len += snprintf(content + len, maxLen - len, " },{\n"); + } else { + len += snprintf(content + len, maxLen - len, " }]\n"); + } + } + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file); + return TSDB_CODE_SUCCESS; +} diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 9c283dd9c4..e206933116 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -31,18 +31,14 @@ #include "vnodeInt.h" #include "query.h" #include "dnode.h" - -#define TSDB_VNODE_VERSION_CONTENT_LEN 31 +#include "vnodeCfg.h" +#include "vnodeVersion.h" static SHashObj*tsDnodeVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); -static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); -static int32_t vnodeReadCfg(SVnodeObj *pVnode); -static int32_t vnodeSaveVersion(SVnodeObj *pVnode); -static int32_t vnodeReadVersion(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); -static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); +static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeCtrlFlow(void *handle, int32_t mseconds); static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); @@ -90,7 +86,7 @@ void vnodeCleanupResources() { syncCleanUp(); } -int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { +int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { int32_t code; SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId); @@ -128,7 +124,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { } } - code = vnodeSaveCfg(pVnodeCfg); + code = vnodeWriteCfg(pVnodeCfg); if (code != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to save vnode cfg, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code)); return code; @@ -138,7 +134,6 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - // tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables; tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; @@ -176,7 +171,7 @@ int32_t vnodeDrop(int32_t vgId) { return TSDB_CODE_SUCCESS; } -int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { +int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) { SVnodeObj *pVnode = param; // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS @@ -186,7 +181,7 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_SUCCESS; } - int32_t code = vnodeSaveCfg(pVnodeCfg); + int32_t code = vnodeWriteCfg(pVnodeCfg); if (code != TSDB_CODE_SUCCESS) { pVnode->status = TAOS_VN_STATUS_READY; return code; @@ -259,8 +254,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->fversion = pVnode->version; - pVnode->wqueue = dnodeAllocateVnodeWqueue(pVnode); - pVnode->rqueue = dnodeAllocateVnodeRqueue(pVnode); + pVnode->wqueue = dnodeAllocVWriteQueue(pVnode); + pVnode->rqueue = dnodeAllocVReadQueue(pVnode); if (pVnode->wqueue == NULL || pVnode->rqueue == NULL) { vnodeCleanUp(pVnode); return terrno; @@ -303,6 +298,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } sprintf(temp, "%s/wal", rootDir); + pVnode->walCfg.vgId = pVnode->vgId; pVnode->wal = walOpen(temp, &pVnode->walCfg); if (pVnode->wal == NULL) { vnodeCleanUp(pVnode); @@ -314,6 +310,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->version = walGetVersion(pVnode->wal); } + walRenew(pVnode->wal); + SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; @@ -323,7 +321,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.getWalInfo = vnodeGetWalInfo; syncInfo.getFileInfo = vnodeGetFileInfo; syncInfo.writeToCache = vnodeWriteToQueue; - syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp; + syncInfo.confirmForward = dnodeSendRpcVWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; @@ -388,6 +386,10 @@ void vnodeRelease(void *pVnodeRaw) { pVnode->qMgmt = NULL; } + if (pVnode->wal) { + walStop(pVnode->wal); + } + if (pVnode->tsdb) { tsdbCloseRepo(pVnode->tsdb, 1); pVnode->tsdb = NULL; @@ -406,12 +408,12 @@ void vnodeRelease(void *pVnodeRaw) { } if (pVnode->wqueue) { - dnodeFreeVnodeWqueue(pVnode->wqueue); + dnodeFreeVWriteQueue(pVnode->wqueue); pVnode->wqueue = NULL; } if (pVnode->rqueue) { - dnodeFreeVnodeRqueue(pVnode->rqueue); + dnodeFreeVReadQueue(pVnode->rqueue); pVnode->rqueue = NULL; } @@ -499,7 +501,7 @@ void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } -static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { +static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { int64_t totalStorage = 0; int64_t compStorage = 0; int64_t pointsWritten = 0; @@ -543,7 +545,7 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { } void vnodeBuildStatusMsg(void *param) { - SDMStatusMsg *pStatus = param; + SStatusMsg *pStatus = param; SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash); while (taosHashIterNext(pIter)) { @@ -557,7 +559,7 @@ void vnodeBuildStatusMsg(void *param) { taosHashDestroyIter(pIter); } -void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) { +void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { for (int32_t i = 0; i < numOfVnodes; ++i) { pAccess[i].vgId = htonl(pAccess[i].vgId); SVnodeObj *pVnode = vnodeAcquire(pAccess[i].vgId); @@ -605,25 +607,27 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { SVnodeObj *pVnode = arg; if (status == TSDB_STATUS_COMMIT_START) { - pVnode->fversion = pVnode->version; + pVnode->fversion = pVnode->version; return walRenew(pVnode->wal); } - if (status == TSDB_STATUS_COMMIT_OVER) + if (status == TSDB_STATUS_COMMIT_OVER) { return vnodeSaveVersion(pVnode); + } - return 0; + return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, + uint64_t *fversion) { SVnodeObj *pVnode = ahandle; *fversion = pVnode->fversion; return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); } -static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { +static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { SVnodeObj *pVnode = ahandle; - return walGetWalFile(pVnode->wal, name, index); + return walGetWalFile(pVnode->wal, fileName, fileId); } static void vnodeNotifyRole(void *ahandle, int8_t role) { @@ -640,17 +644,18 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) { SVnodeObj *pVnode = ahandle; - if (pVnode->delay != mseconds) + if (pVnode->delay != mseconds) { vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); + } pVnode->delay = mseconds; } -static int vnodeResetTsdb(SVnodeObj *pVnode) -{ +static int vnodeResetTsdb(SVnodeObj *pVnode) { char rootDir[128] = "\0"; sprintf(rootDir, "%s/tsdb", pVnode->rootDir); - if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != TAOS_VN_STATUS_READY) { + if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != + TAOS_VN_STATUS_READY) { return -1; } @@ -675,7 +680,7 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) pVnode->tsdb = tsdbOpenRepo(rootDir, &appH); pVnode->status = TAOS_VN_STATUS_READY; - vnodeRelease(pVnode); + vnodeRelease(pVnode); return 0; } @@ -690,360 +695,3 @@ static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { return vnodeResetTsdb(pVnode); } - -static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { - char cfgFile[TSDB_FILENAME_LEN + 30] = {0}; - sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId); - FILE *fp = fopen(cfgFile, "w"); - if (!fp) { - vError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - int32_t len = 0; - int32_t maxLen = 1000; - char * content = calloc(1, maxLen + 1); - if (content == NULL) { - fclose(fp); - return TSDB_CODE_VND_OUT_OF_MEMORY; - } - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pVnodeCfg->db); - len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); - len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); - len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); - // len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); - len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); - len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); - len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); - len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); - len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - // len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); - len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); - len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); - len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnodeCfg->cfg.walLevel); - len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pVnodeCfg->cfg.fsyncPeriod); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); - len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); - len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); - - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - - vInfo("vgId:%d, save vnode cfg, replica:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.replications); - for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { - len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp); - vInfo("vgId:%d, save vnode cfg, nodeId:%d nodeEp:%s", pVnodeCfg->cfg.vgId, pVnodeCfg->nodes[i].nodeId, - pVnodeCfg->nodes[i].nodeEp); - - if (i < pVnodeCfg->cfg.replications - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }]\n"); - } - } - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - free(content); - - vInfo("vgId:%d, save vnode cfg successed", pVnodeCfg->cfg.vgId); - - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeReadCfg(SVnodeObj *pVnode) { - cJSON *root = NULL; - char *content = NULL; - char cfgFile[TSDB_FILENAME_LEN + 30] = {0}; - int maxLen = 1000; - - terrno = TSDB_CODE_VND_APP_ERROR; - sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(cfgFile, "r"); - if (!fp) { - vError("vgId:%d, failed to open vnode cfg file:%s to read, error:%s", pVnode->vgId, - cfgFile, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto PARSE_OVER; - } - - content = calloc(1, maxLen + 1); - if (content == NULL) goto PARSE_OVER; - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - vError("vgId:%d, failed to read vnode cfg, content is null", pVnode->vgId); - free(content); - fclose(fp); - return errno; - } - - root = cJSON_Parse(content); - if (root == NULL) { - vError("vgId:%d, failed to read vnode cfg, invalid json format", pVnode->vgId); - goto PARSE_OVER; - } - - cJSON *db = cJSON_GetObjectItem(root, "db"); - if (!db || db->type != cJSON_String || db->valuestring == NULL) { - vError("vgId:%d, failed to read vnode cfg, db not found", pVnode->vgId); - goto PARSE_OVER; - } - strcpy(pVnode->db, db->valuestring); - - cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); - if (!cfgVersion || cfgVersion->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->cfgVersion = cfgVersion->valueint; - - cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); - if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint; - - cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); - if (!totalBlocks || totalBlocks->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; - - // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); - // if (!maxTables || maxTables->type != cJSON_Number) { - // vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId); - // goto PARSE_OVER; - // } - // pVnode->tsdbCfg.maxTables = maxTables->valueint; - - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); - if (!daysPerFile || daysPerFile->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint; - - cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); - if (!daysToKeep || daysToKeep->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.keep = daysToKeep->valueint; - - cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); - if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.keep1 = daysToKeep1->valueint; - - cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); - if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.keep2 = daysToKeep2->valueint; - - cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); - if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; - - cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); - if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, maxRowsPerFileBlock not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - - // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); - // if (!commitTime || commitTime->type != cJSON_Number) { - // vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId); - // goto PARSE_OVER; - // } - // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; - - cJSON *precision = cJSON_GetObjectItem(root, "precision"); - if (!precision || precision->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, precision not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.precision = (int8_t)precision->valueint; - - cJSON *compression = cJSON_GetObjectItem(root, "compression"); - if (!compression || compression->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, compression not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.compression = (int8_t)compression->valueint; - - cJSON *walLevel = cJSON_GetObjectItem(root, "walLevel"); - if (!walLevel || walLevel->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, walLevel not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->walCfg.walLevel = (int8_t) walLevel->valueint; - - cJSON *fsyncPeriod = cJSON_GetObjectItem(root, "fsync"); - if (!walLevel || walLevel->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, fsyncPeriod not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->walCfg.fsyncPeriod = fsyncPeriod->valueint; - - cJSON *wals = cJSON_GetObjectItem(root, "wals"); - if (!wals || wals->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, wals not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->walCfg.wals = (int8_t)wals->valueint; - pVnode->walCfg.keep = 0; - - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, replica not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.replica = (int8_t)replica->valueint; - - cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); - if (!quorum || quorum->type != cJSON_Number) { - vError("vgId: %d, failed to read vnode cfg, quorum not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.quorum = (int8_t)quorum->valueint; - - cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); - if (!nodeInfos || nodeInfos->type != cJSON_Array) { - vError("vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode->vgId); - goto PARSE_OVER; - } - - int size = cJSON_GetArraySize(nodeInfos); - if (size != pVnode->syncCfg.replica) { - vError("vgId:%d, failed to read vnode cfg, nodeInfos size not matched", pVnode->vgId); - goto PARSE_OVER; - } - - for (int i = 0; i < size; ++i) { - cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); - if (nodeInfo == NULL) continue; - - cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); - if (!nodeId || nodeId->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, nodeId not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint; - - cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); - if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - vError("vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode->vgId); - goto PARSE_OVER; - } - - taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort); - pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC; - } - - terrno = TSDB_CODE_SUCCESS; - - vInfo("vgId:%d, read vnode cfg successfully, replcia:%d", pVnode->vgId, pVnode->syncCfg.replica); - for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) { - vInfo("vgId:%d, dnode:%d, %s:%d", pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId, - pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort); - } - -PARSE_OVER: - taosTFree(content); - cJSON_Delete(root); - if (fp) fclose(fp); - return terrno; -} - -static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { - char versionFile[TSDB_FILENAME_LEN + 30] = {0}; - sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(versionFile, "w"); - if (!fp) { - vError("vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode->vgId, - versionFile, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - int32_t len = 0; - int32_t maxLen = 30; - char content[TSDB_VNODE_VERSION_CONTENT_LEN] = {0}; - - len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion); - len += snprintf(content + len, maxLen - len, "}\n"); - - fwrite(content, 1, len, fp); - fflush(fp); - fclose(fp); - - vInfo("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion); - - return TSDB_CODE_SUCCESS; -} - -static int32_t vnodeReadVersion(SVnodeObj *pVnode) { - char versionFile[TSDB_FILENAME_LEN + 30] = {0}; - char *content = NULL; - cJSON *root = NULL; - int maxLen = 100; - - terrno = TSDB_CODE_VND_INVALID_VRESION_FILE; - sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(versionFile, "r"); - if (!fp) { - if (errno != ENOENT) { - vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } else { - terrno = TSDB_CODE_SUCCESS; - } - goto PARSE_OVER; - } - - content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - vError("vgId:%d, failed to read vnode version, content is null", pVnode->vgId); - goto PARSE_OVER; - } - - root = cJSON_Parse(content); - if (root == NULL) { - vError("vgId:%d, failed to read vnode version, invalid json format", pVnode->vgId); - goto PARSE_OVER; - } - - cJSON *ver = cJSON_GetObjectItem(root, "version"); - if (!ver || ver->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode version, version not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->version = ver->valueint; - - terrno = TSDB_CODE_SUCCESS; - vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version); - -PARSE_OVER: - taosTFree(content); - cJSON_Delete(root); - if (fp) fclose(fp); - return terrno; -} diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c new file mode 100644 index 0000000000..1d0695fb53 --- /dev/null +++ b/src/vnode/src/vnodeVersion.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "cJSON.h" +#include "tglobal.h" +#include "tsdb.h" +#include "vnodeInt.h" +#include "vnodeVersion.h" + +int32_t vnodeReadVersion(SVnodeObj *pVnode) { + int32_t len = 0; + int32_t maxLen = 100; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + + terrno = TSDB_CODE_VND_INVALID_VRESION_FILE; + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); + + fp = fopen(file, "r"); + if (!fp) { + if (errno != ENOENT) { + vError("vgId:%d, failed to read %s, error:%s", pVnode->vgId, file, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + } else { + terrno = TSDB_CODE_SUCCESS; + } + goto PARSE_VER_ERROR; + } + + len = fread(content, 1, maxLen, fp); + if (len <= 0) { + vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); + goto PARSE_VER_ERROR; + } + + root = cJSON_Parse(content); + if (root == NULL) { + vError("vgId:%d, failed to read %s, invalid json format", pVnode->vgId, file); + goto PARSE_VER_ERROR; + } + + cJSON *ver = cJSON_GetObjectItem(root, "version"); + if (!ver || ver->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file); + goto PARSE_VER_ERROR; + } + pVnode->version = ver->valueint; + + terrno = TSDB_CODE_SUCCESS; + vInfo("vgId:%d, read %s successfully, version:%" PRId64, pVnode->vgId, file, pVnode->version); + +PARSE_VER_ERROR: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + return terrno; +} + +int32_t vnodeSaveVersion(SVnodeObj *pVnode) { + char file[TSDB_FILENAME_LEN + 30] = {0}; + sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); + + FILE *fp = fopen(file, "w"); + if (!fp) { + vError("vgId:%d, failed to write %s, reason:%s", pVnode->vgId, file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 100; + char * content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + fflush(fp); + fclose(fp); + free(content); + terrno = 0; + + vInfo("vgId:%d, successed to write %s, version:%" PRId64, pVnode->vgId, file, pVnode->fversion); + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 855df81a1b..c2771c73c6 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -183,7 +183,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet } static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - SMDDropSTableMsg *pTable = pCont; + SDropSTableMsg *pTable = pCont; int32_t code = TSDB_CODE_SUCCESS; vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId); @@ -204,7 +204,6 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } - int vnodeWriteCqMsgToQueue(void *param, void *data, int type) { SVnodeObj *pVnode = param; SWalHead * pHead = data; @@ -222,7 +221,6 @@ int vnodeWriteCqMsgToQueue(void *param, void *data, int type) { return 0; } - int vnodeWriteToQueue(void *param, void *data, int type) { SVnodeObj *pVnode = param; SWalHead * pHead = data; diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h index 593611589d..5273eb5b1c 100644 --- a/src/wal/inc/walInt.h +++ b/src/wal/inc/walInt.h @@ -31,8 +31,34 @@ extern int32_t wDebugFlag; #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define walPrefix "wal" -#define walSignature (uint32_t)(0xFAFBFDFE) +#define WAL_PREFIX "wal" +#define WAL_PREFIX_LEN 3 +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (1024 * 1024) +#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (TSDB_FILENAME_LEN + 32) +#define WAL_FILE_NUM 3 + +typedef struct { + uint64_t version; + int64_t fileId; + int32_t vgId; + int32_t fd; + int32_t keep; + int32_t level; + int32_t fsyncPeriod; + int32_t fsyncSeq; + int8_t stop; + int8_t reserved[3]; + char path[WAL_PATH_LEN]; + char name[WAL_FILE_LEN]; + pthread_mutex_t mutex; +} SWal; + +int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId); +int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId); +int32_t walGetNewFile(SWal *pWal, int64_t *newFileId); #ifdef __cplusplus } diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 2dd094d860..c8f0274174 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -16,6 +16,219 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" +#include "tref.h" #include "twal.h" #include "walInt.h" -#include "walMgmt.h" \ No newline at end of file + +typedef struct { + int32_t refId; + int32_t seq; + int8_t stop; + pthread_t thread; + pthread_mutex_t mutex; +} SWalMgmt; + +static SWalMgmt tsWal; +static int32_t walCreateThread(); +static void walStopThread(); +static int32_t walInitObj(SWal *pWal); +static void walFreeObj(void *pWal); + +int32_t walInit() { + tmemzero(&tsWal, sizeof(SWalMgmt)); + tsWal.refId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); + + int32_t code = walCreateThread(); + if (code != TSDB_CODE_SUCCESS) { + wError("failed to init wal module since %s", tstrerror(code)); + return code; + } + + wInfo("wal module is initialized, refId:%d", tsWal.refId); + return code; +} + +void walCleanUp() { + walStopThread(); + taosCloseRef(tsWal.refId); + wInfo("wal module is cleaned up"); +} + +void *walOpen(char *path, SWalCfg *pCfg) { + SWal *pWal = tcalloc(1, sizeof(SWal)); + if (pWal == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + + pWal->vgId = pCfg->vgId; + pWal->fd = -1; + pWal->fileId = -1; + pWal->level = pCfg->walLevel; + pWal->keep = pCfg->keep; + pWal->fsyncPeriod = pCfg->fsyncPeriod; + tstrncpy(pWal->path, path, sizeof(pWal->path)); + pthread_mutex_init(&pWal->mutex, NULL); + + pWal->fsyncSeq = pCfg->fsyncPeriod % 1000; + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; + + if (walInitObj(pWal) != TSDB_CODE_SUCCESS) { + walFreeObj(pWal); + return NULL; + } + + if (taosAddRef(tsWal.refId, pWal) != TSDB_CODE_SUCCESS) { + walFreeObj(pWal); + return NULL; + } + + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); + + return pWal; +} + +int32_t walAlter(void *handle, SWalCfg *pCfg) { + if (handle == NULL) return TSDB_CODE_WAL_APP_ERROR; + SWal *pWal = handle; + + if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { + wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level, + pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + return TSDB_CODE_SUCCESS; + } + + wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level, + pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + + pWal->level = pCfg->walLevel; + pWal->fsyncPeriod = pCfg->fsyncPeriod; + pWal->fsyncSeq = pCfg->fsyncPeriod % 1000; + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; + + return TSDB_CODE_SUCCESS; +} + +void walStop(void *handle) { + if (handle == NULL) return; + SWal *pWal = handle; + + pthread_mutex_lock(&pWal->mutex); + pWal->stop = 1; + pthread_mutex_unlock(&pWal->mutex); + wDebug("vgId:%d, stop write wal", pWal->vgId); +} + +void walClose(void *handle) { + if (handle == NULL) return; + + SWal *pWal = handle; + pthread_mutex_lock(&pWal->mutex); + + taosClose(pWal->fd); + + if (!pWal->keep) { + int64_t fileId = -1; + while (walGetNextFile(pWal, &fileId) >= 0) { + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); + + if (remove(pWal->name) < 0) { + wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); + } else { + wDebug("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); + } + } + } else { + wDebug("vgId:%d, wal:%p file:%s, it is closed and kept", pWal->vgId, pWal, pWal->name); + } + + pthread_mutex_unlock(&pWal->mutex); + taosRemoveRef(tsWal.refId, pWal); +} + +static int32_t walInitObj(SWal *pWal) { + if (taosMkDir(pWal->path, 0755) != 0) { + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + wDebug("vgId:%d, object is initialized", pWal->vgId); + return TSDB_CODE_SUCCESS; +} + +static void walFreeObj(void *wal) { + SWal *pWal = wal; + wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); + + taosClose(pWal->fd); + pthread_mutex_destroy(&pWal->mutex); + tfree(pWal); +} + +static bool walNeedFsync(SWal *pWal) { + if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) { + return false; + } + + if (tsWal.seq % pWal->fsyncSeq == 0) { + return true; + } + + return false; +} + +static void walUpdateSeq() { + taosMsleep(WAL_REFRESH_MS); + if (++tsWal.seq <= 0) { + tsWal.seq = 1; + } +} + +static void walFsyncAll() { + SWal *pWal = taosIterateRef(tsWal.refId, NULL); + while (pWal) { + if (walNeedFsync(pWal)) { + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); + int32_t code = fsync(pWal->fd); + if (code != 0) { + wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code)); + } + } + pWal = taosIterateRef(tsWal.refId, pWal); + } +} + +static void *walThreadFunc(void *param) { + while (1) { + walUpdateSeq(); + walFsyncAll(); + if (tsWal.stop) break; + } + + return NULL; +} + +static int32_t walCreateThread() { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&tsWal.thread, &thAttr, walThreadFunc, NULL) != 0) { + wError("failed to create wal thread since %s", strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pthread_attr_destroy(&thAttr); + wDebug("wal thread is launched"); + + return TSDB_CODE_SUCCESS; +} + +static void walStopThread() { + tsWal.stop = 1; + if (tsWal.thread) { + pthread_join(tsWal.thread, NULL); + } + + wDebug("wal thread is stopped"); +} diff --git a/src/wal/src/walUtil.c b/src/wal/src/walUtil.c new file mode 100644 index 0000000000..e4d9a555b3 --- /dev/null +++ b/src/wal/src/walUtil.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "walInt.h" + +int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) { + int64_t curFileId = *nextFileId; + int64_t minFileId = INT64_MAX; + + DIR *dir = opendir(pWal->path); + if (dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + char *name = ent->d_name; + + if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { + int64_t id = atoll(name + WAL_PREFIX_LEN); + if (id <= curFileId) continue; + + if (id < minFileId) { + minFileId = id; + } + } + } + closedir(dir); + + if (minFileId == INT64_MAX) return -1; + + *nextFileId = minFileId; + wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " nextFileId:%" PRId64, pWal->vgId, pWal->path, curFileId, *nextFileId); + + return 0; +} + +int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId) { + int64_t minFileId = INT64_MAX; + + DIR *dir = opendir(pWal->path); + if (dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + char *name = ent->d_name; + + if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { + int64_t id = atoll(name + WAL_PREFIX_LEN); + if (id >= curFileId) continue; + + minDiff--; + if (id < minFileId) { + minFileId = id; + } + } + } + closedir(dir); + + if (minFileId == INT64_MAX) return -1; + if (minDiff > 0) return -1; + + *oldFileId = minFileId; + wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " oldFildId:%" PRId64, pWal->vgId, pWal->path, curFileId, *oldFileId); + + return 0; +} + +int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) { + int64_t maxFileId = INT64_MIN; + + DIR *dir = opendir(pWal->path); + if (dir == NULL) { + wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + return -1; + } + + struct dirent *ent; + while ((ent = readdir(dir)) != NULL) { + char *name = ent->d_name; + + if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { + int64_t id = atoll(name + WAL_PREFIX_LEN); + if (id > maxFileId) { + maxFileId = id; + } + } + } + closedir(dir); + + if (maxFileId == INT64_MIN) { + *newFileId = 0; + } else { + *newFileId = maxFileId; + } + + wTrace("vgId:%d, path:%s, newFileId:%" PRId64, pWal->vgId, pWal->path, *newFileId); + + return 0; +} \ No newline at end of file diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 95587caa14..e57cb0e042 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -14,239 +14,101 @@ */ #define _DEFAULT_SOURCE +#define TAOS_RANDOM_FILE_FAIL_TEST #include "os.h" +#include "taoserror.h" +#include "tchecksum.h" #include "twal.h" #include "walInt.h" -#include "walMgmt.h" -#include "tchecksum.h" -#include "tutil.h" -#include "ttimer.h" -#include "taoserror.h" -#include "twal.h" -#include "tqueue.h" +static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId); -typedef struct { - uint64_t version; - int fd; - int keep; - int level; - int32_t fsyncPeriod; - void *timer; - void *signature; - int max; // maximum number of wal files - uint32_t id; // increase continuously - int num; // number of wal files - char path[TSDB_FILENAME_LEN]; - char name[TSDB_FILENAME_LEN+16]; - pthread_mutex_t mutex; -} SWal; - -static void *walTmrCtrl = NULL; -static int tsWalNum = 0; -static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT; -static int walHandleExistingFiles(const char *path); -static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp); -static int walRemoveWalFiles(const char *path); -static void walProcessFsyncTimer(void *param, void *tmrId); -static void walRelease(SWal *pWal); -static int walGetMaxOldFileId(char *odir); - -static void walModuleInitFunc() { - walTmrCtrl = taosTmrInit(1000, 100, 300000, "WAL"); - if (walTmrCtrl == NULL) - walModuleInit = PTHREAD_ONCE_INIT; - else - wDebug("WAL module is initialized"); -} - -static inline bool walNeedFsyncTimer(SWal *pWal) { - if (pWal->fsyncPeriod > 0 && pWal->level == TAOS_WAL_FSYNC) { - return true; - } - return false; -} - -void *walOpen(const char *path, const SWalCfg *pCfg) { - SWal *pWal = calloc(sizeof(SWal), 1); - if (pWal == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - pthread_once(&walModuleInit, walModuleInitFunc); - if (walTmrCtrl == NULL) { - free(pWal); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - atomic_add_fetch_32(&tsWalNum, 1); - pWal->fd = -1; - pWal->max = pCfg->wals; - pWal->id = 0; - pWal->num = 0; - pWal->level = pCfg->walLevel; - pWal->keep = pCfg->keep; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - pWal->signature = pWal; - tstrncpy(pWal->path, path, sizeof(pWal->path)); - pthread_mutex_init(&pWal->mutex, NULL); - - if (walNeedFsyncTimer(pWal)) { - pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl); - if (pWal->timer == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - walRelease(pWal); - return NULL; - } - } - - if (taosMkDir(path, 0755) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("wal:%s, failed to create directory(%s)", path, strerror(errno)); - walRelease(pWal); - pWal = NULL; - } - - if (pCfg->keep == 1) return pWal; - - if (walHandleExistingFiles(path) == 0) walRenew(pWal); - - if (pWal && pWal->fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("wal:%s, failed to open(%s)", path, strerror(errno)); - walRelease(pWal); - pWal = NULL; - } - - if (pWal) wDebug("wal:%s, it is open, level:%d fsyncPeriod:%d", path, pWal->level, pWal->fsyncPeriod); - return pWal; -} - -int walAlter(twalh wal, const SWalCfg *pCfg) { - SWal *pWal = wal; - if (pWal == NULL) { - return TSDB_CODE_WAL_APP_ERROR; - } - - if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { - wDebug("wal:%s, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->name, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); - return TSDB_CODE_SUCCESS; - } - - wInfo("wal:%s, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->name, pWal->level, pWal->fsyncPeriod, - pCfg->walLevel, pCfg->fsyncPeriod); - - pthread_mutex_lock(&pWal->mutex); - pWal->level = pCfg->walLevel; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - if (walNeedFsyncTimer(pWal)) { - wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod); - taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer, walTmrCtrl); - } else { - wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod); - taosTmrStop(pWal->timer); - pWal->timer = NULL; - } - pthread_mutex_unlock(&pWal->mutex); - - return TSDB_CODE_SUCCESS; -} - -void walClose(void *handle) { - if (handle == NULL) return; - - SWal *pWal = handle; - taosClose(pWal->fd); - if (pWal->timer) taosTmrStopA(&pWal->timer); - - if (pWal->keep == 0) { - // remove all files in the directory - for (int i = 0; i < pWal->num; ++i) { - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id - i); - if (remove(pWal->name) < 0) { - wError("wal:%s, failed to remove", pWal->name); - } else { - wDebug("wal:%s, it is removed", pWal->name); - } - } - } else { - wDebug("wal:%s, it is closed and kept", pWal->name); - } - - walRelease(pWal); -} - -int walRenew(void *handle) { +int32_t walRenew(void *handle) { if (handle == NULL) return 0; - SWal *pWal = handle; - terrno = 0; + SWal * pWal = handle; + int32_t code = 0; + + if (pWal->stop) { + wDebug("vgId:%d, do not create a new wal file", pWal->vgId); + return 0; + } pthread_mutex_lock(&pWal->mutex); if (pWal->fd >= 0) { - close(pWal->fd); - pWal->id++; - wDebug("wal:%s, it is closed", pWal->name); + tclose(pWal->fd); + wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); } - pWal->num++; + if (pWal->keep) { + pWal->fileId = 0; + } else { + if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0; + pWal->fileId++; + } - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", pWal->path, walPrefix, pWal->id); + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (pWal->fd < 0) { - wError("wal:%s, failed to open(%s)", pWal->name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + code = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); } else { - wDebug("wal:%s, it is created", pWal->name); + wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name); + } - if (pWal->num > pWal->max) { - // remove the oldest wal file - char name[TSDB_FILENAME_LEN * 3]; - snprintf(name, sizeof(name), "%s/%s%d", pWal->path, walPrefix, pWal->id - pWal->max); - if (remove(name) < 0) { - wError("wal:%s, failed to remove(%s)", name, strerror(errno)); + if (!pWal->keep) { + // remove the oldest wal file + int64_t oldFileId = -1; + if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) { + char walName[WAL_FILE_LEN] = {0}; + snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId); + + if (remove(walName) < 0) { + wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno)); } else { - wDebug("wal:%s, it is removed", name); + wDebug("vgId:%d, file:%s, it is removed", pWal->vgId, walName); } - - pWal->num--; } } pthread_mutex_unlock(&pWal->mutex); - return terrno; + return code; } -int walWrite(void *handle, SWalHead *pHead) { - SWal *pWal = handle; - if (pWal == NULL) return -1; +int32_t walWrite(void *handle, SWalHead *pHead) { + if (handle == NULL) return -1; - terrno = 0; + SWal * pWal = handle; + int32_t code = 0; // no wal + if (pWal->fd <= 0) return 0; if (pWal->level == TAOS_WAL_NOLOG) return 0; if (pHead->version <= pWal->version) return 0; - pHead->signature = walSignature; + pHead->signature = WAL_SIGNATURE; taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead)); - int contLen = pHead->len + sizeof(SWalHead); + int32_t contLen = pHead->len + sizeof(SWalHead); - if (taosTWrite(pWal->fd, pHead, contLen) != contLen) { - wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + pthread_mutex_lock(&pWal->mutex); + + if (taosWrite(pWal->fd, pHead, contLen) != contLen) { + code = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno)); } else { + wTrace("vgId:%d, fileId:%" PRId64 " fd:%d, write wal ver:%" PRId64 ", head ver:%" PRIu64 ", len:%d ", pWal->vgId, + pWal->fileId, pWal->fd, pWal->version, pHead->version, pHead->len); pWal->version = pHead->version; } + + pthread_mutex_unlock(&pWal->mutex); + ASSERT(contLen == pHead->len + sizeof(SWalHead)); - return 0; + return code; } void walFsync(void *handle) { @@ -255,295 +117,203 @@ void walFsync(void *handle) { if (pWal->fsyncPeriod == 0) { if (fsync(pWal->fd) < 0) { - wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno)); + wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno)); } } } -int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) { - SWal *pWal = handle; - struct dirent *ent; - int count = 0; - uint32_t maxId = 0, minId = -1, index =0; +int32_t walRestore(void *handle, void *pVnode, int32_t (*writeFp)(void *, void *, int32_t)) { + if (handle == NULL) return -1; - terrno = 0; - int plen = strlen(walPrefix); - char opath[TSDB_FILENAME_LEN + 5]; + SWal * pWal = handle; + int32_t count = 0; + int32_t code = 0; + int64_t fileId = -1; - int slen = snprintf(opath, sizeof(opath), "%s", pWal->path); - if (pWal->keep == 0) strcpy(opath + slen, "/old"); + while ((code = walGetNextFile(pWal, &fileId)) >= 0) { + if (fileId == pWal->fileId) continue; - DIR *dir = opendir(opath); - if (dir == NULL && errno == ENOENT) return 0; - if (dir == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } + char walName[WAL_FILE_LEN]; + snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - index = atol(ent->d_name + plen); - if (index > maxId) maxId = index; - if (index < minId) minId = index; - count++; + wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName); + int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); + if (code != TSDB_CODE_SUCCESS) { + wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code)); + continue; } + + if (!pWal->keep) { + wDebug("vgId:%d, file:%s, restore success, remove this file", pWal->vgId, walName); + remove(walName); + } else { + wDebug("vgId:%d, file:%s, restore success and keep it", pWal->vgId, walName); + } + + count++; } - closedir(dir); + if (!pWal->keep) return TSDB_CODE_SUCCESS; if (count == 0) { - if (pWal->keep) terrno = walRenew(pWal); - return terrno; - } - - if (count != (maxId - minId + 1)) { - wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId); - terrno = TSDB_CODE_WAL_APP_ERROR; + wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId); + return walRenew(pWal); } else { - wDebug("wal:%s, %d files will be restored", opath, count); - - for (index = minId; index <= maxId; ++index) { - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, index); - terrno = walRestoreWalFile(pWal, pVnode, writeFp); - if (terrno < 0) continue; + // open the existing WAL file in append mode + pWal->fileId = 0; + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); + pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + if (pWal->fd < 0) { + wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); } + wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name); } - if (terrno == 0) { - if (pWal->keep == 0) { - terrno = walRemoveWalFiles(opath); - if (terrno == 0) { - if (remove(opath) < 0) { - wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } - } - } else { - // open the existing WAL file in append mode - pWal->num = count; - pWal->id = maxId; - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%d", opath, walPrefix, maxId); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - if (pWal->fd < 0) { - wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } - } - } - - return terrno; + return TSDB_CODE_SUCCESS; } -int walGetWalFile(void *handle, char *name, uint32_t *index) { - SWal * pWal = handle; - int code = 1; - int32_t first = 0; +int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { + if (handle == NULL) return -1; + SWal *pWal = handle; - name[0] = 0; - if (pWal == NULL || pWal->num == 0) return 0; + // for keep + if (*fileId == 0) *fileId = -1; pthread_mutex_lock(&(pWal->mutex)); - first = pWal->id + 1 - pWal->num; - if (*index == 0) *index = first; // set to first one - - if (*index < first && *index > pWal->id) { - code = -1; // index out of range - } else { - sprintf(name, "wal/%s%d", walPrefix, *index); - code = (*index == pWal->id) ? 0 : 1; + int32_t code = walGetNextFile(pWal, fileId); + if (code >= 0) { + sprintf(fileName, "wal/%s%" PRId64, WAL_PREFIX, *fileId); + code = (*fileId == pWal->fileId) ? 0 : 1; } + wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId); pthread_mutex_unlock(&(pWal->mutex)); return code; } -static void walRelease(SWal *pWal) { - pthread_mutex_destroy(&pWal->mutex); - pWal->signature = NULL; - free(pWal); - - if (atomic_sub_fetch_32(&tsWalNum, 1) == 0) { - if (walTmrCtrl) taosTmrCleanUp(walTmrCtrl); - walTmrCtrl = NULL; - walModuleInit = PTHREAD_ONCE_INIT; - wDebug("WAL module is cleaned up"); - } +static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) { + taosFtruncate(fd, offset); + fsync(fd); } -static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { - char *name = pWal->name; - int size = 1024 * 1024; // default 1M buffer size - - terrno = 0; - char *buffer = malloc(size); - if (buffer == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - SWalHead *pHead = (SWalHead *)buffer; - - int fd = open(name, O_RDWR); - if (fd < 0) { - wError("wal:%s, failed to open for restore(%s)", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - free(buffer); - return terrno; - } - - wDebug("wal:%s, start to restore", name); - - size_t offset = 0; +static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) { + int64_t pos = *offset; while (1) { - int ret = taosTRead(fd, pHead, sizeof(SWalHead)); + pos++; + + if (lseek(fd, pos, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno)); + return TSDB_CODE_WAL_FILE_CORRUPTED; + } + + if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) { + wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); + return TSDB_CODE_WAL_FILE_CORRUPTED; + } + + if (pHead->signature != WAL_SIGNATURE) { + continue; + } + + if (taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { + wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos); + *offset = pos; + return TSDB_CODE_SUCCESS; + } + } + + return TSDB_CODE_WAL_FILE_CORRUPTED; +} + +static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) { + int32_t size = WAL_MAX_SIZE; + void * buffer = tmalloc(size); + if (buffer == NULL) { + wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + int32_t fd = open(name, O_RDWR); + if (fd < 0) { + wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); + tfree(buffer); + return TAOS_SYSTEM_ERROR(errno); + } + + wDebug("vgId:%d, file:%s, start to restore", pWal->vgId, name); + + int32_t code = TSDB_CODE_SUCCESS; + int64_t offset = 0; + SWalHead *pHead = buffer; + + while (1) { + int32_t ret = taosRead(fd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { - wError("wal:%s, failed to read wal head part since %s", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to read wal head since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); break; } if (ret < sizeof(SWalHead)) { - wError("wal:%s, failed to read head, ret:%d, skip the rest of file", name, ret); - taosFtruncate(fd, offset); - fsync(fd); + wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret); + walFtruncate(pWal, fd, offset); break; } if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { - wWarn("wal:%s, cksum is messed up, skip the rest of file", name); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - ASSERT(false); - break; + wError("vgId:%d, file:%s, wal head cksum is messed up, ver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, fd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, fd, offset); + break; + } } if (pHead->len > size - sizeof(SWalHead)) { size = sizeof(SWalHead) + pHead->len; buffer = realloc(buffer, size); if (buffer == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); break; } - pHead = (SWalHead *)buffer; + pHead = buffer; } - ret = taosTRead(fd, pHead->cont, pHead->len); + ret = taosRead(fd, pHead->cont, pHead->len); if (ret < 0) { - wError("wal:%s failed to read wal body part since %s", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); break; } if (ret < pHead->len) { - wError("wal:%s, failed to read body, len:%d ret:%d, skip the rest of file", name, pHead->len, ret); - taosFtruncate(fd, offset); - fsync(fd); - break; + wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len); + offset += sizeof(SWalHead); + continue; } offset = offset + sizeof(SWalHead) + pHead->len; + wTrace("vgId:%d, fileId:%" PRId64 ", restore wal ver:%" PRIu64 ", head ver:%" PRIu64 " len:%d", pWal->vgId, fileId, + pWal->version, pHead->version, pHead->len); + if (pWal->keep) pWal->version = pHead->version; + (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL); } - close(fd); - free(buffer); + tclose(fd); + tfree(buffer); - return terrno; -} - -int walHandleExistingFiles(const char *path) { - char oname[TSDB_FILENAME_LEN * 3]; - char nname[TSDB_FILENAME_LEN * 3]; - char opath[TSDB_FILENAME_LEN]; - - snprintf(opath, sizeof(opath), "%s/old", path); - - struct dirent *ent; - DIR *dir = opendir(path); - int plen = strlen(walPrefix); - terrno = 0; - - int midx = walGetMaxOldFileId(opath); - int count = 0; - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - midx++; - snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name); - snprintf(nname, sizeof(nname), "%s/old/wal%d", path, midx); - if (taosMkDir(opath, 0755) != 0) { - wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (rename(oname, nname) < 0) { - wError("wal:%s, failed to move to new:%s", oname, nname); - terrno = TAOS_SYSTEM_ERROR(errno); - break; - } - - count++; - } - - wDebug("wal:%s, %d files are moved for restoration", path, count); - } - - closedir(dir); - return terrno; -} - -static int walRemoveWalFiles(const char *path) { - int plen = strlen(walPrefix); - char name[TSDB_FILENAME_LEN * 3]; - - terrno = 0; - - struct dirent *ent; - DIR *dir = opendir(path); - if (dir == NULL && errno == ENOENT) return 0; - if (dir == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - - while ((ent = readdir(dir)) != NULL) { - if (strncmp(ent->d_name, walPrefix, plen) == 0) { - snprintf(name, sizeof(name), "%s/%s", path, ent->d_name); - if (remove(name) < 0) { - wError("wal:%s, failed to remove(%s)", name, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - } - } - } - - closedir(dir); - - return terrno; -} - -static void walProcessFsyncTimer(void *param, void *tmrId) { - SWal *pWal = param; - - if (pWal->signature != pWal) return; - if (pWal->fd < 0) return; - - if (fsync(pWal->fd) < 0) { - wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno)); - } - - if (walNeedFsyncTimer(pWal)) { - pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl); - } else { - wInfo("wal:%s, stop fsync timer for walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod); - taosTmrStop(pWal->timer); - pWal->timer = NULL; - } + return code; } int64_t walGetVersion(twalh param) { @@ -552,29 +322,3 @@ int64_t walGetVersion(twalh param) { return pWal->version; } - -static int walGetMaxOldFileId(char *odir) { - int midx = 0; - DIR * dir = NULL; - struct dirent *dp = NULL; - int plen = strlen(walPrefix); - - if (access(odir, F_OK) != 0) return midx; - - dir = opendir(odir); - if (dir == NULL) { - wError("failed to open directory %s since %s", odir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - while ((dp = readdir(dir)) != NULL) { - if (strncmp(dp->d_name, walPrefix, plen) == 0) { - int idx = atol(dp->d_name + plen); - if (midx < idx) midx = idx; - } - } - - closedir(dir); - return midx; -} \ No newline at end of file diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index bbee1347b8..186f2ef5ff 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -115,17 +115,17 @@ int main(int argc, char *argv[]) { printf("%d wal files are written\n", total); - uint32_t index = 0; - char name[256]; + int64_t index = 0; + char name[256]; while (1) { int code = walGetWalFile(pWal, name, &index); if (code == -1) { - printf("failed to get wal file, index:%d\n", index); + printf("failed to get wal file, index:%" PRId64 "\n", index); break; } - printf("index:%d wal:%s\n", index, name); + printf("index:%" PRId64 " wal:%s\n", index, name); if (code == 0) break; index++; diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go index 14a67b93d3..2c3a7d09b6 100644 --- a/tests/examples/go/taosdemo.go +++ b/tests/examples/go/taosdemo.go @@ -87,7 +87,7 @@ func init() { func printAllArgs() { fmt.Printf("\n============= args parse result: =============\n") - fmt.Printf("dbName: %v\n", configPara.hostName) + fmt.Printf("hostName: %v\n", configPara.hostName) fmt.Printf("serverPort: %v\n", configPara.serverPort) fmt.Printf("usr: %v\n", configPara.user) fmt.Printf("password: %v\n", configPara.password) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py new file mode 100644 index 0000000000..a6c9b074e1 --- /dev/null +++ b/tests/pytest/insert/restfulInsert.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import requests +import threading +import random +import time + +class RestfulInsert: + def init(self): + self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} + self.url = "http://127.0.0.1:6041/rest/sql" + self.ts = 1500000000000 + self.numOfThreads = 20 + self.numOfTables = 10000 + self.recordsPerTable = 10000 + self.batchSize = 1000 + self.tableNamePerfix = 't' + + def createTable(self, threadID): + tablesPerThread = int (self.numOfTables / self.numOfThreads) + print("create table %d to %d" % (tablesPerThread * threadID, tablesPerThread * (threadID + 1) - 1)) + for i in range(tablesPerThread): + tableID = threadID * tablesPerThread + name = 'beijing' if tableID % 2 == 0 else 'shanghai' + data = "create table test.%s%d using test.meters tags(%d, '%s')" % (self.tableNamePerfix, tableID + i, tableID + i, name) + requests.post(self.url, data, headers = self.header) + + def insertData(self, threadID): + print("thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + for i in range(tablesPerThread): + tableID = i + threadID * tablesPerThread + start = self.ts + for j in range(int(self.recordsPerTable / self.batchSize)): + data = "insert into test.%s%d values" % (self.tableNamePerfix, tableID) + for k in range(self.batchSize): + data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) + requests.post(self.url, data, headers = self.header) + + def run(self): + data = "drop database if exists test" + requests.post(self.url, data, headers = self.header) + data = "create database test" + requests.post(self.url, data, headers = self.header) + data = "create table test.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" + requests.post(self.url, data, headers = self.header) + + threads = [] + startTime = time.time() + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.createTable, args=(i,)) + thread.start() + threads.append(thread) + for i in range(self.numOfThreads): + threads[i].join() + print("createing %d tables takes %d seconds" % (self.numOfTables, (time.time() - startTime))) + + print("inserting data =======") + threads = [] + startTime = time.time() + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.insertData, args=(i,)) + thread.start() + threads.append(thread) + + for i in range(self.numOfThreads): + threads[i].join() + print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime))) + +ri = RestfulInsert() +ri.init() +ri.run() \ No newline at end of file diff --git a/tests/pytest/query/sliding.py b/tests/pytest/query/sliding.py new file mode 100644 index 0000000000..810d90117a --- /dev/null +++ b/tests/pytest/query/sliding.py @@ -0,0 +1,63 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1500000000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table meters(ts timestamp, col1 int) tags(id int, loc nchar(20))") + sql = "insert into t0 using meters tags(1, 'beijing') values" + for i in range(100): + sql += "(%d, %d)" % (self.ts + i * 1000, random.randint(1, 100)) + tdSql.execute(sql) + + sql = "insert into t1 using meters tags(2, 'shanghai') values" + for i in range(100): + sql += "(%d, %d)" % (self.ts + i * 1000, random.randint(1, 100)) + tdSql.execute(sql) + + tdSql.query("select count(*) from meters interval(10s) sliding(5s)") + tdSql.checkRows(21) + + tdSql.error("select count(*) from meters sliding(5s)") + + tdSql.error("select count(*) from meters sliding(5s) interval(10s)") + + tdSql.error("select * from meters sliding(5s) order by ts desc") + + tdSql.query("select count(*) from meters group by loc") + tdSql.checkRows(2) + + tdSql.error("select * from meters group by loc sliding(5s)") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/script/general/insert/basic.sim b/tests/script/general/insert/basic.sim index ba8cff83fa..3f0f25a95b 100644 --- a/tests/script/general/insert/basic.sim +++ b/tests/script/general/insert/basic.sim @@ -8,8 +8,8 @@ sleep 3000 sql connect $i = 0 -$dbPrefix = tb_in_db -$tbPrefix = tb_in_tb +$dbPrefix = d +$tbPrefix = t $db = $dbPrefix . $i $tb = $tbPrefix . $i @@ -22,28 +22,27 @@ sql create table $tb (ts timestamp, speed int) $x = 0 while $x < 10 - $ms = $x . m - sql insert into $tb values (now + $ms , $x ) + $cc = $x * 60000 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x ) $x = $x + 1 endw print =============== step 2 -sql insert into $tb values (now - 5m , 10) -sql insert into $tb values (now - 6m , 10) -sql insert into $tb values (now - 7m , 10) -sql insert into $tb values (now - 8m , 10) +$x = 0 +while $x < 5 + $cc = $x * 60000 + $ms = 1551481600000 + $cc + + sql insert into $tb values ($ms , $x ) + $x = $x + 1 +endw sql select * from $tb print $rows points data are retrieved -if $rows != 14 then - return -1 -endi - -sql drop database $db -sleep 1000 -sql show databases -if $rows != 0 then +if $rows != 15 then return -1 endi diff --git a/tests/script/unique/dnode/offline1.sim b/tests/script/unique/dnode/offline1.sim index 02d03dee97..beebbfda60 100644 --- a/tests/script/unique/dnode/offline1.sim +++ b/tests/script/unique/dnode/offline1.sim @@ -49,7 +49,7 @@ print dnode1 $data4_2 if $data4_1 != ready then return -1 endi -if $data4_2 != offline then +if $data4_2 == ready then return -1 endi diff --git a/tests/test-all.sh b/tests/test-all.sh index e45dd15fed..f4e992eb5a 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -9,7 +9,7 @@ NC='\033[0m' function runSimCaseOneByOne { while read -r line; do - if [[ $line =~ ^./test.sh* ]]; then + if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then case=`echo $line | grep sim$ |awk '{print $NF}'` start_time=`date +%s`