xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/crash_gen

This commit is contained in:
Steven Li 2020-09-27 23:43:09 +00:00
parent d1e2f0209c 72cef1d81d
commit 664b9bff3a
71 changed files with 2093 additions and 614 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmake/install.inc
View File

@ -10,9 +10,8 @@ ELSEIF (TD_WINDOWS)
SET(CMAKE_INSTALL_PREFIX C:/TDengine)
ENDIF ()
IF (NOT TD_GODLL)
#INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
#INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
@ -25,6 +24,7 @@ ELSEIF (TD_WINDOWS)
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .)
ELSE ()
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosdemo.exe DESTINATION .)
ENDIF ()
#INSTALL(TARGETS taos RUNTIME DESTINATION driver)
@ -32,10 +32,6 @@ ELSEIF (TD_WINDOWS)
IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.0-dist.jar DESTINATION connector/jdbc)
ENDIF ()
ELSE ()
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll.a DESTINATION driver)
ENDIF ()
ELSEIF (TD_DARWIN)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")

13
deps/zlib-1.2.11/CMakeLists.txt vendored
View File

@ -1,8 +1,11 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_LINUX)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(z ${SRC})
ENDIF ()
IF (TD_WINDOWS)
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /WX-")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /WX-")
ENDIF()
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(z ${SRC})

4
documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
View File

@ -98,12 +98,12 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
KEEP参数是指修改数据文件保存的天数，缺省值为3650，取值范围[days, 365000]，必须大于或等于days参数值。
```mysql
ALTER DATABASE db_name QUORUM 365;
ALTER DATABASE db_name QUORUM 2;
```
QUORUM参数是指数据写入成功所需要的确认数。取值范围[1, 3]。对于异步复制，quorum设为1，具有master角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于2。原则上，Quorum >=1 并且 Quorum <= replica(副本数)，这个参数在启动一个同步模块实例时需要提供。
```mysql
ALTER DATABASE db_name BLOCKS 365;
ALTER DATABASE db_name BLOCKS 100;
```
BLOCKS参数是每个VNODE (TSDB) 中有多少cache大小的内存块，因此一个VNODE的用的内存大小粗略为（cache * blocks）。取值范围[3, 1000]。

10
documentation20/webdocs/markdowndocs/advanced features-ch.md
View File

@ -39,7 +39,7 @@ create table D1002 using meters tags ("Beijing.Haidian", 2);
我们已经知道，可以通过下面这条SQL语句以一分钟为时间窗口、30秒为前向增量统计这些电表的平均电压。
```sql
select avg(voltage) from meters interval(1m) sliding(30s)；
select avg(voltage) from meters interval(1m) sliding(30s);
```
每次执行这条语句，都会重新计算所有数据。
@ -47,14 +47,14 @@ select avg(voltage) from meters interval(1m) sliding(30s)；
可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行：
```sql
select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s)；
select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s);
```
这样做没有问题，但TDengine提供了更简单的方法，
只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如：
```sql
create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s)；
create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s);
```
会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句，
@ -80,7 +80,7 @@ taos> select * from avg_vol;
比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。
```sql
create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s)；
create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s);
```
需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。
@ -396,7 +396,7 @@ ts: 1597465200000 current: 11.2 voltage: 220 phase: 1 location: Beijing.Haidian
```sql
# taos
taos> use power；
taos> use power;
taos> insert into d1001 values("2020-08-15 12:40:00.000", 12.4, 220, 1);
```

4
documentation20/webdocs/markdowndocs/architecture-ch.md
View File

@ -276,14 +276,14 @@ SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成
在TDengine中引入关键词interval来进行时间轴上固定长度时间窗口的切分，并按照时间窗口对数据进行聚合，对窗口范围内的数据按需进行聚合。例如：
```mysql
select count(*) from d1001 interval(1h)；
select count(*) from d1001 interval(1h);
```
针对d1001设备采集的数据，按照1小时的时间窗口返回每小时存储的记录数量。
在需要连续获得查询结果的应用场景下，如果给定的时间区间存在数据缺失，会导致该区间数据结果也丢失。TDengine提供策略针对时间轴聚合计算的结果进行插值，通过使用关键词Fill就能够对时间轴聚合结果进行插值。例如：
```mysql
select count(*) from d1001 interval(1h) fill(prev)；
select count(*) from d1001 interval(1h) fill(prev);
```
针对d1001设备采集数据统计每小时记录数，如果某一个小时不存在数据，这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。

6
documentation20/webdocs/markdowndocs/cluster-ch.md
View File

@ -89,7 +89,7 @@ taos>
2. 在第一个数据节点，使用CLI程序taos, 登录进TDengine系统, 执行命令:
```
CREATE DNODE "h2.taos.com:6030"；
CREATE DNODE "h2.taos.com:6030";
```
将新数据节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新数据节点的End Point。
@ -97,7 +97,7 @@ taos>
3. 然后执行命令
```
SHOW DNODES；
SHOW DNODES;
```
查看新节点是否被成功加入。如果该被加入的数据节点处于离线状态，请做两个检查
@ -122,7 +122,7 @@ taos>
执行CLI程序taos, 使用root账号登录进系统, 执行:
```
CREATE DNODE "fqdn:port"；
CREATE DNODE "fqdn:port";
```
将新数据节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个数据节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置FQDN，可能导致生成的数据节点的End Point不是所期望的】

6
documentation20/webdocs/markdowndocs/cluster.md
View File

@ -46,7 +46,7 @@ taos>
5. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 使用命令:
```
CREATE DNODE "h2.taos.com:6030"；
CREATE DNODE "h2.taos.com:6030";
```
将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为你自己第一个节点的End Point
@ -54,7 +54,7 @@ taos>
6. 使用命令
```
SHOW DNODES；
SHOW DNODES;
```
查看新节点是否被成功加入。
@ -71,7 +71,7 @@ taos>
###添加节点
执行CLI程序taos, 使用root账号登录进系统, 执行:
```
CREATE DNODE "fqdn:port"；
CREATE DNODE "fqdn:port";
```
将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。

1
packaging/deb/makedeb.sh
View File

@ -57,6 +57,7 @@ cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_pat
cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector
cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector
cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector
cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/

1
packaging/rpm/tdengine.spec
View File

@ -64,6 +64,7 @@ cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/incl
cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector
cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector
cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples

3
packaging/tools/makeclient.sh
View File

@ -97,6 +97,8 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
cp -r ${examples_dir}/python ${install_dir}/examples
cp -r ${examples_dir}/R ${install_dir}/examples
cp -r ${examples_dir}/go ${install_dir}/examples
cp -r ${examples_dir}/nodejs ${install_dir}/examples
cp -r ${examples_dir}/C# ${install_dir}/examples
fi
# Copy driver
mkdir -p ${install_dir}/driver
@ -113,6 +115,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
cp -r ${connector_dir}/go ${install_dir}/connector
cp -r ${connector_dir}/nodejs ${install_dir}/connector
fi
# Copy release note
# cp ${script_dir}/release_note ${install_dir}

3
packaging/tools/makepkg.sh
View File

@ -113,6 +113,8 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
cp -r ${examples_dir}/python ${install_dir}/examples
cp -r ${examples_dir}/R ${install_dir}/examples
cp -r ${examples_dir}/go ${install_dir}/examples
cp -r ${examples_dir}/nodejs ${install_dir}/examples
cp -r ${examples_dir}/C# ${install_dir}/examples
fi
# Copy driver
mkdir -p ${install_dir}/driver
@ -126,6 +128,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/
cp -r ${connector_dir}/python ${install_dir}/connector/
cp -r ${connector_dir}/go ${install_dir}/connector
cp -r ${connector_dir}/nodejs ${install_dir}/connector
fi
# Copy release note
# cp ${script_dir}/release_note ${install_dir}

2
src/balance/src/balance.c
View File

@ -389,6 +389,7 @@ void balanceReset() {
pDnode->lastAccess = 0;
if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
}
mnodeDecDnodeRef(pDnode);
@ -551,6 +552,7 @@ static void balanceCheckDnodeAccess() {
if (tsAccessSquence - pDnode->lastAccess > 3) {
if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
balanceSetVgroupOffline(pDnode);
}

1
src/client/inc/tsclient.h
View File

@ -277,6 +277,7 @@ typedef struct {
int8_t dataSourceType; // load data from file or not
int8_t submitSchema; // submit block is built with table schema
STagData tagData;
SHashObj *pTableList; // referred table involved in sql
SArray *pDataBlocks; // SArray<STableDataBlocks*> submit data blocks after parsing sql
} SSqlCmd;

3
src/client/src/tscParseInsert.c
View File

@ -791,7 +791,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
sql += index;
tscAllocPayload(pCmd, sizeof(STagData));
STagData *pTag = (STagData *) pCmd->payload;
STagData *pTag = &pCmd->tagData;
memset(pTag, 0, sizeof(STagData));
@ -946,7 +946,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z);
}
pCmd->payloadLen = sizeof(pTag->name) + sizeof(pTag->dataLen) + pTag->dataLen;
pTag->dataLen = htonl(pTag->dataLen);
if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {

12
src/client/src/tscSQLParser.c
View File

@ -820,19 +820,19 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
if (hasSpecifyDB(pzTableName)) {
// db has been specified in sql string so we ignore current db path
code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL);
if (code != 0) {
if (code != TSDB_CODE_SUCCESS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
} else { // get current DB name first, then set it into path
SStrToken t = {0};
getCurrentDBName(pSql, &t);
if (t.n == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} else {
code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL);
if (code != 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
if (code != TSDB_CODE_SUCCESS) {
code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
}
}

33
src/client/src/tscServer.c
View File

@ -1495,43 +1495,29 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SCMTableInfoMsg *pInfoMsg;
char * pMsg;
int msgLen = 0;
char *tmpData = NULL;
uint32_t len = pSql->cmd.payloadLen;
if (len > 0) {
if ((tmpData = calloc(1, len)) == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
// STagData is in binary format, strncpy is not available
memcpy(tmpData, pSql->cmd.payload, len);
}
SSqlCmd * pCmd = &pSql->cmd;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
pInfoMsg = (SCMTableInfoMsg *)pCmd->payload;
SCMTableInfoMsg* pInfoMsg = (SCMTableInfoMsg *)pCmd->payload;
strcpy(pInfoMsg->tableId, pTableMetaInfo->name);
pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0);
pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
char* pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg);
if (pSql->cmd.autoCreated && len > 0) {
memcpy(pInfoMsg->tags, tmpData, len);
size_t len = htonl(pCmd->tagData.dataLen);
if (pSql->cmd.autoCreated) {
if (len > 0) {
len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen);
memcpy(pInfoMsg->tags, &pCmd->tagData, len);
pMsg += len;
}
}
pCmd->payloadLen = (int32_t)(pMsg - (char*)pInfoMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META;
taosTFree(tmpData);
assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize);
return TSDB_CODE_SUCCESS;
}
@ -2184,8 +2170,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
memcpy(pNew->cmd.payload, pSql->cmd.payload, pSql->cmd.payloadLen); // tag information if table does not exists.
pNew->cmd.payloadLen = pSql->cmd.payloadLen;
memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData));
tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
pNew->fp = tscTableMetaCallBack;

1
src/client/src/tscUtil.c
View File

@ -1806,6 +1806,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
pCmd->command = cmd;
pCmd->parseFinished = 1;
pCmd->autoCreated = pSql->cmd.autoCreated;
memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData));
if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew);

2
src/common/src/tglobal.c
View File

@ -1014,7 +1014,7 @@ static void doInitGlobalConfig(void) {
cfg.ptr = &tsLogKeepDays;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.minValue = -365000;
cfg.maxValue = 365000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;

2
src/connector/go

@ -1 +1 @@
Subproject commit 06ec30a0f1762e8169bf6b9045c82bcaa52bcdf0
Subproject commit 8d7bf743852897110cbdcc7c4322cd7a74d4167b

5
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
View File

@ -179,7 +179,7 @@ public class TSDBDriver implements java.sql.Driver {
}
//load taos.cfg start
if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null){
if (info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null && info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null) {
File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR));
File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0];
List<String> endpoints = loadConfigEndpoints(cfgFile);
@ -244,7 +244,7 @@ public class TSDBDriver implements java.sql.Driver {
}
public boolean acceptsURL(String url) throws SQLException {
return (url != null && url.length() > 0 && url.trim().length() > 0) && url.toLowerCase().startsWith(URL_PREFIX);
return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX);
}
public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@ -291,7 +291,6 @@ public class TSDBDriver implements java.sql.Driver {
return null;
}
// String lowerUrl = url.toLowerCase();
if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) {
return null;
}

4
src/dnode/src/dnodeModule.c
View File

@ -147,13 +147,13 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
}
bool dnodeCheckMnodeStarting() {
if (tsModuleStatus & TSDB_MOD_MNODE) return false;
if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) return false;
SDMMnodeInfos *mnodes = dnodeGetMnodeInfos();
for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
SDMMnodeInfo *node = &mnodes->nodeInfos[i];
if (node->nodeId == dnodeGetDnodeId()) {
uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE);;
uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE);
dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus);
dnodeProcessModuleStatus(moduleStatus);
return true;

2
src/inc/taosdef.h
View File

@ -294,6 +294,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MIN_VNODES 64
#define TSDB_MAX_VNODES 2048
#define TSDB_MIN_VNODES_PER_DB 2
#define TSDB_MAX_VNODES_PER_DB 16
#define TSDB_DNODE_ROLE_ANY 0
#define TSDB_DNODE_ROLE_MGMT 1

3
src/mnode/inc/mnodeDef.h
View File

@ -69,7 +69,8 @@ typedef struct SDnodeObj {
int16_t cpuAvgUsage; // calc from sys.cpu
int16_t memoryAvgUsage; // calc from sys.mem
int16_t bandwidthUsage; // calc from sys.band
int8_t reserved2[2];
int8_t offlineReason;
int8_t reserved2[1];
} SDnodeObj;
typedef struct SMnodeObj {

22
src/mnode/inc/mnodeDnode.h
View File

@ -33,6 +33,28 @@ typedef enum {
TAOS_DN_ALTERNATIVE_ROLE_VNODE
} EDnodeAlternativeRole;
typedef enum EDnodeOfflineReason {
TAOS_DN_OFF_ONLINE = 0,
TAOS_DN_OFF_STATUS_MSG_TIMEOUT,
TAOS_DN_OFF_STATUS_NOT_RECEIVED,
TAOS_DN_OFF_RESET_BY_MNODE,
TAOS_DN_OFF_VERSION_NOT_MATCH,
TAOS_DN_OFF_DNODE_ID_NOT_MATCH,
TAOS_DN_OFF_CLUSTER_ID_NOT_MATCH,
TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH,
TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH,
TAOS_DN_OFF_MN_EQUAL_VN_NOT_MATCH,
TAOS_DN_OFF_OFFLINE_THRESHOLD_NOT_MATCH,
TAOS_DN_OFF_STATUS_INTERVAL_NOT_MATCH,
TAOS_DN_OFF_MAX_TAB_PER_VN_NOT_MATCH,
TAOS_DN_OFF_MAX_VG_PER_DB_NOT_MATCH,
TAOS_DN_OFF_ARBITRATOR_NOT_MATCH,
TAOS_DN_OFF_TIME_ZONE_NOT_MATCH,
TAOS_DN_OFF_LOCALE_NOT_MATCH,
TAOS_DN_OFF_CHARSET_NOT_MATCH,
TAOS_DN_OFF_OTHERS
} EDnodeOfflineReason;
int32_t mnodeInitDnodes();
void mnodeCleanupDnodes();

110
src/mnode/src/mnodeDnode.c
View File

@ -60,6 +60,28 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole);
static char* offlineReason[] = {
"",
"status msg timeout",
"status not received",
"status reset by mnode",
"version not match",
"dnodeId not match",
"clusterId not match",
"numOfMnodes not match",
"balance not match",
"mnEqualVn not match",
"offThreshold not match",
"interval not match",
"maxTabPerVn not match",
"maxVgPerDb not match",
"arbitrator not match",
"timezone not match",
"locale not match",
"charset not match",
"unknown",
};
static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) {
taosTFree(pOper->pObj);
return TSDB_CODE_SUCCESS;
@ -70,6 +92,7 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->lastAccess = tsAccessSquence;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
}
mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
@ -334,61 +357,69 @@ static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
mInfo("cfg dnode rsp is received");
}
static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
if (clusterCfg->numOfMnodes != htonl(tsNumOfMnodes)) {
mError("\"numOfMnodes\"[%d - %d] cfg parameters inconsistent", clusterCfg->numOfMnodes, htonl(tsNumOfMnodes));
return false;
return TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH;
}
if (clusterCfg->enableBalance != htonl(tsEnableBalance)) {
mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, htonl(tsEnableBalance));
return false;
return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH;
}
if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) {
mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum, htonl(tsMnodeEqualVnodeNum));
return false;
mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum,
htonl(tsMnodeEqualVnodeNum));
return TAOS_DN_OFF_MN_EQUAL_VN_NOT_MATCH;
}
if (clusterCfg->offlineThreshold != htonl(tsOfflineThreshold)) {
mError("\"offlineThreshold\"[%d - %d] cfg parameters inconsistent", clusterCfg->offlineThreshold, htonl(tsOfflineThreshold));
return false;
mError("\"offlineThreshold\"[%d - %d] cfg parameters inconsistent", clusterCfg->offlineThreshold,
htonl(tsOfflineThreshold));
return TAOS_DN_OFF_OFFLINE_THRESHOLD_NOT_MATCH;
}
if (clusterCfg->statusInterval != htonl(tsStatusInterval)) {
mError("\"statusInterval\"[%d - %d] cfg parameters inconsistent", clusterCfg->statusInterval, htonl(tsStatusInterval));
return false;
mError("\"statusInterval\"[%d - %d] cfg parameters inconsistent", clusterCfg->statusInterval,
htonl(tsStatusInterval));
return TAOS_DN_OFF_STATUS_INTERVAL_NOT_MATCH;
}
if (clusterCfg->maxtablesPerVnode != htonl(tsMaxTablePerVnode)) {
mError("\"maxTablesPerVnode\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxtablesPerVnode, htonl(tsMaxTablePerVnode));
return false;
mError("\"maxTablesPerVnode\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxtablesPerVnode,
htonl(tsMaxTablePerVnode));
return TAOS_DN_OFF_MAX_TAB_PER_VN_NOT_MATCH;
}
if (clusterCfg->maxVgroupsPerDb != htonl(tsMaxVgroupsPerDb)) {
mError("\"maxVgroupsPerDb\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxVgroupsPerDb, htonl(tsMaxVgroupsPerDb));
return false;
mError("\"maxVgroupsPerDb\"[%d - %d] cfg parameters inconsistent", clusterCfg->maxVgroupsPerDb,
htonl(tsMaxVgroupsPerDb));
return TAOS_DN_OFF_MAX_VG_PER_DB_NOT_MATCH;
}
if (0 != strncasecmp(clusterCfg->arbitrator, tsArbitrator, strlen(tsArbitrator))) {
mError("\"arbitrator\"[%s - %s] cfg parameters inconsistent", clusterCfg->arbitrator, tsArbitrator);
return false;
return TAOS_DN_OFF_ARBITRATOR_NOT_MATCH;
}
int64_t checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) {
mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64"] cfg parameters inconsistent", clusterCfg->timezone, tsTimezone, clusterCfg->checkTime, checkTime);
return false;
if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) &&
(checkTime != clusterCfg->checkTime)) {
mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone,
tsTimezone, clusterCfg->checkTime, checkTime);
return TAOS_DN_OFF_TIME_ZONE_NOT_MATCH;
}
if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) {
mError("\"locale\"[%s - %s] cfg parameters inconsistent", clusterCfg->locale, tsLocale);
return false;
return TAOS_DN_OFF_LOCALE_NOT_MATCH;
}
if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) {
mError("\"charset\"[%s - %s] cfg parameters inconsistent.", clusterCfg->charset, tsCharset);
return false;
return TAOS_DN_OFF_CHARSET_NOT_MATCH;
}
return true;
return 0;
}
static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
SDnodeObj *pDnode = NULL;
SDMStatusMsg *pStatus = pMsg->rpcMsg.pCont;
pStatus->dnodeId = htonl(pStatus->dnodeId);
pStatus->moduleStatus = htonl(pStatus->moduleStatus);
@ -397,11 +428,14 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
uint32_t version = htonl(pStatus->version);
if (version != tsVersion) {
mError("status msg version:%d not equal with mnode:%d", version, tsVersion);
pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp);
if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) {
pDnode->offlineReason = TAOS_DN_OFF_VERSION_NOT_MATCH;
}
mError("dnode:%d, status msg version:%d not equal with cluster:%d", pStatus->dnodeId, version, tsVersion);
return TSDB_CODE_MND_INVALID_MSG_VERSION;
}
SDnodeObj *pDnode = NULL;
if (pStatus->dnodeId == 0) {
pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp);
if (pDnode == NULL) {
@ -411,7 +445,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
} else {
pDnode = mnodeGetDnode(pStatus->dnodeId);
if (pDnode == NULL) {
mError("dnode id:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp);
pDnode = mnodeGetDnodeByEp(pStatus->dnodeEp);
if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) {
pDnode->offlineReason = TAOS_DN_OFF_DNODE_ID_NOT_MATCH;
}
mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp);
return TSDB_CODE_MND_DNODE_NOT_EXIST;
}
}
@ -426,6 +464,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
mDebug("dnode:%d %s, first access, set clusterId %s", pDnode->dnodeId, pDnode->dnodeEp, mnodeGetClusterId());
} else {
if (strncmp(pStatus->clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN - 1) != 0) {
if (pDnode != NULL && pDnode->status != TAOS_DN_STATUS_READY) {
pDnode->offlineReason = TAOS_DN_OFF_CLUSTER_ID_NOT_MATCH;
}
mError("dnode:%d, input clusterId %s not match with exist %s", pDnode->dnodeId, pStatus->clusterId,
mnodeGetClusterId());
return TSDB_CODE_MND_INVALID_CLUSTER_ID;
@ -469,16 +510,19 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
// Verify whether the cluster parameters are consistent when status change from offline to ready
bool ret = mnodeCheckClusterCfgPara(&(pStatus->clusterCfg));
if (false == ret) {
int32_t ret = mnodeCheckClusterCfgPara(&(pStatus->clusterCfg));
if (0 != ret) {
pDnode->offlineReason = ret;
mnodeDecDnodeRef(pDnode);
rpcFreeCont(pRsp);
mError("dnode:%d, %s cluster cfg parameters inconsistent", pDnode->dnodeId, pStatus->dnodeEp);
mError("dnode:%d, %s cluster cfg parameters inconsistent, reason:%s", pDnode->dnodeId, pStatus->dnodeEp,
offlineReason[ret]);
return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
}
mDebug("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
balanceSyncNotify();
balanceAsyncNotify();
}
@ -529,6 +573,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
pDnode->createdTime = taosGetTimestampMs();
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN);
taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
@ -654,13 +699,13 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE;
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 6 + VARSTR_HEADER_SIZE;
pShow->bytes[cols] = 5 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "role");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
@ -672,6 +717,12 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "offline reason");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
@ -731,6 +782,9 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo
*(int64_t *)pWrite = pDnode->createdTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]);
cols++;
numOfRows++;
mnodeDecDnodeRef(pDnode);

5
src/mnode/src/mnodeInt.c
View File

@ -39,6 +39,11 @@ void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) {
}
int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
if (pMsg->pUser != NULL) {
mDebug("app:%p:%p, user info already inited", pMsg->rpcMsg.ahandle, pMsg);
return TSDB_CODE_SUCCESS;
}
pMsg->pUser = mnodeGetUserFromConn(pMsg->rpcMsg.handle);
if (pMsg->pUser == NULL) {
return TSDB_CODE_MND_INVALID_USER;

4
src/mnode/src/mnodeTable.c
View File

@ -2093,11 +2093,11 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
pMeta->precision = pDb->cfg.precision;
pMeta->tableType = pTable->info.type;
tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN);
if (pTable->superTable) {
if (pTable->superTable != NULL) {
tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN);
}
if (pTable->info.type == TSDB_CHILD_TABLE) {
if (pTable->info.type == TSDB_CHILD_TABLE && pTable->superTable != NULL) {
pMeta->sversion = htons(pTable->superTable->sversion);
pMeta->tversion = htons(pTable->superTable->tversion);
pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags;

3
src/mnode/src/mnodeVgroup.c
View File

@ -434,7 +434,8 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
int maxVgroupsPerDb = tsMaxVgroupsPerDb;
if (maxVgroupsPerDb <= 0) {
maxVgroupsPerDb = mnodeGetOnlinDnodesCpuCoreNum();
maxVgroupsPerDb = MAX(maxVgroupsPerDb, 2);
maxVgroupsPerDb = MAX(maxVgroupsPerDb, TSDB_MIN_VNODES_PER_DB);
maxVgroupsPerDb = MIN(maxVgroupsPerDb, TSDB_MAX_VNODES_PER_DB);
}
int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES;

1
src/os/inc/osDir.h
View File

@ -25,6 +25,7 @@ void taosRemoveDir(char *rootDir);
int taosMkDir(const char *pathname, mode_t mode);
void taosRename(char* oldName, char *newName);
void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays);
int32_t taosCompressFile(char *srcFileName, char *destFileName);
#ifdef __cplusplus
}

1
src/os/src/detail/CMakeLists.txt
View File

@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(.)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
AUX_SOURCE_DIRECTORY(. SRC)
SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w)
SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w)

54
src/os/src/detail/osDir.c
View File

@ -17,6 +17,9 @@
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
#include "zlib.h"
#define COMPRESS_STEP_SIZE 163840
void taosRemoveDir(char *rootDir) {
DIR *dir = opendir(rootDir);
@ -73,11 +76,11 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
if (de->d_type & DT_DIR) {
continue;
} else {
// struct stat fState;
// if (stat(fname, &fState) < 0) {
// continue;
// }
int32_t len = (int32_t)strlen(filename);
if (len > 3 && strcmp(filename + len - 3, ".gz") == 0) {
len -= 3;
}
int64_t fileSec = 0;
for (int i = len - 1; i >= 0; i--) {
if (filename[i] == '.') {
@ -100,3 +103,46 @@ void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
closedir(dir);
rmdir(rootDir);
}
int32_t taosCompressFile(char *srcFileName, char *destFileName) {
int32_t ret = 0;
int32_t len = 0;
char * data = malloc(COMPRESS_STEP_SIZE);
FILE * srcFp = NULL;
gzFile dstFp = NULL;
srcFp = fopen(srcFileName, "r");
if (srcFp == NULL) {
ret = -1;
goto cmp_end;
}
int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
if (fd < 0) {
ret = -2;
goto cmp_end;
}
dstFp = gzdopen(fd, "wb6f");
if (dstFp == NULL) {
ret = -3;
close(fd);
goto cmp_end;
}
while (!feof(srcFp)) {
len = (int32_t)fread(data, 1, COMPRESS_STEP_SIZE, srcFp);
(void)gzwrite(dstFp, data, len);
}
cmp_end:
if (srcFp) {
fclose(srcFp);
}
if (dstFp) {
gzclose(dstFp);
}
free(data);
return ret;
}

23
src/plugins/http/src/httpParser.c
View File

@ -153,10 +153,10 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target
for (int32_t i = 0; i < HTTP_MAX_URL; i++) {
char *pSeek = strchr(pStart, '/');
if (pSeek == NULL) {
httpAppendString(pParser->path + i, pStart, strlen(pStart));
(void)httpAppendString(pParser->path + i, pStart, strlen(pStart));
break;
} else {
httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart));
(void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart));
}
pStart = pSeek + 1;
}
@ -485,11 +485,11 @@ void httpClearParser(HttpParser *parser) {
}
void httpDestroyParser(HttpParser *parser) {
if (!parser) return;
HttpContext *pContext = parser->pContext;
httpTrace("context:%p, fd:%d, destroy parser", pContext, pContext->fd);
if (!parser) return;
free(parser->method); parser->method = NULL;
free(parser->target); parser->target = NULL;
free(parser->version); parser->version = NULL;
@ -684,12 +684,13 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state,
break;
}
if (c!='0' && c!='1') {
if (c != '0' && c != '1' && c != '2') {
httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c);
ok = -1;
httpOnError(parser, 400, TSDB_CODE_HTTP_PARSE_VERSION_FAILED);
break;
}
if (httpAppendString(&parser->str, &c, 1)) {
httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c);
ok = -1;
@ -697,10 +698,14 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state,
break;
}
if (c == '0') parser->httpVersion = HTTP_VERSION_10;
else if (c == '1') parser->httpVersion = HTTP_VERSION_11;
else if (c == '2') parser->httpVersion = HTTP_VERSION_12;
else parser->httpVersion = HTTP_INVALID_VERSION;
if (c == '0')
parser->httpVersion = HTTP_VERSION_10;
else if (c == '1')
parser->httpVersion = HTTP_VERSION_11;
else if (c == '2')
parser->httpVersion = HTTP_VERSION_12;
else {
}
parser->version = strdup(parser->str.str);
if (!parser->version) {

6
src/rpc/src/rpcMain.c
View File

@ -323,14 +323,14 @@ void *rpcMallocCont(int contLen) {
tError("failed to malloc msg, size:%d", size);
return NULL;
} else {
tDebug("malloc mem: %p", start);
tDebug("malloc msg: %p", start);
}
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
}
void rpcFreeCont(void *cont) {
if ( cont ) {
if (cont) {
char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext);
free(temp);
tDebug("free mem: %p", temp);
@ -553,7 +553,7 @@ static void rpcFreeMsg(void *msg) {
if ( msg ) {
char *temp = (char *)msg - sizeof(SRpcReqContext);
free(temp);
tDebug("free mem: %p", temp);
tDebug("free msg: %p", temp);
}
}

2
src/rpc/src/rpcTcp.c
View File

@ -421,7 +421,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen);
buffer = malloc(msgLen + tsRpcOverhead);
if ( NULL == buffer) {
if (NULL == buffer) {
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
} else {

9
src/sync/inc/syncInt.h
View File

@ -118,7 +118,7 @@ typedef struct SsyncPeer {
uint32_t ip;
uint16_t port;
char fqdn[TSDB_FQDN_LEN]; // peer ip string
char id[TSDB_EP_LEN+16]; // peer vgId + end point
char id[TSDB_EP_LEN + 32]; // peer vgId + end point
int8_t role;
int8_t sstatus; // sync status
uint64_t version;
@ -127,11 +127,11 @@ typedef struct SsyncPeer {
int peerFd; // forward FD
int numOfRetrieves; // number of retrieves tried
int fileChanged; // a flag to indicate file is changed during retrieving process
void *timer;
void *pConn;
void * timer;
void * pConn;
int notifyFd;
int watchNum;
int *watchFd;
int * watchFd;
int8_t refCount; // reference count
struct SSyncNode *pSyncNode;
} SSyncPeer;
@ -171,7 +171,6 @@ void syncBroadcastStatus(SSyncNode *pNode);
void syncAddPeerRef(SSyncPeer *pPeer);
int syncDecPeerRef(SSyncPeer *pPeer);
#ifdef __cplusplus
}
#endif

1
src/sync/inc/taosTcpPool.h
View File

@ -38,7 +38,6 @@ void taosCloseTcpThreadPool(ttpool_h);
void *taosAllocateTcpConn(void *, void *ahandle, int connFd);
void taosFreeTcpConn(void *);
#ifdef __cplusplus
}
#endif

31
src/sync/src/syncMain.c
View File

@ -134,7 +134,7 @@ void syncCleanUp() {
void *syncStart(const SSyncInfo *pInfo) {
const SSyncCfg *pCfg = &pInfo->syncCfg;
SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1);
SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
if (pNode == NULL) {
sError("no memory to allocate syncNode");
terrno = TAOS_SYSTEM_ERROR(errno);
@ -167,6 +167,8 @@ void *syncStart(const SSyncInfo *pInfo) {
}
}
syncAddNodeRef(pNode);
if (pNode->selfIndex < 0) {
sInfo("vgId:%d, this node is not configured", pNode->vgId);
terrno = TSDB_CODE_SYN_INVALID_CONFIG;
@ -176,9 +178,10 @@ void *syncStart(const SSyncInfo *pInfo) {
nodeVersion = pInfo->version; // set the initial version
nodeRole = (pNode->replica > 1) ? TAOS_SYNC_ROLE_UNSYNCED : TAOS_SYNC_ROLE_MASTER;
sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum,
syncRole[nodeRole]);
pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo*sizeof(SFwdInfo), 1);
pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo * sizeof(SFwdInfo), 1);
if (pNode->pSyncFwds == NULL) {
sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
terrno = TAOS_SYSTEM_ERROR(errno);
@ -194,7 +197,6 @@ void *syncStart(const SSyncInfo *pInfo) {
}
syncAddArbitrator(pNode);
syncAddNodeRef(pNode);
taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
if (pNode->notifyRole) {
@ -442,9 +444,7 @@ static void syncAddArbitrator(SSyncNode *pNode) {
pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
}
static void syncAddNodeRef(SSyncNode *pNode) {
atomic_add_fetch_8(&pNode->refCount, 1);
}
static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); }
static void syncDecNodeRef(SSyncNode *pNode) {
if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
@ -455,9 +455,7 @@ static void syncDecNodeRef(SSyncNode *pNode) {
}
}
void syncAddPeerRef(SSyncPeer *pPeer) {
atomic_add_fetch_8(&pPeer->refCount, 1);
}
void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); }
int syncDecPeerRef(SSyncPeer *pPeer) {
if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
@ -500,6 +498,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn));
pPeer->ip = ip;
pPeer->port = pInfo->nodePort;
pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%d", pNode->vgId, pPeer->fqdn, pPeer->port);
pPeer->peerFd = -1;
@ -572,10 +571,10 @@ static void syncChooseMaster(SSyncNode *pNode) {
replica = pNode->replica + 1;
}
if (index < 0 && onlineNum > replica/2.0) {
if (index < 0 && onlineNum > replica / 2.0) {
// over half of nodes are online
for (int i = 0; i < pNode->replica; ++i) {
//slave with highest version shall be master
// slave with highest version shall be master
pPeer = pNode->peerInfo[i];
if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
if (index < 0 || pPeer->version > pNode->peerInfo[index]->version) {
@ -621,7 +620,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
if (onlineNum <= replica * 0.5) {
if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
pNode->peerInfo[pNode->selfIndex]->role = nodeRole;
// pNode->peerInfo[pNode->selfIndex]->role = nodeRole;
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
}
@ -670,7 +669,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
int8_t selfOldRole = nodeRole;
int8_t i, syncRequired = 0;
pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
// pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
pPeer->role = newRole;
sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]);
@ -876,8 +875,6 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
}
}
return;
}
static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
@ -922,7 +919,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
static int syncProcessPeerMsg(void *param, void *buffer) {
SSyncPeer *pPeer = param;
SSyncHead head;
char * cont = (char *)buffer;
char * cont = buffer;
SSyncNode *pNode = pPeer->pSyncNode;
pthread_mutex_lock(&(pNode->mutex));

53
src/sync/src/syncRestore.c
View File

@ -23,9 +23,9 @@
#include "tsync.h"
#include "syncInt.h"
static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eindex) {
char name[TSDB_FILENAME_LEN*2] = {0};
char fname[TSDB_FILENAME_LEN*3] = {0};
static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) {
char name[TSDB_FILENAME_LEN * 2] = {0};
char fname[TSDB_FILENAME_LEN * 3] = {0};
uint32_t magic;
uint64_t fversion;
int64_t size;
@ -40,7 +40,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind
if (magic == 0) break;
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
remove(fname);
(void)remove(fname);
sDebug("%s, %s is removed", pPeer->id, fname);
index++;
@ -62,34 +62,35 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
while (1) {
// read file info
int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
if (ret < 0 ) break;
if (ret < 0) break;
// if no more file from master, break;
if (minfo.name[0] == 0 || minfo.magic == 0) {
sDebug("%s, no more files to restore", pPeer->id);
// remove extra files after the current index
syncRemoveExtraFile(pPeer, sinfo.index+1, TAOS_SYNC_MAX_INDEX);
syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX);
code = 0;
break;
}
// remove extra files on slave between the current and last index
syncRemoveExtraFile(pPeer, pindex+1, minfo.index-1);
syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1);
pindex = minfo.index;
// check the file info
sinfo = minfo;
sDebug("%s, get file info:%s", pPeer->id, minfo.name);
sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size,
&sinfo.fversion);
// if file not there or magic is not the same, file shall be synced
memset(&fileAck, 0, sizeof(fileAck));
fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1:0;
fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0;
// send file ack
ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
if (ret <0) break;
if (ret < 0) break;
// if sync is not required, continue
if (fileAck.sync == 0) {
@ -99,10 +100,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
// if sync is required, open file, receive from master, and write to file
// get the full path to file
minfo.name[sizeof(minfo.name) - 1] = 0;
snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
if ( dfd < 0 ) {
if (dfd < 0) {
sError("%s, failed to open file:%s", pPeer->id, name);
break;
}
@ -110,10 +112,9 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
fsync(dfd);
close(dfd);
if (ret<0) break;
if (ret < 0) break;
sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
}
if (code == 0 && (minfo.fversion != sinfo.fversion)) {
@ -140,18 +141,21 @@ static int syncRestoreWal(SSyncPeer *pPeer) {
while (1) {
ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
if (ret <0) break;
if (ret < 0) break;
if (pHead->len == 0) {code = 0; break;} // wal sync over
if (pHead->len == 0) {
code = 0;
break;
} // wal sync over
ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
if (ret <0) break;
if (ret < 0) break;
sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
(*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL);
}
if (code<0) {
if (code < 0) {
sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno));
}
@ -159,10 +163,9 @@ static int syncRestoreWal(SSyncPeer *pPeer) {
return code;
}
static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset)
{
static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead *pHead = (SWalHead *) offset;
SWalHead * pHead = (SWalHead *)offset;
(*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
offset += pHead->len + sizeof(SWalHead);
@ -171,7 +174,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset)
}
static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
int forwards = 0;
@ -199,7 +202,7 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
}
int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
SSyncNode *pNode = pPeer->pSyncNode;
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
if (pRecv == NULL) return -1;
@ -261,7 +264,7 @@ static int syncRestoreDataStepByStep(SSyncPeer *pPeer) {
// if code > 0, data file is changed, notify app, and pass the version
if (code > 0 && pNode->notifyFileSynced) {
if ( (*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0 ) {
if ((*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0) {
sError("%s, app not in ready state", pPeer->id);
return -1;
}
@ -297,7 +300,7 @@ void *syncRestoreData(void *param) {
if (syncOpenRecvBuffer(pNode) < 0) {
sError("%s, failed to allocate recv buffer", pPeer->id);
} else {
if ( syncRestoreDataStepByStep(pPeer) == 0) {
if (syncRestoreDataStepByStep(pPeer) == 0) {
sInfo("%s, it is synced successfully", pPeer->id);
nodeRole = TAOS_SYNC_ROLE_SLAVE;
syncBroadcastStatus(pNode);
@ -311,7 +314,7 @@ void *syncRestoreData(void *param) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
nodeSStatus = TAOS_SYNC_STATUS_INIT;
taosClose(pPeer->syncFd)
taosClose(pPeer->syncFd);
syncCloseRecvBuffer(pNode);
__sync_fetch_and_sub(&tsSyncNum, 1);
syncDecPeerRef(pPeer);

63
src/sync/src/syncRetrieve.c
View File

@ -38,13 +38,13 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return -1;
}
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
}
int *wd = pPeer->watchFd + pPeer->watchNum;
@ -64,7 +64,7 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum);
}
pPeer->watchNum = (pPeer->watchNum +1) % tsMaxWatchFiles;
pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles;
return 0;
}
@ -84,7 +84,7 @@ static int syncAreFilesModified(SSyncPeer *pPeer) {
const struct inotify_event *event;
char *ptr;
for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
event = (const struct inotify_event *) ptr;
event = (const struct inotify_event *)ptr;
if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) {
sDebug("%s, processed file is changed", pPeer->id);
pPeer->fileChanged = 1;
@ -98,7 +98,7 @@ static int syncAreFilesModified(SSyncPeer *pPeer) {
}
static int syncRetrieveFile(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo fileInfo;
SFileAck fileAck;
int code = -1;
@ -110,17 +110,19 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
while (1) {
// retrieve file info
fileInfo.name[0] = 0;
fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion);
//fileInfo.size = htonl(size);
fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
&fileInfo.size, &fileInfo.fversion);
// fileInfo.size = htonl(size);
// send the file info
int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
if (ret < 0 ) break;
if (ret < 0) break;
// if no file anymore, break
if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
sDebug("%s, no more files to sync", pPeer->id);
code = 0; break;
code = 0;
break;
}
// wait for the ack from peer
@ -134,7 +136,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
// add the file into watch list
if ( syncAddIntoWatchList(pPeer, name) <0) break;
if (syncAddIntoWatchList(pPeer, name) < 0) break;
// if sync is not required, continue
if (fileAck.sync == 0) {
@ -201,13 +203,13 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
return -1;
}
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
int *wd = pPeer->watchFd;
*wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
@ -219,7 +221,7 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
return 0;
}
static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
char buf[2048];
int len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
@ -231,26 +233,29 @@ static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
struct inotify_event *event;
for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
event = (struct inotify_event *) ptr;
event = (struct inotify_event *)ptr;
if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY;
if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE;
}
if (pEvent != 0)
sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
if (pEvent != 0) sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
return 0;
}
static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
SWalHead *pHead = (SWalHead *) malloc(640000);
SWalHead *pHead = malloc(640000);
int code = -1;
int32_t bytes = 0;
int sfd;
sfd = open(name, O_RDONLY);
if (sfd < 0) return -1;
lseek(sfd, offset, SEEK_SET);
if (sfd < 0) {
free(pHead);
return -1;
}
(void)lseek(sfd, offset, SEEK_SET);
sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
while (1) {
@ -263,7 +268,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if ( ret != wsize ) break;
if (ret != wsize) break;
pPeer->sversion = pHead->version;
bytes += wsize;
@ -276,7 +281,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
free(pHead);
taosClose(sfd);
close(sfd);
if (code == 0) return bytes;
return -1;
@ -300,14 +305,14 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) {
sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
// monitor last wal
if (syncMonitorLastWal(pPeer, fname) <0) break;
if (syncMonitorLastWal(pPeer, fname) < 0) break;
while (1) {
int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event);
if (bytes < 0) break;
// check file changes
if (syncCheckLastWalChanges(pPeer, &event) <0) break;
if (syncCheckLastWalChanges(pPeer, &event) < 0) break;
// if file is not updated or updated once, set the fversion and sstatus
if (((event & IN_MODIFY) == 0) || once) {
@ -403,7 +408,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
code = taosTSendFile(pPeer->syncFd, sfd, NULL, size);
close(sfd);
if (code <0) break;
if (code < 0) break;
index++;
@ -433,7 +438,7 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort;
if (write(pPeer->syncFd, (char *) &firstPkt, sizeof(firstPkt)) < 0) {
if (write(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) {
sError("%s, failed to send syncCmd", pPeer->id);
return -1;
}
@ -447,8 +452,7 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
}
// if no files are synced, there must be wal to sync, sversion must be larger than one
if (pPeer->sversion == 0)
pPeer->sversion = 1;
if (pPeer->sversion == 0) pPeer->sversion = 1;
sDebug("%s, start to retrieve wal", pPeer->id);
if (syncRetrieveWal(pPeer) < 0) {
@ -460,7 +464,7 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
}
void *syncRetrieveData(void *param) {
SSyncPeer * pPeer = (SSyncPeer *)param;
SSyncPeer *pPeer = (SSyncPeer *)param;
SSyncNode *pNode = pPeer->pSyncNode;
taosBlockSIGPIPE();
@ -486,8 +490,7 @@ void *syncRetrieveData(void *param) {
(*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2));
} else {
pPeer->numOfRetrieves = 0;
if (pNode->notifyFlowCtrl)
(*pNode->notifyFlowCtrl)(pNode->ahandle, 0);
if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);
}
pPeer->fileChanged = 0;

49
src/sync/src/taosTcpPool.c
View File

@ -45,8 +45,8 @@ typedef struct {
static void *taosAcceptPeerTcpConnection(void *argv);
static void *taosProcessTcpData(void *param);
static void taosStopPoolThread(SThreadObj *pThread);
static SThreadObj *taosGetTcpThread(SPoolObj *pPool);
static void taosStopPoolThread(SThreadObj* pThread);
void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
pthread_attr_t thattr;
@ -59,7 +59,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
pPool->info = *pInfo;
pPool->pThread = (SThreadObj **) calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
if (pPool->pThread == NULL) {
uError("TCP server, no enough memory");
free(pPool);
@ -68,17 +68,19 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
if (pPool->acceptFd < 0) {
free(pPool->pThread); free(pPool);
free(pPool->pThread);
free(pPool);
uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
return NULL;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pPool->thread), &thattr, (void *) taosAcceptPeerTcpConnection, pPool) != 0) {
if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) {
uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
close(pPool->acceptFd);
free(pPool->pThread); free(pPool);
free(pPool->pThread);
free(pPool);
return NULL;
}
@ -89,7 +91,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
}
void taosCloseTcpThreadPool(void *param) {
SPoolObj *pPool = (SPoolObj *)param;
SPoolObj * pPool = (SPoolObj *)param;
SThreadObj *pThread;
shutdown(pPool->acceptFd, SHUT_RD);
@ -100,16 +102,17 @@ void taosCloseTcpThreadPool(void *param) {
if (pThread) taosStopPoolThread(pThread);
}
uDebug("%p TCP pool is closed", pPool);
taosTFree(pPool->pThread);
free(pPool);
uDebug("%p TCP pool is closed", pPool);
}
void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
struct epoll_event event;
SPoolObj *pPool = (SPoolObj *)param;
SConnObj *pConn = (SConnObj *) calloc(sizeof(SConnObj), 1);
SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1);
if (pConn == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@ -153,8 +156,8 @@ void taosFreeTcpConn(void *param) {
static void taosProcessBrokenLink(SConnObj *pConn) {
SThreadObj *pThread = pConn->pThread;
SPoolObj *pPool = pThread->pPool;
SPoolInfo *pInfo = &pPool->info;
SPoolObj * pPool = pThread->pPool;
SPoolInfo * pInfo = &pPool->info;
if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
(*pInfo->processBrokenLink)(pConn->ahandle);
@ -169,10 +172,10 @@ static void taosProcessBrokenLink(SConnObj *pConn) {
#define maxEvents 10
static void *taosProcessTcpData(void *param) {
SThreadObj *pThread = (SThreadObj *) param;
SPoolObj *pPool = pThread->pPool;
SPoolInfo *pInfo = &pPool->info;
SConnObj *pConn = NULL;
SThreadObj *pThread = (SThreadObj *)param;
SPoolObj * pPool = pThread->pPool;
SPoolInfo * pInfo = &pPool->info;
SConnObj * pConn = NULL;
struct epoll_event events[maxEvents];
void *buffer = malloc(pInfo->bufferSize);
@ -219,15 +222,16 @@ static void *taosProcessTcpData(void *param) {
}
}
uDebug("%p TCP epoll thread exits", pThread);
close(pThread->pollFd);
free(pThread);
free(buffer);
uDebug("%p TCP epoll thread exits", pThread);
return NULL;
}
static void *taosAcceptPeerTcpConnection(void *argv) {
SPoolObj *pPool = (SPoolObj *)argv;
SPoolObj * pPool = (SPoolObj *)argv;
SPoolInfo *pInfo = &pPool->info;
taosBlockSIGPIPE();
@ -235,7 +239,7 @@ static void *taosAcceptPeerTcpConnection(void *argv) {
while (1) {
struct sockaddr_in clientAddr;
socklen_t addrlen = sizeof(clientAddr);
int connFd = accept(pPool->acceptFd, (struct sockaddr *) &clientAddr, &addrlen);
int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 0) {
if (errno == EINVAL) {
uDebug("%p TCP server accept is exiting...", pPool);
@ -246,7 +250,7 @@ static void *taosAcceptPeerTcpConnection(void *argv) {
}
}
//uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
// uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
taosKeepTcpAlive(connFd);
(*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
}
@ -260,7 +264,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
if (pThread) return pThread;
pThread = (SThreadObj *) calloc(1, sizeof(SThreadObj));
pThread = (SThreadObj *)calloc(1, sizeof(SThreadObj));
if (pThread == NULL) return NULL;
pThread->pPool = pPool;
@ -273,7 +277,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
int ret = pthread_create(&(pThread->thread), &thattr, (void *) taosProcessTcpData, pThread);
int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
pthread_attr_destroy(&thattr);
if (ret != 0) {
@ -290,7 +294,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
return pThread;
}
static void taosStopPoolThread(SThreadObj* pThread) {
static void taosStopPoolThread(SThreadObj *pThread) {
pThread->stop = true;
if (pThread->thread == pthread_self()) {
@ -303,7 +307,7 @@ static void taosStopPoolThread(SThreadObj* pThread) {
// signal the thread to stop, try graceful method first,
// and use pthread_cancel when failed
struct epoll_event event = { .events = EPOLLIN };
struct epoll_event event = {.events = EPOLLIN};
eventfd_t fd = eventfd(1, 0);
if (fd == -1) {
// failed to create eventfd, call pthread_cancel instead, which may result in data corruption
@ -319,4 +323,3 @@ static void taosStopPoolThread(SThreadObj* pThread) {
pthread_join(thread, NULL);
taosClose(fd);
}

20
src/sync/src/tarbitrator.c
View File

@ -35,7 +35,7 @@ static tsem_t tsArbSem;
static ttpool_h tsArbTcpPool;
typedef struct {
char id[TSDB_EP_LEN+24];
char id[TSDB_EP_LEN + 24];
int nodeFd;
void *pConn;
} SNodeConn;
@ -43,12 +43,12 @@ typedef struct {
int main(int argc, char *argv[]) {
char arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
tsArbitratorPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
debugFlag = atoi(argv[++i]);
} else if (strcmp(argv[i], "-g")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-g") == 0 && i < argc - 1) {
if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue;
tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath));
} else {
@ -108,8 +108,7 @@ int main(int argc, char *argv[]) {
return 0;
}
static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp)
{
static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
char ipstr[24];
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@ -121,13 +120,14 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp)
return;
}
SNodeConn *pNode = (SNodeConn *) calloc(sizeof(SNodeConn), 1);
SNodeConn *pNode = (SNodeConn *)calloc(sizeof(SNodeConn), 1);
if (pNode == NULL) {
sError("failed to allocate memory(%s)", strerror(errno));
taosCloseSocket(connFd);
return;
}
firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0;
snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
if (firstPkt.syncHead.vgId) {
sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
@ -151,10 +151,10 @@ static void arbProcessBrokenLink(void *param) {
}
static int arbProcessPeerMsg(void *param, void *buffer) {
SNodeConn * pNode = param;
SNodeConn *pNode = param;
SSyncHead head;
int bytes = 0;
char *cont = (char *)buffer;
char * cont = (char *)buffer;
int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
if (hlen != sizeof(head)) {

60
src/sync/test/syncClient.c
View File

@ -26,14 +26,15 @@ typedef struct {
int numOfReqs;
int msgSize;
tsem_t rspSem;
tsem_t *pOverSem;
tsem_t * pOverSem;
pthread_t thread;
void *pRpc;
void * pRpc;
} SInfo;
void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle;
uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code);
uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
pMsg->code);
if (pEpSet) pInfo->epSet = *pEpSet;
rpcFreeCont(pMsg->pCont);
@ -44,12 +45,12 @@ void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
int tcount = 0;
void *sendRequest(void *param) {
SInfo *pInfo = (SInfo *)param;
SInfo * pInfo = (SInfo *)param;
SRpcMsg rpcMsg = {0};
uDebug("thread:%d, start to send request", pInfo->index);
while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) {
while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) {
pInfo->num++;
rpcMsg.pCont = rpcMallocCont(pInfo->msgSize);
rpcMsg.contLen = pInfo->msgSize;
@ -57,8 +58,9 @@ void *sendRequest(void *param) {
rpcMsg.msgType = 1;
uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
if ( pInfo->num % 20000 == 0 )
if (pInfo->num % 20000 == 0) {
uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
}
tsem_wait(&pInfo->rspSem);
}
@ -102,30 +104,30 @@ int main(int argc, char *argv[]) {
rpcInit.spi = 1;
rpcInit.connType = TAOS_CONN_CLIENT;
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
epSet.port[0] = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
strcpy(epSet.fqdn[0], argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) {
tstrncpy(epSet.fqdn[0], argv[++i], TSDB_FQDN_LEN);
} else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) {
msgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) {
rpcInit.sessions = atoi(argv[++i]);
} else if (strcmp(argv[i], "-n")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) {
numOfReqs = atoi(argv[++i]);
} else if (strcmp(argv[i], "-a")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) {
appThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) {
tsCompressMsgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-u")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) {
rpcInit.user = argv[++i];
} else if (strcmp(argv[i], "-k")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) {
rpcInit.secret = argv[++i];
} else if (strcmp(argv[i], "-spi")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) {
rpcInit.spi = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
rpcDebugFlag = atoi(argv[++i]);
} else {
printf("\nusage: %s [options] \n", argv[0]);
@ -157,14 +159,14 @@ int main(int argc, char *argv[]) {
uInfo("client is initialized");
gettimeofday(&systemTime, NULL);
startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;
startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads);
SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads);
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
for (int i=0; i<appThreads; ++i) {
for (int i = 0; i < appThreads; ++i) {
pInfo->index = i;
pInfo->epSet = epSet;
pInfo->numOfReqs = numOfReqs;
@ -177,18 +179,16 @@ int main(int argc, char *argv[]) {
do {
usleep(1);
} while ( tcount < appThreads);
} while (tcount < appThreads);
gettimeofday(&systemTime, NULL);
endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;
float usedTime = (endTime - startTime)/1000.0; // mseconds
endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
float usedTime = (endTime - startTime) / 1000.0; // mseconds
uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads);
uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize);
taosCloseLog();
return 0;
}

121
src/sync/test/syncServer.c
View File

@ -27,23 +27,22 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
void *qhandle = NULL;
void * qhandle = NULL;
int walNum = 0;
uint64_t tversion = 0;
void *syncHandle;
void * syncHandle;
int role;
int nodeId;
char path[256];
int numOfWrites ;
int numOfWrites;
SSyncInfo syncInfo;
SSyncCfg *pCfg;
int writeIntoWal(SWalHead *pHead)
{
int writeIntoWal(SWalHead *pHead) {
if (dataFd < 0) {
char walName[280];
snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum);
remove(walName);
(void)remove(walName);
dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) {
uInfo("failed to open wal file:%s(%s)", walName, strerror(errno));
@ -71,16 +70,15 @@ int writeIntoWal(SWalHead *pHead)
return 0;
}
void confirmForward(void *ahandle, void *mhandle, int32_t code)
{
SRpcMsg *pMsg = (SRpcMsg *)mhandle;
void confirmForward(void *ahandle, void *mhandle, int32_t code) {
SRpcMsg * pMsg = (SRpcMsg *)mhandle;
SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
uDebug("ver:%" PRIu64 ", confirm is received", pHead->version);
rpcFreeCont(pMsg->pCont);
SRpcMsg rpcMsg;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(msgSize);
rpcMsg.contLen = msgSize;
rpcMsg.handle = pMsg->handle;
@ -91,14 +89,13 @@ void confirmForward(void *ahandle, void *mhandle, int32_t code)
}
int processRpcMsg(void *item) {
SRpcMsg *pMsg = (SRpcMsg *)item;
SRpcMsg * pMsg = (SRpcMsg *)item;
SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
int code = -1;
if (role != TAOS_SYNC_ROLE_MASTER) {
uError("not master, write failed, role:%s", syncRole[role]);
} else {
pHead->version = ++tversion;
pHead->msgType = pMsg->msgType;
pHead->len = pMsg->contLen;
@ -111,10 +108,10 @@ int processRpcMsg(void *item) {
}
if (pCfg->quorum <= 1) {
taosFreeQitem(item);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(item);
SRpcMsg rpcMsg;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(msgSize);
rpcMsg.contLen = msgSize;
rpcMsg.handle = pMsg->handle;
@ -126,7 +123,6 @@ int processRpcMsg(void *item) {
}
int processFwdMsg(void *item) {
SWalHead *pHead = (SWalHead *)item;
if (pHead->version <= tversion) {
@ -142,11 +138,11 @@ int processFwdMsg(void *item) {
// write into cache
/*
/*
if (pHead->handle) {
syncSendFwdAck(syncHandle, pHead->handle, 0);
}
*/
*/
taosFreeQitem(item);
@ -154,7 +150,6 @@ int processFwdMsg(void *item) {
}
int processWalMsg(void *item) {
SWalHead *pHead = (SWalHead *)item;
if (pHead->version <= tversion) {
@ -168,11 +163,11 @@ int processWalMsg(void *item) {
// write into cache
/*
/*
if (pHead->handle) {
syncSendFwdAck(syncHandle, pHead->handle, 0);
}
*/
*/
taosFreeQitem(item);
@ -197,7 +192,6 @@ void *processWriteQueue(void *param) {
} else if (type == TAOS_QTYPE_FWD) {
processFwdMsg(item);
}
}
return NULL;
@ -224,7 +218,6 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char
}
void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
SRpcMsg *pTemp;
pTemp = taosAllocateQitem(sizeof(SRpcMsg));
@ -234,8 +227,7 @@ void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp);
}
uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion)
{
uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
uint32_t magic;
struct stat fstat;
char aname[280];
@ -254,7 +246,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex
}
uInfo("get file info:%s", aname);
if ( stat(aname, &fstat) < 0 ) return 0;
if (stat(aname, &fstat) < 0) return 0;
*size = fstat.st_size;
magic = fstat.st_size;
@ -263,23 +255,21 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex
}
int getWalInfo(void *ahandle, char *name, uint32_t *index) {
struct stat fstat;
char aname[280];
name[0] = 0;
if (*index + 1> walNum) return 0;
if (*index + 1 > walNum) return 0;
snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index);
sprintf(name, "wal/wal.%d", *index);
uInfo("get wal info:%s", aname);
if ( stat(aname, &fstat) < 0 ) return -1;
if (stat(aname, &fstat) < 0) return -1;
if (*index >= walNum-1) return 0; // no more
if (*index >= walNum - 1) return 0; // no more
return 1;
}
int writeToCache(void *ahandle, void *data, int type) {
@ -295,19 +285,14 @@ int writeToCache(void *ahandle, void *data, int type) {
return 0;
}
void confirmFwd(void *ahandle, int64_t version) {
return;
}
void confirmFwd(void *ahandle, int64_t version) { return; }
void notifyRole(void *ahandle, int8_t r) {
role = r;
printf("current role:%s\n", syncRole[role]);
}
void initSync() {
pCfg->replica = 1;
pCfg->quorum = 1;
syncInfo.vgId = 1;
@ -339,17 +324,15 @@ void initSync() {
taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn);
}
void doSync()
{
for (int i=0; i<5; ++i) {
if (tsSyncPort == pCfg->nodeInfo[i].nodePort)
nodeId = pCfg->nodeInfo[i].nodeId;
void doSync() {
for (int i = 0; i < 5; ++i) {
if (tsSyncPort == pCfg->nodeInfo[i].nodePort) nodeId = pCfg->nodeInfo[i].nodeId;
}
snprintf(path, sizeof(path), "/root/test/d%d", nodeId);
strcpy(syncInfo.path, path);
tstrncpy(syncInfo.path, path, sizeof(syncInfo.path));
if ( syncHandle == NULL) {
if (syncHandle == NULL) {
syncHandle = syncStart(&syncInfo);
} else {
if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL;
@ -371,29 +354,29 @@ int main(int argc, char *argv[]) {
rpcInit.numOfThreads = 1;
rpcInit.cfp = processRequestMsg;
rpcInit.sessions = 1000;
rpcInit.idleTime = tsShellActivityTimer*1500;
rpcInit.idleTime = tsShellActivityTimer * 1500;
rpcInit.afp = retrieveAuthInfo;
for (int i=1; i<argc; ++i) {
if (strcmp(argv[i], "-p")==0 && i < argc-1) {
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
rpcInit.localPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) {
rpcInit.numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) {
msgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) {
rpcInit.sessions = atoi(argv[++i]);
} else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) {
tsCompressMsgSize = atoi(argv[++i]);
} else if (strcmp(argv[i], "-w")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) {
commit = atoi(argv[++i]);
} else if (strcmp(argv[i], "-v")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-v") == 0 && i < argc - 1) {
syncInfo.version = atoi(argv[++i]);
} else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-r") == 0 && i < argc - 1) {
pCfg->replica = atoi(argv[++i]);
} else if (strcmp(argv[i], "-q")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-q") == 0 && i < argc - 1) {
pCfg->quorum = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
rpcDebugFlag = atoi(argv[++i]);
} else {
printf("\nusage: %s [options] \n", argv[0]);
@ -414,7 +397,7 @@ int main(int argc, char *argv[]) {
uDebugFlag = rpcDebugFlag;
dDebugFlag = rpcDebugFlag;
//tmrDebugFlag = rpcDebugFlag;
// tmrDebugFlag = rpcDebugFlag;
tsAsyncLog = 0;
taosInitLog("server.log", 1000000, 10);
@ -443,35 +426,39 @@ int main(int argc, char *argv[]) {
SNodesRole nroles;
while (1) {
char c = getchar();
int c = getchar();
switch(c) {
switch (c) {
case '1':
pCfg->replica = 1; doSync();
pCfg->replica = 1;
doSync();
break;
case '2':
pCfg->replica = 2; doSync();
pCfg->replica = 2;
doSync();
break;
case '3':
pCfg->replica = 3; doSync();
pCfg->replica = 3;
doSync();
break;
case '4':
pCfg->replica = 4; doSync();
pCfg->replica = 4;
doSync();
break;
case '5':
pCfg->replica = 5; doSync();
pCfg->replica = 5;
doSync();
break;
case 's':
syncGetNodesRole(syncHandle, &nroles);
for (int i=0; i<pCfg->replica; ++i)
for (int i = 0; i < pCfg->replica; ++i)
printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]);
break;
default:
break;
}
if (c=='q') break;
if (c == 'q') break;
}
syncStop(syncHandle);
@ -483,5 +470,3 @@ int main(int argc, char *argv[]) {
return 0;
}

2
src/tsdb/src/tsdbMemTable.c
View File

@ -262,7 +262,9 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
if (pIMem != NULL) {
ASSERT(pRepo->commit);
tsdbDebug("vgId:%d waiting for the commit thread", REPO_ID(pRepo));
code = pthread_join(pRepo->commitThread, NULL);
tsdbDebug("vgId:%d commit thread is finished", REPO_ID(pRepo));
if (code != 0) {
tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);

2
src/util/CMakeLists.txt
View File

@ -3,7 +3,7 @@ PROJECT(TDengine)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4)
TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(tutil m rt)

12
src/util/src/tlog.c
View File

@ -139,14 +139,22 @@ static void taosUnLockFile(int32_t fd) {
}
static void taosKeepOldLog(char *oldName) {
if (tsLogKeepDays <= 0) return;
if (tsLogKeepDays == 0) return;
int64_t fileSec = taosGetTimestampSec();
char fileName[LOG_FILE_NAME_LEN + 20];
snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec);
taosRename(oldName, fileName);
taosRemoveOldLogFiles(tsLogDir, tsLogKeepDays);
if (tsLogKeepDays < 0) {
char compressFileName[LOG_FILE_NAME_LEN + 20];
snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec);
if (taosCompressFile(fileName, compressFileName) == 0) {
(void)remove(fileName);
}
}
taosRemoveOldLogFiles(tsLogDir, ABS(tsLogKeepDays));
}
static void *taosThreadToOpenNewFile(void *param) {

11
src/vnode/src/vnodeWrite.c
View File

@ -130,8 +130,15 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
int code = TSDB_CODE_SUCCESS;
STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
if (pCfg == NULL) return terrno;
if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno;
if (pCfg == NULL) {
ASSERT(terrno != 0);
return terrno;
}
if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) {
code = terrno;
ASSERT(code != 0);
}
tsdbClearTableCfg(pCfg);
return code;

19
tests/examples/JDBC/JDBCDemo/.gitignore vendored Normal file
View File

@ -0,0 +1,19 @@
# custom
/out/
/logs/
*.jar
# Created by .ignore support plugin (hsz.mobi)
.gitignore
# Build Artifacts
.gradle/*
build/*
target/*
bin/*
dependency-reduced-pom.xml
# Eclipse Project Files
.classpath
.project
.settings/*

5
tests/examples/JDBC/JDBCDemo/pom.xml
View File

@ -65,5 +65,10 @@
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.4</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
</dependencies>
</project>

279
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java Normal file
View File

@ -0,0 +1,279 @@
package com.taosdata.example.jdbcTaosdemo;
import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig;
import com.taosdata.example.jdbcTaosdemo.task.CreateTableTask;
import com.taosdata.example.jdbcTaosdemo.task.InsertTableDatetimeTask;
import com.taosdata.example.jdbcTaosdemo.task.InsertTableTask;
import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory;
import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller;
import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil;
import org.apache.log4j.Logger;
import java.sql.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class JdbcTaosdemo {
private static Logger logger = Logger.getLogger(JdbcTaosdemo.class);
private final JdbcTaosdemoConfig config;
private Connection connection;
public JdbcTaosdemo(JdbcTaosdemoConfig config) {
this.config = config;
}
public static void main(String[] args) {
JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args);
boolean isHelp = Arrays.asList(args).contains("--help");
if (isHelp) {
JdbcTaosdemoConfig.printHelp();
return;
}
if (config.getHost() == null) {
JdbcTaosdemoConfig.printHelp();
return;
}
JdbcTaosdemo taosdemo = new JdbcTaosdemo(config);
taosdemo.init();
taosdemo.dropDatabase();
taosdemo.createDatabase();
taosdemo.useDatabase();
taosdemo.createSuperTable();
taosdemo.createTableMultiThreads();
boolean infinite = Arrays.asList(args).contains("--infinite");
if (infinite) {
logger.info("!!! Infinite Insert Mode Started. !!!!");
taosdemo.insertInfinite();
} else {
taosdemo.insertMultiThreads();
// single table select
taosdemo.selectFromTableLimit();
taosdemo.selectCountFromTable();
taosdemo.selectAvgMinMaxFromTable();
// super table select
taosdemo.selectFromSuperTableLimit();
taosdemo.selectCountFromSuperTable();
taosdemo.selectAvgMinMaxFromSuperTable();
// drop super table
if (config.isDeleteTable())
taosdemo.dropSuperTable();
taosdemo.close();
}
}
/**
* establish the connection
*/
private void init() {
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
connection = ConnectionFactory.build(config);
if (connection != null)
logger.info("[ OK ] Connection established.");
} catch (ClassNotFoundException | SQLException e) {
logger.error(e.getMessage());
throw new RuntimeException("connection failed: " + config.getHost());
}
}
/**
* create database
*/
private void createDatabase() {
String sql = SqlSpeller.createDatabaseSQL(config.getDbName(), config.getKeep(), config.getDays());
execute(sql);
}
/**
* drop database
*/
private void dropDatabase() {
String sql = SqlSpeller.dropDatabaseSQL(config.getDbName());
execute(sql);
}
/**
* use database
*/
private void useDatabase() {
String sql = SqlSpeller.useDatabaseSQL(config.getDbName());
execute(sql);
}
/**
* create super table
*/
private void createSuperTable() {
String sql = SqlSpeller.createSuperTableSQL(config.getStbName());
execute(sql);
}
/**
* create table use super table with multi threads
*/
private void createTableMultiThreads() {
try {
final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads();
List<Thread> threads = new ArrayList<>();
for (int i = 0; i < config.getNumberOfThreads(); i++) {
Thread thread = new Thread(new CreateTableTask(config, i * tableSize, tableSize), "Thread-" + i);
threads.add(thread);
thread.start();
}
for (Thread thread : threads) {
thread.join();
}
logger.info("<<< Multi Threads create table finished.");
} catch (InterruptedException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
/**
* insert data infinitely
*/
private void insertInfinite() {
try {
final long startDatetime = TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000");
final long finishDatetime = TimeStampUtil.datetimeToLong("2030-01-01 00:00:00.000");
final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads();
List<Thread> threads = new ArrayList<>();
for (int i = 0; i < config.getNumberOfThreads(); i++) {
Thread thread = new Thread(new InsertTableDatetimeTask(config, i * tableSize, tableSize, startDatetime, finishDatetime), "Thread-" + i);
threads.add(thread);
thread.start();
}
for (Thread thread : threads) {
thread.join();
}
logger.info("<<< Multi Threads insert table finished.");
} catch (InterruptedException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
private void insertMultiThreads() {
try {
final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads();
final int numberOfRecordsPerTable = config.getNumberOfRecordsPerTable();
List<Thread> threads = new ArrayList<>();
for (int i = 0; i < config.getNumberOfThreads(); i++) {
Thread thread = new Thread(new InsertTableTask(config, i * tableSize, tableSize, numberOfRecordsPerTable), "Thread-" + i);
threads.add(thread);
thread.start();
}
for (Thread thread : threads) {
thread.join();
}
logger.info("<<< Multi Threads insert table finished.");
} catch (InterruptedException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
private void selectFromTableLimit() {
String sql = SqlSpeller.selectFromTableLimitSQL(config.getDbName(), config.getTbPrefix(), 1, 10, 0);
executeQuery(sql);
}
private void selectCountFromTable() {
String sql = SqlSpeller.selectCountFromTableSQL(config.getDbName(), config.getTbPrefix(), 1);
executeQuery(sql);
}
private void selectAvgMinMaxFromTable() {
String sql = SqlSpeller.selectAvgMinMaxFromTableSQL("current", config.getDbName(), config.getTbPrefix(), 1);
executeQuery(sql);
}
private void selectFromSuperTableLimit() {
String sql = SqlSpeller.selectFromSuperTableLimitSQL(config.getDbName(), config.getStbName(), 10, 0);
executeQuery(sql);
}
private void selectCountFromSuperTable() {
String sql = SqlSpeller.selectCountFromSuperTableSQL(config.getDbName(), config.getStbName());
executeQuery(sql);
}
private void selectAvgMinMaxFromSuperTable() {
String sql = SqlSpeller.selectAvgMinMaxFromSuperTableSQL("current", config.getDbName(), config.getStbName());
executeQuery(sql);
}
private void close() {
try {
if (connection != null) {
this.connection.close();
logger.info("connection closed.");
}
} catch (SQLException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
/**
* drop super table
*/
private void dropSuperTable() {
String sql = SqlSpeller.dropSuperTableSQL(config.getDbName(), config.getStbName());
execute(sql);
}
/**
* execute sql, use this method when sql is create, alter, drop..
*/
private void execute(String sql) {
try (Statement statement = connection.createStatement()) {
long start = System.currentTimeMillis();
boolean execute = statement.execute(sql);
long end = System.currentTimeMillis();
printSql(sql, execute, (end - start));
} catch (SQLException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
private static void printSql(String sql, boolean succeed, long cost) {
System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql);
}
private void executeQuery(String sql) {
try (Statement statement = connection.createStatement()) {
long start = System.currentTimeMillis();
ResultSet resultSet = statement.executeQuery(sql);
long end = System.currentTimeMillis();
printSql(sql, true, (end - start));
printResult(resultSet);
} catch (SQLException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
private static void printResult(ResultSet resultSet) throws SQLException {
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
StringBuilder sb = new StringBuilder();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String columnLabel = metaData.getColumnLabel(i);
String value = resultSet.getString(i);
sb.append(columnLabel + ": " + value + "\t");
}
System.out.println(sb.toString());
}
}
}

153
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java Normal file
View File

@ -0,0 +1,153 @@
package com.taosdata.example.jdbcTaosdemo.domain;
public final class JdbcTaosdemoConfig {
//The host to connect to TDengine. Must insert one
private String host;
//The TCP/IP port number to use for the connection. Default is 6030.
private int port = 6030;
//The TDengine user name to use when connecting to the server. Default is 'root'
private String user = "root";
//The password to use when connecting to the server. Default is 'taosdata'
private String password = "taosdata";
//Destination database. Default is 'test'
private String dbName = "test";
//keep
private int keep = 365 * 20;
//days
private int days = 30;
//Super table Name. Default is 'meters'
private String stbName = "meters";
//Table name prefix. Default is 'd'
private String tbPrefix = "d";
//The number of tables. Default is 10.
private int numberOfTable = 10;
//The number of records per table. Default is 2
private int numberOfRecordsPerTable = 2;
//The number of records per request. Default is 100
private int numberOfRecordsPerRequest = 100;
//The number of threads. Default is 1.
private int numberOfThreads = 1;
//Delete data. Default is false
private boolean deleteTable = false;
public static void printHelp() {
System.out.println("Usage: java -jar JDBCConnectorChecker.jar [OPTION...]");
System.out.println("-h host The host to connect to TDengine. you must input one");
System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030");
System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'");
System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'");
System.out.println("-d database Destination database. Default is 'test'");
System.out.println("-m tablePrefix Table prefix name. Default is 'd'");
System.out.println("-t num_of_tables The number of tables. Default is 10");
System.out.println("-n num_of_records_per_table The number of records per table. Default is 2");
System.out.println("-r num_of_records_per_req The number of records per request. Default is 100");
System.out.println("-T num_of_threads The number of threads. Default is 1");
System.out.println("-D delete table Delete data methods. Default is false");
System.out.println("--help Give this help list");
// System.out.println("--infinite infinite insert mode");
}
/**
* parse args from command line
*
* @param args command line args
* @return JdbcTaosdemoConfig
*/
public JdbcTaosdemoConfig(String[] args) {
for (int i = 0; i < args.length; i++) {
if ("-h".equals(args[i]) && i < args.length - 1) {
host = args[++i];
}
if ("-p".equals(args[i]) && i < args.length - 1) {
port = Integer.parseInt(args[++i]);
}
if ("-u".equals(args[i]) && i < args.length - 1) {
user = args[++i];
}
if ("-P".equals(args[i]) && i < args.length - 1) {
password = args[++i];
}
if ("-d".equals(args[i]) && i < args.length - 1) {
dbName = args[++i];
}
if ("-m".equals(args[i]) && i < args.length - 1) {
tbPrefix = args[++i];
}
if ("-t".equals(args[i]) && i < args.length - 1) {
numberOfTable = Integer.parseInt(args[++i]);
}
if ("-n".equals(args[i]) && i < args.length - 1) {
numberOfRecordsPerTable = Integer.parseInt(args[++i]);
}
if ("-r".equals(args[i]) && i < args.length - 1) {
numberOfRecordsPerRequest = Integer.parseInt(args[++i]);
}
if ("-T".equals(args[i]) && i < args.length - 1) {
numberOfThreads = Integer.parseInt(args[++i]);
}
if ("-D".equals(args[i]) && i < args.length - 1) {
deleteTable = Boolean.parseBoolean(args[++i]);
}
}
}
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public String getUser() {
return user;
}
public String getPassword() {
return password;
}
public String getDbName() {
return dbName;
}
public int getKeep() {
return keep;
}
public int getDays() {
return days;
}
public String getStbName() {
return stbName;
}
public String getTbPrefix() {
return tbPrefix;
}
public int getNumberOfTable() {
return numberOfTable;
}
public int getNumberOfRecordsPerTable() {
return numberOfRecordsPerTable;
}
public int getNumberOfThreads() {
return numberOfThreads;
}
public boolean isDeleteTable() {
return deleteTable;
}
public int getNumberOfRecordsPerRequest() {
return numberOfRecordsPerRequest;
}
}

42
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java Normal file
View File

@ -0,0 +1,42 @@
package com.taosdata.example.jdbcTaosdemo.task;
import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig;
import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory;
import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller;
import org.apache.log4j.Logger;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
public class CreateTableTask implements Runnable {
private static Logger logger = Logger.getLogger(CreateTableTask.class);
private final JdbcTaosdemoConfig config;
private final int startIndex;
private final int tableNumber;
public CreateTableTask(JdbcTaosdemoConfig config, int startIndex, int tableNumber) {
this.config = config;
this.startIndex = startIndex;
this.tableNumber = tableNumber;
}
@Override
public void run() {
try {
Connection connection = ConnectionFactory.build(config);
for (int i = startIndex; i < startIndex + tableNumber; i++) {
Statement statement = connection.createStatement();
String sql = SqlSpeller.createTableSQL(i + 1, config.getDbName(), config.getStbName());
statement.execute(sql);
statement.close();
logger.info(">>> " + sql);
}
connection.close();
} catch (SQLException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
}

49
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java Normal file
View File

@ -0,0 +1,49 @@
package com.taosdata.example.jdbcTaosdemo.task;
import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig;
import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory;
import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller;
import org.apache.log4j.Logger;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
public class InsertTableDatetimeTask implements Runnable {
private static Logger logger = Logger.getLogger(InsertTableDatetimeTask.class);
private final JdbcTaosdemoConfig config;
private final int startTableIndex;
private final int tableNumber;
private final long startDatetime;
private final long finishedDatetime;
public InsertTableDatetimeTask(JdbcTaosdemoConfig config, int startTableIndex, int tableNumber, long startDatetime, long finishedDatetime) {
this.config = config;
this.startTableIndex = startTableIndex;
this.tableNumber = tableNumber;
this.startDatetime = startDatetime;
this.finishedDatetime = finishedDatetime;
}
@Override
public void run() {
try {
Connection connection = ConnectionFactory.build(config);
int valuesCount = config.getNumberOfRecordsPerRequest();
for (long ts = startDatetime; ts < finishedDatetime; ts += valuesCount) {
for (int i = startTableIndex; i < startTableIndex + tableNumber; i++) {
String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, valuesCount);
Statement statement = connection.createStatement();
statement.execute(sql);
statement.close();
logger.info(Thread.currentThread().getName() + ">>> " + sql);
}
}
connection.close();
} catch (SQLException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
}

52
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java Normal file
View File

@ -0,0 +1,52 @@
package com.taosdata.example.jdbcTaosdemo.task;
import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig;
import com.taosdata.example.jdbcTaosdemo.utils.ConnectionFactory;
import com.taosdata.example.jdbcTaosdemo.utils.SqlSpeller;
import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil;
import org.apache.log4j.Logger;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.concurrent.atomic.AtomicLong;
public class InsertTableTask implements Runnable {
private static final Logger logger = Logger.getLogger(InsertTableTask.class);
private static AtomicLong beginTimestamp = new AtomicLong(TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000"));
private final JdbcTaosdemoConfig config;
private final int startIndex;
private final int tableNumber;
private final int recordsNumber;
public InsertTableTask(JdbcTaosdemoConfig config, int startIndex, int tableNumber, int recordsNumber) {
this.config = config;
this.startIndex = startIndex;
this.tableNumber = tableNumber;
this.recordsNumber = recordsNumber;
}
@Override
public void run() {
try {
Connection connection = ConnectionFactory.build(config);
// iterate insert
for (int j = 0; j < recordsNumber; j++) {
long ts = beginTimestamp.getAndIncrement();
// insert data into echo table
for (int i = startIndex; i < startIndex + tableNumber; i++) {
String sql = SqlSpeller.insertOneRowSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts);
Statement statement = connection.createStatement();
statement.execute(sql);
statement.close();
logger.info(Thread.currentThread().getName() + ">>> " + sql);
}
}
connection.close();
} catch (SQLException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}
}

32
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java Normal file
View File

@ -0,0 +1,32 @@
package com.taosdata.example.jdbcTaosdemo.utils;
import com.taosdata.example.jdbcTaosdemo.domain.JdbcTaosdemoConfig;
import com.taosdata.jdbc.TSDBDriver;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
public class ConnectionFactory {
public static Connection build(JdbcTaosdemoConfig config) throws SQLException {
return build(config.getHost(), config.getPort(), config.getDbName(), config.getUser(), config.getPassword());
}
public static Connection build(String host, int port, String dbName) throws SQLException {
return build(host, port, dbName, "root", "taosdata");
}
private static Connection build(String host, int port, String dbName, String user, String password) throws SQLException {
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, user);
properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, password);
properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
return DriverManager.getConnection("jdbc:TAOS://" + host + ":" + port + "/" + dbName + "", properties);
}
}

82
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/SqlSpeller.java Normal file
View File

@ -0,0 +1,82 @@
package com.taosdata.example.jdbcTaosdemo.utils;
import java.util.Random;
public class SqlSpeller {
private static final Random random = new Random(System.currentTimeMillis());
private static final String[] locations = {
"Beijing", "Shanghai", "Guangzhou", "Shenzhen",
"HangZhou", "Tianjin", "Wuhan", "Changsha", "Nanjing", "Xian"
};
public static String createDatabaseSQL(String dbName, int keep, int days) {
return "create database if not exists " + dbName + " keep " + keep + " days " + days;
}
public static String dropDatabaseSQL(String dbName) {
return "drop database if exists " + dbName;
}
public static String useDatabaseSQL(String dbName) {
return "use " + dbName;
}
public static String createSuperTableSQL(String superTableName) {
return "create table if not exists " + superTableName + "(ts timestamp, current float, voltage int, phase float) tags(location binary(64), groupId int)";
}
public static String dropSuperTableSQL(String dbName, String superTableName) {
return "drop table if exists " + dbName + "." + superTableName;
}
public static String createTableSQL(int tableIndex, String dbName, String superTableName) {
String location = locations[random.nextInt(locations.length)];
return "create table d" + tableIndex + " using " + dbName + "." + superTableName + " tags('" + location + "'," + tableIndex + ")";
}
public static String insertOneRowSQL(String dbName, String tbPrefix, int tableIndex, long ts) {
float current = 10 + random.nextFloat();
int voltage = 200 + random.nextInt(20);
float phase = random.nextFloat();
String sql = "insert into " + dbName + "." + tbPrefix + "" + tableIndex + " " + "values(" + ts + ", " + current + ", " + voltage + ", " + phase + ")";
return sql;
}
public static String insertBatchSizeRowsSQL(String dbName, String tbPrefix, int tbIndex, long ts, int valuesCount) {
float current = 10 + random.nextFloat();
int voltage = 200 + random.nextInt(20);
float phase = random.nextFloat();
StringBuilder sb = new StringBuilder();
sb.append("insert into " + dbName + "." + tbPrefix + "" + tbIndex + " " + "values");
for (int i = 0; i < valuesCount; i++) {
sb.append("(" + (ts + i) + ", " + current + ", " + voltage + ", " + phase + ") ");
}
return sb.toString();
}
public static String selectFromTableLimitSQL(String dbName, String tbPrefix, int tbIndex, int limit, int offset) {
return "select * from " + dbName + "." + tbPrefix + "" + tbIndex + " limit " + limit + " offset " + offset;
}
public static String selectCountFromTableSQL(String dbName, String tbPrefix, int tbIndex) {
return "select count(*) from " + dbName + "." + tbPrefix + "" + tbIndex;
}
public static String selectAvgMinMaxFromTableSQL(String field, String dbName, String tbPrefix, int tbIndex) {
return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + tbPrefix + "" + tbIndex;
}
public static String selectFromSuperTableLimitSQL(String dbName, String stbName, int limit, int offset) {
return "select * from " + dbName + "." + stbName + " limit " + limit + " offset " + offset;
}
public static String selectCountFromSuperTableSQL(String dbName, String stableName) {
return "select count(*) from " + dbName + "." + stableName;
}
public static String selectAvgMinMaxFromSuperTableSQL(String field, String dbName, String stbName) {
return "select avg(" + field + "),min(" + field + "),max(" + field + ") from " + dbName + "." + stbName + "";
}
}

35
tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/TimeStampUtil.java Normal file
View File

@ -0,0 +1,35 @@
package com.taosdata.example.jdbcTaosdemo.utils;
import java.sql.Date;
import java.text.ParseException;
import java.text.SimpleDateFormat;
public class TimeStampUtil {
private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS";
public static long datetimeToLong(String dateTime) {
SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat);
try {
return sdf.parse(dateTime).getTime();
} catch (ParseException e) {
throw new RuntimeException(e);
}
}
public static String longToDatetime(long time) {
SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat);
return sdf.format(new Date(time));
}
public static void main(String[] args) {
final String startTime = "2005-01-01 00:00:00.000";
long start = TimeStampUtil.datetimeToLong(startTime);
System.out.println(start);
String datetime = TimeStampUtil.longToDatetime(1519833600000L);
System.out.println(datetime);
}
}

21
tests/examples/JDBC/JDBCDemo/src/main/resources/log4j.properties Normal file
View File

@ -0,0 +1,21 @@
### 设置###
log4j.rootLogger=debug,stdout,DebugLog,ErrorLog
### 输出信息到控制抬 ###
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
### 输出DEBUG 级别以上的日志到=logs/error.log ###
log4j.appender.DebugLog=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DebugLog.File=logs/debug.log
log4j.appender.DebugLog.Append=true
log4j.appender.DebugLog.Threshold=DEBUG
log4j.appender.DebugLog.layout=org.apache.log4j.PatternLayout
log4j.appender.DebugLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
### 输出ERROR 级别以上的日志到=logs/error.log ###
log4j.appender.ErrorLog=org.apache.log4j.DailyRollingFileAppender
log4j.appender.ErrorLog.File=logs/error.log
log4j.appender.ErrorLog.Append=true
log4j.appender.ErrorLog.Threshold=ERROR
log4j.appender.ErrorLog.layout=org.apache.log4j.PatternLayout
log4j.appender.ErrorLog.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n

4
tests/examples/python/read_example.py
View File

@ -22,6 +22,10 @@ if __name__ == '__main__':
# @password : Password
# @database : Database to use when connecting to TDengine server
# @config : Configuration directory
if len(sys.argv)>1:
hostname=sys.argv[1]
conn = taos.connect(host=hostname, user="root", password="taosdata", config="/etc/taos")
else:
conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
# Generate a cursor object to run SQL commands

55
tests/pytest/client/alterDatabase.py Normal file
View File

@ -0,0 +1,55 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.prepare()
tdSql.query('select database()')
tdSql.checkData(0, 0, "db")
tdSql.execute("alter database db comp 2")
tdSql.query("show databases")
tdSql.checkData(0, 14, 2)
tdSql.execute("alter database db keep 365")
tdSql.query("show databases")
tdSql.checkData(0, 7, "3650,3650,365")
tdSql.execute("alter database db quorum 2")
tdSql.query("show databases")
tdSql.checkData(0, 5, 2)
tdSql.execute("alter database db blocks 100")
tdSql.query("show databases")
tdSql.checkData(0, 9, 100)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

2
tests/pytest/fulltest.sh
View File

@ -17,6 +17,7 @@ python3 ./test.py -f insert/nchar-unicode.py
python3 ./test.py -f insert/multi.py
python3 ./test.py -f insert/randomNullCommit.py
python3 insert/retentionpolicy.py
python3 ./test.py -f insert/alterTableAndInsert.py
python3 ./test.py -f table/column_name.py
python3 ./test.py -f table/column_num.py
@ -163,6 +164,7 @@ python3 ./test.py -f alter/alter_table_crash.py
# client
python3 ./test.py -f client/client.py
python3 ./test.py -f client/version.py
python3 ./test.py -f client/alterDatabase.py
# Misc
python3 testCompress.py

40
tests/pytest/insert/alterTableAndInsert.py Normal file
View File

@ -0,0 +1,40 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.prepare()
tdSql.execute("create table cars(ts timestamp, speed int) tags(id int)")
tdSql.execute("create table car0 using cars tags(0)")
tdSql.execute("insert into car0 values(now, 1)")
tdSql.execute("alter table cars add column c2 int")
tdSql.execute("insert into car0(ts, 'speed') values(now, 2)")
tdSql.checkAffectedRows(1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

60
tests/pytest/insert/restful.py Normal file
View File

@ -0,0 +1,60 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import requests, json
import threading
import string
import random
class RestfulInsert:
def init(self):
self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
self.url = "http://127.0.0.1:6041/rest/sql"
self.ts = 1104508800000
self.numOfThreads = 50
def get_random_string(self, length):
letters = string.ascii_lowercase
result_str = ''.join(random.choice(letters) for i in range(length))
return result_str
def insertData(self, threadID):
print("thread %d started" % threadID)
data = "create table test.tb%d(ts timestamp, name nchar(20))" % threadID
requests.post(self.url, data, headers = self.header)
name = self.get_random_string(10)
start = self.ts
while True:
start += 1
data = "insert into test.tb%d values(%d, '%s')" % (threadID, start, name)
requests.post(self.url, data, headers = self.header)
def run(self):
data = "drop database if exists test"
requests.post(self.url, data, headers = self.header)
data = "create database test keep 7300"
requests.post(self.url, data, headers = self.header)
threads = []
for i in range(self.numOfThreads):
thread = threading.Thread(target=self.insertData, args=(i,))
thread.start()
threads.append(thread)
for i in range(self.numOfThreads):
threads[i].join()
ri = RestfulInsert()
ri.init()
ri.run()

48
tests/pytest/insert/retentionpolicy.py
View File

@ -43,7 +43,8 @@ class TDTestRetetion:
else:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows)
os.system("timedatectl set-ntp true")
os.system("sudo timedatectl set-ntp true")
time.sleep(40)
tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args)
def run(self):
@ -53,7 +54,7 @@ class TDTestRetetion:
tdSql.execute('use test;')
tdSql.execute('create table test(ts timestamp,i int);')
cmd = 'insert into test values(now-2d,11)(now-1d,11)(now,11)(now+1d,11);'
cmd = 'insert into test values(now-2d,1)(now-1d,2)(now,3)(now+1d,4);'
tdLog.info(cmd)
tdSql.execute(cmd)
tdSql.query('select * from test')
@ -61,46 +62,55 @@ class TDTestRetetion:
tdLog.info("=============== step2")
tdDnodes.stop(1)
os.system("timedatectl set-ntp false")
os.system("date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
os.system("sudo timedatectl set-ntp false")
os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
tdDnodes.start(1)
cmd = 'insert into test values(now,11);'
cmd = 'insert into test values(now,5);'
tdDnodes.stop(1)
tdDnodes.start(1)
tdLog.info(cmd)
tdSql.execute(cmd)
queryRows=tdSql.query('select * from test')
if queryRows==4:
tdSql.checkRows(4)
self.queryRows=tdSql.query('select * from test')
if self.queryRows==4:
self.checkRows(4,cmd)
return 0
else:
tdSql.checkRows(5)
self.checkRows(5,cmd)
tdLog.info("=============== step3")
tdDnodes.stop(1)
os.system("date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")")
tdDnodes.start(1)
cmd = 'insert into test values(now-1d,11);'
tdLog.info(cmd)
tdSql.execute(cmd)
queryRows=tdSql.query('select * from test')
tdSql.checkRows(6)
self.queryRows=tdSql.query('select * from test')
if self.queryRows==4:
self.checkRows(4,cmd)
return 0
cmd = 'insert into test values(now-1d,6);'
tdLog.info(cmd)
tdSql.execute(cmd)
self.queryRows=tdSql.query('select * from test')
self.checkRows(6,cmd)
tdLog.info("=============== step4")
tdDnodes.stop(1)
tdDnodes.start(1)
cmd = 'insert into test values(now,11);'
cmd = 'insert into test values(now,7);'
tdLog.info(cmd)
tdSql.execute(cmd)
tdSql.query('select * from test')
tdSql.checkRows(7)
self.queryRows=tdSql.query('select * from test')
self.checkRows(7,cmd)
tdLog.info("=============== step5")
tdDnodes.stop(1)
tdDnodes.start(1)
cmd='select * from test where ts > now-1d'
queryRows=tdSql.query('select * from test where ts > now-1d')
self.queryRows=tdSql.query('select * from test where ts > now-1d')
self.checkRows(1,cmd)
def stop(self):
os.system("timedatectl set-ntp true")
os.system("sudo timedatectl set-ntp true")
time.sleep(40)
tdSql.close()
tdLog.success("%s successfully executed" % __file__)

3
tests/pytest/query/queryNormal.py
View File

@ -35,7 +35,8 @@ class TDTestCase:
tdSql.execute(
"insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:02.000', 3, 2.1), ('2020-04-18 15:00:03.000', 4, 2.2)")
# inner join --- bug
tdSql.error("select * from tb 1")
tdSql.query("select * from tb1 a, tb2 b where a.ts = b.ts")
tdSql.checkRows(0)

5
tests/script/jenkins/basic.txt
View File

@ -278,6 +278,7 @@ cd ../../../debug; make
./test.sh -f unique/dnode/balancex.sim
./test.sh -f unique/dnode/offline1.sim
./test.sh -f unique/dnode/offline2.sim
./test.sh -f unique/dnode/reason.sim
./test.sh -f unique/dnode/remove1.sim
./test.sh -f unique/dnode/remove2.sim
./test.sh -f unique/dnode/vnode_clean.sim
@ -302,8 +303,8 @@ cd ../../../debug; make
./test.sh -f unique/mnode/mgmt22.sim
./test.sh -f unique/mnode/mgmt23.sim
./test.sh -f unique/mnode/mgmt24.sim
./test.sh -f unique/mnode/mgmt25.sim
./test.sh -f unique/mnode/mgmt26.sim
#./test.sh -f unique/mnode/mgmt25.sim
#./test.sh -f unique/mnode/mgmt26.sim
./test.sh -f unique/mnode/mgmt33.sim
./test.sh -f unique/mnode/mgmt34.sim
./test.sh -f unique/mnode/mgmtr2.sim

132
tests/script/unique/dnode/reason.sim Normal file
View File

@ -0,0 +1,132 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
print ========== step1
system sh/exec.sh -n dnode1 -s start
sleep 3000
sql connect
sql create dnode $hostname2
sql show dnodes
print dnode1 off: $data7_1
print dnode2 off: $data7_2
if $data7_2 != @status not received@ then
return -1
endi
print ========== step2
system sh/exec.sh -n dnode2 -s start
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode2 off: $data7_2
print ========== step3
system sh/exec.sh -n dnode2 -s stop
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode2 off: $data7_2
if $data7_2 != @status msg timeout@ then
return -1
endi
print ========== step4
sql drop dnode $hostname2
sleep 5000
sql show dnodes
if $rows != 1 then
return -1
endi
print ========== step5
system sh/exec.sh -n dnode2 -s start
sql create dnode $hostname2
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode2 off: $data7_3
if $data7_3 != @dnodeId not match@ then
return -1
endi
print ========== step6
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3
system sh/exec.sh -n dnode4 -s start
sql create dnode $hostname4
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode4 off: $data7_4
if $data7_4 != @mnEqualVn not match@ then
return -1
endi
print ========== step7
system sh/deploy.sh -n dnode5 -i 5
system sh/cfg.sh -n dnode5 -c statusInterval -v 3
system sh/exec.sh -n dnode5 -s start
sql create dnode $hostname5
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode5 off: $data7_5
if $data7_5 != @interval not match@ then
return -1
endi
print ========== step8
system sh/deploy.sh -n dnode6 -i 6
system sh/cfg.sh -n dnode6 -c balance -v 0
system sh/exec.sh -n dnode6 -s start
sql create dnode $hostname6
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode6 off: $data7_6
if $data7_6 != @balance not match@ then
return -1
endi
print ========== step9
system sh/deploy.sh -n dnode7 -i 7
system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000
system sh/exec.sh -n dnode7 -s start
sql create dnode $hostname7
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode7 off: $data7_7
if $data7_7 != @maxTabPerVn not match@ then
return -1
endi
print ========== step10
system sh/deploy.sh -n dnode8 -i 8
system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3
system sh/exec.sh -n dnode8 -s start
sql create dnode $hostname8
sleep 3000
sql show dnodes
print dnode1 off: $data7_1
print dnode8 off: $data7_8
if $data7_8 != @maxVgPerDb not match@ then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode5 -s stop -x SIGINT
system sh/exec.sh -n dnode6 -s stop -x SIGINT
system sh/exec.sh -n dnode7 -s stop -x SIGINT
system sh/exec.sh -n dnode8 -s stop -x SIGINT

11
tests/test/c/CMakeLists.txt
View File

@ -37,10 +37,13 @@ IF (TD_LINUX)
#add_executable(queryPerformance queryPerformance.c)
#target_link_libraries(queryPerformance taos_static tutil common pthread)
add_executable(httpTest httpTest.c)
target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4)
#add_executable(httpTest httpTest.c)
#target_link_libraries(httpTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4)
add_executable(cacheTest cacheTest.c)
target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4)
#add_executable(cacheTest cacheTest.c)
#target_link_libraries(cacheTest taos_static tutil common pthread mnode monitor http tsdb twal vnode cJson lz4)
#add_executable(invalidTableId invalidTableId.c)
#target_link_libraries(invalidTableId taos_static tutil common pthread)
ENDIF()

19
tests/test/c/createNormalTable.c
View File

@ -50,7 +50,9 @@ void createDbAndSTable();
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
taos_init();
if (replica != 0) {
createDbAndSTable();
}
pPrint("%d threads are spawned to create table", numOfThreads);
@ -134,6 +136,7 @@ void *threadFunc(void *param) {
int64_t startMs = taosGetTimestampMs();
if (replica != 0) {
for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t);
TAOS_RES *pSql = taos_query(con, qstr);
@ -143,6 +146,22 @@ void *threadFunc(void *param) {
}
taos_free_result(pSql);
}
} else {
for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
sprintf(qstr, "insert into %s%d values(now, 1)", stableName, t);
TAOS_RES *pSql = taos_query(con, qstr);
code = taos_errno(pSql);
if (code != 0) {
if (code != TSDB_CODE_MND_INVALID_TABLE_NAME) {
pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code));
}
if (code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
exit(0);
}
}
taos_free_result(pSql);
}
}
float createTableSpeed = 0;
for (int i = 0; i < numOfThreads; ++i) {

73
tests/test/c/importPerTable.c
View File

@ -20,6 +20,7 @@
#include "ttimer.h"
#include "tutil.h"
#include "tglobal.h"
#include "osTime.h"
#define MAX_RANDOM_POINTS 20000
#define GREEN "\033[1;32m"
@ -43,14 +44,16 @@ void createDbAndTable();
void insertData();
int32_t randomData[MAX_RANDOM_POINTS];
int64_t rowsPerTable = 10000;
int64_t rowsPerTable = 1000000;
int64_t pointsPerTable = 1;
int64_t numOfThreads = 1;
int64_t numOfTablesPerThread = 1;
int64_t numOfThreads = 10;
int64_t numOfTablesPerThread = 100;
char dbName[32] = "db";
char stableName[64] = "st";
int32_t cache = 16384;
int32_t tables = 1000;
int64_t totalUs = 0;
int64_t reqNum = 0;
int64_t maxUs = 0;
int64_t minUs = 100000000;
int main(int argc, char *argv[]) {
shellParseArgument(argc, argv);
@ -58,6 +61,38 @@ int main(int argc, char *argv[]) {
taos_init();
createDbAndTable();
insertData();
int64_t avgUs = totalUs / reqNum;
pError("%s totalUs:%ld, avgUs:%ld maxUs:%ld minUs:%ld reqNum:%ld %s\n", GREEN, totalUs, avgUs, maxUs, minUs, reqNum, NC);
}
int32_t query(void *con, char *qstr) {
int64_t begin = taosGetTimestampUs();
TAOS_RES *pSql = taos_query(con, qstr);
int32_t code = taos_errno(pSql);
if (code != 0) {
pError("failed to exec sql:%s, code:%d reason:%s", qstr, taos_errno(con), taos_errstr(con));
exit(0);
}
taos_free_result(pSql);
int64_t us = taosGetTimestampUs() - begin;
maxUs = MAX(us, maxUs);
minUs = MIN(us, minUs);
atomic_add_fetch_64(&totalUs, us);
atomic_add_fetch_64(&reqNum, 1);
if (reqNum > 100000) {
int64_t avgUs = totalUs / reqNum;
if (us > avgUs * 100) {
pError("sql:%s", qstr);
pError("%s totalUs:%ld, avgUs:%ld maxUs:%ld minUs:%ld reqNum:%ld %s\n", GREEN, totalUs, avgUs, maxUs, minUs,
reqNum, NC);
taosMsleep(1000);
exit(0);
}
}
return code;
}
void createDbAndTable() {
@ -79,14 +114,14 @@ void createDbAndTable() {
exit(1);
}
sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
if (taos_query(con, qstr)) {
sprintf(qstr, "create database if not exists %s", dbName);
if (query(con, qstr)) {
pError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
exit(0);
}
sprintf(qstr, "use %s", dbName);
if (taos_query(con, qstr)) {
if (query(con, qstr)) {
pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
@ -102,14 +137,14 @@ void createDbAndTable() {
}
sprintf(qstr + len, ") tags(t int)");
if (taos_query(con, qstr)) {
if (query(con, qstr)) {
pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
exit(0);
}
for (int64_t t = 0; t < totalTables; ++t) {
sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
if (taos_query(con, qstr)) {
if (query(con, qstr)) {
pError("failed to create table %s%" PRId64 ", reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
@ -122,7 +157,7 @@ void createDbAndTable() {
}
sprintf(qstr + len, ")");
if (taos_query(con, qstr)) {
if (query(con, qstr)) {
pError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
exit(0);
}
@ -207,7 +242,7 @@ void *syncTest(void *param) {
}
sprintf(qstr, "use %s", pInfo->dbName);
taos_query(con, qstr);
query(con, qstr);
gettimeofday(&systemTime, NULL);
st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
@ -229,7 +264,7 @@ void *syncTest(void *param) {
}
len += sprintf(sql + len, ")");
if (len > maxBytes) {
if (taos_query(con, qstr)) {
if (query(con, qstr)) {
pError("thread:%d, failed to import table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
table, row, taos_errstr(con));
}
@ -246,7 +281,7 @@ void *syncTest(void *param) {
}
if (len != strlen(inserStr)) {
taos_query(con, qstr);
query(con, qstr);
}
gettimeofday(&systemTime, NULL);
@ -284,10 +319,6 @@ void printHelp() {
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads);
printf("%s%s\n", indent, "-n");
printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread);
printf("%s%s\n", indent, "-tables");
printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables);
printf("%s%s\n", indent, "-cache");
printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache);
exit(EXIT_SUCCESS);
}
@ -311,10 +342,6 @@ void shellParseArgument(int argc, char *argv[]) {
numOfThreads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-n") == 0) {
numOfTablesPerThread = atoi(argv[++i]);
} else if (strcmp(argv[i], "-tables") == 0) {
tables = atoi(argv[++i]);
} else if (strcmp(argv[i], "-cache") == 0) {
cache = atoi(argv[++i]);
} else {
}
}
@ -323,8 +350,6 @@ void shellParseArgument(int argc, char *argv[]) {
pPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
pPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
pPrint("%scache:%d%s", GREEN, cache, NC);
pPrint("%stables:%d%s", GREEN, tables, NC);
pPrint("%sdbName:%s%s", GREEN, dbName, NC);
pPrint("%stableName:%s%s", GREEN, stableName, NC);
pPrint("%sstart to run%s", GREEN, NC);

220
tests/test/c/invalidTableId.c Normal file
View File

@ -0,0 +1,220 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "taos.h"
#include "tulog.h"
#include "tutil.h"
#include "tglobal.h"
#include "hash.h"
#define MAX_RANDOM_POINTS 20000
#define GREEN "\033[1;32m"
#define NC "\033[0m"
#define MAX_DB_NUM 100
void * con;
char dbNames[MAX_DB_NUM][48];
int32_t dbNum = 0;
void parseArgument(int argc, char *argv[]);
void connDb();
void getDbNames();
void printDbNames();
void queryTables(char *dbName);
void checkTables(char *dbName);
int main(int argc, char *argv[]) {
parseArgument(argc, argv);
taos_init();
connDb();
getDbNames();
printDbNames();
for (int dbIndex = 0; dbIndex < dbNum; ++dbIndex) {
queryTables((char*)(dbNames[dbIndex]));
checkTables((char*)(dbNames[dbIndex]));
}
pPrint("all %d database is checked", dbNum);
}
void connDb() {
con = taos_connect(NULL, "root", "taosdata", NULL, 0);
if (con == NULL) {
pError("failed to connect to DB, reason:%s", taos_errstr(con));
exit(0);
}
}
void getDbNames() {
if (dbNum != 0) return;
char * qstr = "show databases";
TAOS_RES *result = taos_query(con, qstr);
int32_t code = taos_errno(result);
if (result == NULL || code != 0) {
pError("failed to exec sql:%s, code:0x%x reason:%s", qstr, code & 0XFFFF, tstrerror(code));
exit(0);
}
TAOS_ROW row;
int num_fields = taos_num_fields(result);
if (num_fields <= 0) return;
while ((row = taos_fetch_row(result))) {
char * dbName = (char*)dbNames[dbNum];
int32_t *length = taos_fetch_lengths(result);
int len = length[0];
memcpy(dbName, (char *)row[0], len);
dbName[len] = 0;
dbNum++;
}
taos_free_result(result);
}
void printDbNames() {
for (int dbIndex = 0; dbIndex < dbNum; ++dbIndex) {
pPrint("db:%d %s", dbIndex, dbNames[dbIndex]);
}
}
void queryTables(char *dbName) {
char qstr[1024];
char fileName[1024];
char ts[35] = {0};
int32_t precision = 1000;
sprintf(qstr, "show %s.tables", dbName);
sprintf(fileName, "%s_tables.txt", dbName);
TAOS_RES *result = taos_query(con, qstr);
int32_t code = taos_errno(result);
if (result == NULL || code != 0) {
pError("failed to exec sql:%s, code:0x%x reason:%s", qstr, code & 0XFFFF, tstrerror(code));
exit(0);
}
FILE *fp = fopen(fileName, "w");
if (!fp) return;
TAOS_ROW row;
int32_t rows = 0;
while ((row = taos_fetch_row(result))) {
char tableName[256] = {0};
int32_t *length = taos_fetch_lengths(result);
int len = length[0];
memcpy(tableName, (char *)row[0], len);
tableName[len] = 0;
int64_t t = *((int64_t *)row[1]);
time_t tt = t / 1000;
struct tm *ptm = localtime(&tt);
int32_t tl = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm);
snprintf(ts + tl, 5, ".%03ld", t % precision);
// fprintf(fp, "%s %s\n", tableName, ts);
fprintf(fp, "%s.%s\n", dbName, tableName);
rows++;
}
taos_free_result(result);
fclose(fp);
pPrint("db:%s has %d tables, write to %s", dbName, rows, fileName);
}
void checkTables(char *dbName) {
char qstr[1024];
char fileName1[1024];
char fileName2[1024];
sprintf(qstr, "show %s.tables", dbName);
sprintf(fileName1, "%s_tables.txt", dbName);
sprintf(fileName2, "%s_count.txt", dbName);
FILE *fp1 = fopen(fileName1, "r");
if (!fp1) return;
FILE *fp2 = fopen(fileName2, "w");
if (!fp2) return;
int32_t successRows = 0;
int32_t failedRows = 0;
char tbName[256];
while (!feof(fp1)) {
int size = fscanf(fp1, "%s", tbName);
if (size <= 0) {
break;
}
sprintf(qstr, "select count(*) from %s", tbName);
TAOS_RES *result = taos_query(con, qstr);
int32_t code = taos_errno(result);
if (result == NULL || code != 0) {
pError("failed to exec sql:%s, code:0x%x reason:%s", qstr, code & 0XFFFF, tstrerror(code));
fprintf(fp2, "%s failed to exec sql:%s, code:0x%x reason:%s", tbName, qstr, code & 0XFFFF, tstrerror(code));
taos_free_result(result);
failedRows++;
continue;
}
TAOS_ROW row;
int64_t count = 0;
while ((row = taos_fetch_row(result))) {
count = *((int64_t *)row[0]);
}
fprintf(fp2, "%s %" PRId64 "\n", tbName, count);
successRows++;
if (successRows % 1000 == 0) {
pPrint("query %d tables", successRows);
}
taos_free_result(result);
}
fclose(fp1);
fclose(fp2);
pPrint("db:%s query tables, success:%d failed:%d write to %s", dbName, successRows, failedRows, fileName2);
}
void printHelp() {
char indent[10] = " ";
printf("Used to checkTables\n");
printf("%s%s\n", indent, "-c");
printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
printf("%s%s\n", indent, "-d");
printf("%s%s%s%s\n", indent, indent, "The name of the database to be checked, default is ", "all");
exit(EXIT_SUCCESS);
}
void parseArgument(int argc, char *argv[]) {
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
printHelp();
exit(0);
} else if (strcmp(argv[i], "-d") == 0) {
strcpy(dbNames[0], argv[++i]);
dbNum++;
} else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]);
} else {
}
}
pPrint("%s configDir:%s %s", GREEN, configDir, NC);
pPrint("%s start to checkTables %s", GREEN, NC);
}