xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/table

This commit is contained in:
Shengliang Guan 2020-11-13 08:41:18 +00:00
parent 6b066bcf1a 7bbda1b95b
commit f7c9b2bb40
140 changed files with 2999 additions and 1197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmake/install.inc
View File

@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)

3
documentation20/webdocs/markdowndocs/Documentation-ch.md
View File

@ -34,7 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专
- [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入)：使用SQL insert命令向一张或多张表写入单条或多条记录
- [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入)：配置Telegraf, 不用任何代码，将采集数据直接写入
- [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入)：配置Prometheus, 不用任何代码，将数据直接写入
- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将MQTT数据直接写入
- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将 MQTT 数据直接写入
- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入)：通过 HiveMQ Extension，不用任何代码，就可将 MQTT 数据直接写入
## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries)

2
documentation20/webdocs/markdowndocs/Evaluation-ch.md
View File

@ -10,7 +10,7 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的
* __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10。
* __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。
* __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。
* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
* __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类似标准SQL，支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似，零学习成本。
采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。

2
documentation20/webdocs/markdowndocs/TAOS SQL-ch.md
View File

@ -90,7 +90,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic
```mysql
ALTER DATABASE db_name REPLICA 2;
```
REPLICA参数是指修改数据库副本数，取值范围[1, 3]。在集群中使用，副本数必须小于dnode的数目。
REPLICA参数是指修改数据库副本数，取值范围[1, 3]。在集群中使用，副本数必须小于或等于dnode的数目。
```mysql
ALTER DATABASE db_name KEEP 365;

2
documentation20/webdocs/markdowndocs/administrator-ch.md
View File

@ -253,7 +253,7 @@ ALTER USER <user_name> PASS <'password'>;
修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角
```
ALTER USER <user_name> PRIVILEDGE <super|write|read>;
ALTER USER <user_name> PRIVILEGE <super|write|read>;
```
修改用户权限为：super/write/read，不需要添加单引号

105
documentation20/webdocs/markdowndocs/architecture-ch.md
View File

@ -4,16 +4,99 @@
### 物联网典型场景
在典型的物联网、车联网、运维监测场景中，往往有多种不同类型的数据采集设备，采集一个到多个不同的物理量。而同一种采集设备类型，往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总，然后进行计算和分析。对于同一类设备，其采集的数据都是很规则的。以智能电表为例，假设每个智能电表采集电流、电压、相位三个量，其采集的数据类似如下的表格：
| Device ID | Time Stamp | current | voltage | phase | location | groupId |
| :-------: | :-----------: | :-----: | :-----: | :---: | :--------------: | :-----: |
| d1001 | 1538548685000 | 10.3 | 219 | 0.31 | Beijing.Chaoyang | 2 |
| d1002 | 1538548684000 | 10.2 | 220 | 0.23 | Beijing.Chaoyang | 3 |
| d1003 | 1538548686500 | 11.5 | 221 | 0.35 | Beijing.Haidian | 3 |
| d1004 | 1538548685500 | 13.4 | 223 | 0.29 | Beijing.Haidian | 2 |
| d1001 | 1538548695000 | 12.6 | 218 | 0.33 | Beijing.Chaoyang | 2 |
| d1004 | 1538548696600 | 11.8 | 221 | 0.28 | Beijing.Haidian | 2 |
| d1002 | 1538548696650 | 10.3 | 218 | 0.25 | Beijing.Chaoyang | 3 |
| d1001 | 1538548696800 | 12.3 | 221 | 0.31 | Beijing.Chaoyang | 2 |
<figure><table>
<thead><tr>
<th style="text-align:center;">设备ID</th>
<th style="text-align:center;">时间戳</th>
<th style="text-align:center;" colspan="3">采集量</th>
<th style="text-align:center;" colspan="2">标签</th>
</tr>
<tr>
<th style="text-align:center;">Device ID</th>
<th style="text-align:center;">Time Stamp</th>
<th style="text-align:center;">current</th>
<th style="text-align:center;">voltage</th>
<th style="text-align:center;">phase</th>
<th style="text-align:center;">location</th>
<th style="text-align:center;">groupId</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:center;">d1001</td>
<td style="text-align:center;">1538548685000</td>
<td style="text-align:center;">10.3</td>
<td style="text-align:center;">219</td>
<td style="text-align:center;">0.31</td>
<td style="text-align:center;">Beijing.Chaoyang</td>
<td style="text-align:center;">2</td>
</tr>
<tr>
<td style="text-align:center;">d1002</td>
<td style="text-align:center;">1538548684000</td>
<td style="text-align:center;">10.2</td>
<td style="text-align:center;">220</td>
<td style="text-align:center;">0.23</td>
<td style="text-align:center;">Beijing.Chaoyang</td>
<td style="text-align:center;">3</td>
</tr>
<tr>
<td style="text-align:center;">d1003</td>
<td style="text-align:center;">1538548686500</td>
<td style="text-align:center;">11.5</td>
<td style="text-align:center;">221</td>
<td style="text-align:center;">0.35</td>
<td style="text-align:center;">Beijing.Haidian</td>
<td style="text-align:center;">3</td>
</tr>
<tr>
<td style="text-align:center;">d1004</td>
<td style="text-align:center;">1538548685500</td>
<td style="text-align:center;">13.4</td>
<td style="text-align:center;">223</td>
<td style="text-align:center;">0.29</td>
<td style="text-align:center;">Beijing.Haidian</td>
<td style="text-align:center;">2</td>
</tr>
<tr>
<td style="text-align:center;">d1001</td>
<td style="text-align:center;">1538548695000</td>
<td style="text-align:center;">12.6</td>
<td style="text-align:center;">218</td>
<td style="text-align:center;">0.33</td>
<td style="text-align:center;">Beijing.Chaoyang</td>
<td style="text-align:center;">2</td>
</tr>
<tr>
<td style="text-align:center;">d1004</td>
<td style="text-align:center;">1538548696600</td>
<td style="text-align:center;">11.8</td>
<td style="text-align:center;">221</td>
<td style="text-align:center;">0.28</td>
<td style="text-align:center;">Beijing.Haidian</td>
<td style="text-align:center;">2</td>
</tr>
<tr>
<td style="text-align:center;">d1002</td>
<td style="text-align:center;">1538548696650</td>
<td style="text-align:center;">10.3</td>
<td style="text-align:center;">218</td>
<td style="text-align:center;">0.25</td>
<td style="text-align:center;">Beijing.Chaoyang</td>
<td style="text-align:center;">3</td>
</tr>
<tr>
<td style="text-align:center;">d1001</td>
<td style="text-align:center;">1538548696800</td>
<td style="text-align:center;">12.3</td>
<td style="text-align:center;">221</td>
<td style="text-align:center;">0.31</td>
<td style="text-align:center;">Beijing.Chaoyang</td>
<td style="text-align:center;">2</td>
</tr>
</tbody>
</table></figure>
<center> 表1：智能电表数据示例</center>
@ -221,7 +304,7 @@ TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），
TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，**可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署Redis或其他额外的缓存系统**，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。
每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当一半以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有一半内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。
每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当三分之一以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有三分之一内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。
### 持久化存储
TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时，为了不阻塞后续数据的写入，TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件，在落盘成功后则会删除老的数据库日志文件，避免日志文件无限制的增长。

14
documentation20/webdocs/markdowndocs/insert-ch.md
View File

@ -1,6 +1,6 @@
# 高效写入数据
TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。
TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。
## SQL写入
@ -218,7 +218,15 @@ use telegraf;
select * from cpu;
```
## EMQ X Broker直接写入
MQTT是一流行的物联网数据传输协议，[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDEngine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。
## EMQ Broker 直接写入
[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。
## HiveMQ Broker 直接写入
[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。

4
src/balance/src/balance.c
View File

@ -571,8 +571,8 @@ static void balanceCheckDnodeAccess() {
if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
pDnode->status = TAOS_DN_STATUS_OFFLINE;
pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
mInfo("dnode:%d, set to offline state, access seq:%d, last seq:%d", pDnode->dnodeId, tsAccessSquence,
pDnode->lastAccess);
mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
pDnode->lastAccess, pDnode->status);
balanceSetVgroupOffline(pDnode);
}
}

2
src/client/src/tscFunctionImpl.c
View File

@ -2117,7 +2117,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
}
}
taosTFree(pData);
tfree(pData);
}
/*

62
src/client/src/tscLocalMerge.c
View File

@ -227,7 +227,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds == NULL) {
tscError("%p failed to create merge structure", pSql);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
taosTFree(pReducer);
tfree(pReducer);
return;
}
@ -254,7 +254,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (ds->filePage.num == 0) { // no data in this flush, the index does not increase
tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx);
taosTFree(ds);
tfree(ds);
continue;
}
@ -264,7 +264,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
// no data actually, no need to merge result.
if (idx == 0) {
taosTFree(pReducer);
tfree(pReducer);
return;
}
@ -272,7 +272,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
SCompareParam *param = malloc(sizeof(SCompareParam));
if (param == NULL) {
taosTFree(pReducer);
tfree(pReducer);
return;
}
@ -286,8 +286,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
if (pReducer->pLoserTree == NULL || pRes->code != 0) {
taosTFree(param);
taosTFree(pReducer);
tfree(param);
tfree(pReducer);
return;
}
@ -330,14 +330,14 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
/*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
taosTFree(pReducer->pTempBuffer);
taosTFree(pReducer->discardData);
taosTFree(pReducer->pResultBuf);
taosTFree(pReducer->pFinalRes);
taosTFree(pReducer->prevRowOfInput);
taosTFree(pReducer->pLoserTree);
taosTFree(param);
taosTFree(pReducer);
tfree(pReducer->pTempBuffer);
tfree(pReducer->discardData);
tfree(pReducer->pResultBuf);
tfree(pReducer->pFinalRes);
tfree(pReducer->prevRowOfInput);
tfree(pReducer->pLoserTree);
tfree(param);
tfree(pReducer);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return;
}
@ -495,33 +495,33 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];
tVariantDestroy(&pCtx->tag);
taosTFree(pCtx->resultInfo);
tfree(pCtx->resultInfo);
if (pCtx->tagInfo.pTagCtxList != NULL) {
taosTFree(pCtx->tagInfo.pTagCtxList);
tfree(pCtx->tagInfo.pTagCtxList);
}
}
taosTFree(pLocalReducer->pCtx);
tfree(pLocalReducer->pCtx);
}
taosTFree(pLocalReducer->prevRowOfInput);
tfree(pLocalReducer->prevRowOfInput);
taosTFree(pLocalReducer->pTempBuffer);
taosTFree(pLocalReducer->pResultBuf);
tfree(pLocalReducer->pTempBuffer);
tfree(pLocalReducer->pResultBuf);
if (pLocalReducer->pLoserTree) {
taosTFree(pLocalReducer->pLoserTree->param);
taosTFree(pLocalReducer->pLoserTree);
tfree(pLocalReducer->pLoserTree->param);
tfree(pLocalReducer->pLoserTree);
}
taosTFree(pLocalReducer->pFinalRes);
taosTFree(pLocalReducer->discardData);
tfree(pLocalReducer->pFinalRes);
tfree(pLocalReducer->discardData);
tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel,
pLocalReducer->numOfVnode);
for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) {
taosTFree(pLocalReducer->pLocalDataSrc[i]);
tfree(pLocalReducer->pLocalDataSrc[i]);
}
pLocalReducer->numOfBuffer = 0;
@ -588,7 +588,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
}
*pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order);
taosTFree(orderColIndexList);
tfree(orderColIndexList);
if (*pOrderDesc == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -699,7 +699,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
taosTFree(pSchema);
tfree(pSchema);
return pRes->code;
}
@ -736,7 +736,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
}
*pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity);
taosTFree(pSchema);
tfree(pSchema);
return TSDB_CODE_SUCCESS;
}
@ -756,7 +756,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
}
taosTFree(pMemBuffer);
tfree(pMemBuffer);
}
/**
@ -978,10 +978,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
pBeforeFillData->num = 0;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
taosTFree(pResPages[i]);
tfree(pResPages[i]);
}
taosTFree(pResPages);
tfree(pResPages);
}
static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {

10
src/client/src/tscParseInsert.c
View File

@ -1406,7 +1406,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
assert(taos_errno(pSql) == code);
taos_free_result(pSql);
taosTFree(pSupporter);
tfree(pSupporter);
fclose(fp);
pParentSql->res.code = code;
@ -1445,7 +1445,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
char *tokenBuf = calloc(1, 4096);
while ((readLen = taosGetline(&line, &n, fp)) != -1) {
while ((readLen = tgetline(&line, &n, fp)) != -1) {
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
line[--readLen] = 0;
}
@ -1470,7 +1470,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
}
}
taosTFree(tokenBuf);
tfree(tokenBuf);
free(line);
if (count > 0) {
@ -1483,7 +1483,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
} else {
taos_free_result(pSql);
taosTFree(pSupporter);
tfree(pSupporter);
fclose(fp);
pParentSql->fp = pParentSql->fetchFp;
@ -1513,7 +1513,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) {
pSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));
taosTFree(pSupporter)
tfree(pSupporter)
tscQueueAsyncRes(pSql);
return;

4
src/client/src/tscSQLParser.c
View File

@ -3957,7 +3957,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
if (ret != TSDB_CODE_SUCCESS) {
taosStringBuilderDestroy(&sb1);
taosTFree(segments);
tfree(segments);
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
return ret;
@ -3970,7 +3970,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
taosStringBuilderDestroy(&sb1);
taosTFree(segments);
tfree(segments);
return TSDB_CODE_SUCCESS;
}

8
src/client/src/tscServer.c
View File

@ -124,7 +124,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
taosTFree(pVgroupInfo->epAddr[i].fqdn);
tfree(pVgroupInfo->epAddr[i].fqdn);
pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i]));
pVgroupInfo->epAddr[i].port = pEpSet->port[i];
}
@ -1549,7 +1549,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen);
}
taosTFree(tmpData);
tfree(tmpData);
pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg);
pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META;
@ -1956,7 +1956,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput;
tscFieldInfoUpdateOffset(pQueryInfo);
taosTFree(pTableMeta);
tfree(pTableMeta);
return 0;
}
@ -1981,7 +1981,7 @@ static void createHBObj(STscObj* pObj) {
pSql->cmd.command = pQueryInfo->command;
if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) {
taosTFree(pSql);
tfree(pSql);
return;
}

6
src/client/src/tscSql.c
View File

@ -566,7 +566,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
pRes->rspType = 0;
pSql->subState.numOfSub = 0;
taosTFree(pSql->pSubs);
tfree(pSql->pSubs);
assert(pSql->fp == NULL);
@ -894,7 +894,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
if (sqlLen > tsMaxSQLStringLen) {
tscError("%p sql too long", pSql);
pRes->code = TSDB_CODE_TSC_INVALID_SQL;
taosTFree(pSql);
tfree(pSql);
return pRes->code;
}
@ -903,7 +903,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscError("%p failed to malloc sql string buffer", pSql);
tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj);
taosTFree(pSql);
tfree(pSql);
return pRes->code;
}

4
src/client/src/tscStream.c
View File

@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tscFreeSqlResult(pSql);
taosTFree(pSql->pSubs);
tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetNextLaunchTimer(pStream, pSql);
@ -617,6 +617,6 @@ void taos_close_stream(TAOS_STREAM *handle) {
pStream->pSql = NULL;
taos_free_result(pSql);
taosTFree(pStream);
tfree(pStream);
}
}

28
src/client/src/tscSubquery.c
View File

@ -255,7 +255,7 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
pSupporter->pVgroupTables = NULL;
}
taosTFree(pSupporter->pIdTagList);
tfree(pSupporter->pIdTagList);
tscTagCondRelease(&pSupporter->tagCond);
free(pSupporter);
}
@ -308,7 +308,7 @@ static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
assert(taosArrayGetSize(pVgroupTables) > 0);
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
taosTFree(list);
tfree(list);
}
static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) {
@ -335,7 +335,7 @@ static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupT
taosArrayPush(pNew, &info);
}
taosTFree(list);
tfree(list);
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
return pNew;
@ -1624,8 +1624,8 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
SRetrieveSupport* pSupport = pSub->param;
taosTFree(pSupport->localBuffer);
taosTFree(pSupport);
tfree(pSupport->localBuffer);
tfree(pSupport);
taos_free_result(pSub);
}
@ -1666,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
if (ret != 0) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscQueueAsyncRes(pSql);
taosTFree(pMemoryBuf);
tfree(pMemoryBuf);
return ret;
}
@ -1675,7 +1675,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub);
if (pSql->pSubs == NULL) {
taosTFree(pSql->pSubs);
tfree(pSql->pSubs);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub);
@ -1700,7 +1700,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage));
if (trs->localBuffer == NULL) {
tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
taosTFree(trs);
tfree(trs);
break;
}
@ -1711,8 +1711,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL);
if (pNew == NULL) {
tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno));
taosTFree(trs->localBuffer);
taosTFree(trs);
tfree(trs->localBuffer);
tfree(trs);
break;
}
@ -1766,8 +1766,8 @@ static void tscFreeRetrieveSup(SSqlObj *pSql) {
// SSqlObj *pParentSql = trsupport->pParentSql;
// assert(pSql == pParentSql->pSubs[index]);
taosTFree(trsupport->localBuffer);
taosTFree(trsupport);
tfree(trsupport->localBuffer);
tfree(trsupport);
}
static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
@ -2167,7 +2167,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->res.code = pSql->res.code;
}
taosTFree(pSupporter);
tfree(pSupporter);
if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) {
return;
@ -2442,7 +2442,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) {
assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker
taosTFree(pRes->tsrow);
tfree(pRes->tsrow);
return pRes->tsrow;
}

82
src/client/src/tscUtil.c
View File

@ -245,7 +245,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
}
pQueryInfo->fillType = TSDB_FILL_NONE;
taosTFree(pQueryInfo->fillVal);
tfree(pQueryInfo->fillVal);
}
int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
@ -259,7 +259,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
// not enough memory
if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
taosTFree(pRes->tsrow);
tfree(pRes->tsrow);
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return pRes->code;
}
@ -271,24 +271,24 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
static void tscDestroyResPointerInfo(SSqlRes* pRes) {
if (pRes->buffer != NULL) { // free all buffers containing the multibyte string
for (int i = 0; i < pRes->numOfCols; i++) {
taosTFree(pRes->buffer[i]);
tfree(pRes->buffer[i]);
}
pRes->numOfCols = 0;
}
taosTFree(pRes->pRsp);
tfree(pRes->pRsp);
taosTFree(pRes->tsrow);
taosTFree(pRes->length);
taosTFree(pRes->buffer);
tfree(pRes->tsrow);
tfree(pRes->length);
tfree(pRes->buffer);
taosTFree(pRes->pGroupRec);
taosTFree(pRes->pColumnIndex);
tfree(pRes->pGroupRec);
tfree(pRes->pColumnIndex);
if (pRes->pArithSup != NULL) {
taosTFree(pRes->pArithSup->data);
taosTFree(pRes->pArithSup);
tfree(pRes->pArithSup->data);
tfree(pRes->pArithSup);
}
pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free
@ -305,11 +305,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) {
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache);
taosTFree(pQueryInfo);
tfree(pQueryInfo);
}
pCmd->numOfClause = 0;
taosTFree(pCmd->pQueryInfo);
tfree(pCmd->pQueryInfo);
}
void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
@ -387,14 +387,14 @@ void tscFreeTableMetaHelper(void *pTableMeta) {
assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps; ++i) {
taosTFree(p->vgroupInfo.epAddr[i].fqdn);
tfree(p->vgroupInfo.epAddr[i].fqdn);
}
int32_t numOfEps1 = p->corVgroupInfo.numOfEps;
assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA);
for(int32_t i = 0; i < numOfEps1; ++i) {
taosTFree(p->corVgroupInfo.epAddr[i].fqdn);
tfree(p->corVgroupInfo.epAddr[i].fqdn);
}
}
@ -418,9 +418,9 @@ void tscFreeSqlObj(SSqlObj* pSql) {
pSql->signature = NULL;
pSql->fp = NULL;
taosTFree(pSql->sqlstr);
tfree(pSql->sqlstr);
taosTFree(pSql->pSubs);
tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pSql->self = 0;
@ -428,7 +428,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
tscResetSqlCmdObj(pCmd, false);
memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
taosTFree(pCmd->payload);
tfree(pCmd->payload);
pCmd->allocSize = 0;
tsem_destroy(&pSql->rspSem);
@ -440,15 +440,15 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
return;
}
taosTFree(pDataBlock->pData);
taosTFree(pDataBlock->params);
tfree(pDataBlock->pData);
tfree(pDataBlock->params);
// free the refcount for metermeta
if (pDataBlock->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
}
taosTFree(pDataBlock);
tfree(pDataBlock);
}
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
@ -723,7 +723,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
taosHashCleanup(pVnodeDataBlockHashList);
tscDestroyBlockArrayList(pVnodeDataBlockList);
taosTFree(dataBuf->pData);
tfree(dataBuf->pData);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@ -783,7 +783,7 @@ void tscCloseTscObj(void *param) {
pthread_mutex_destroy(&pObj->mutex);
tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
taosTFree(pObj);
tfree(pObj);
}
bool tscIsInsertData(char* sqlstr) {
@ -947,12 +947,12 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
if (pInfo->pArithExprInfo != NULL) {
tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
taosTFree(pInfo->pArithExprInfo);
tfree(pInfo->pArithExprInfo);
}
}
taosArrayDestroy(pFieldInfo->internalField);
taosTFree(pFieldInfo->final);
tfree(pFieldInfo->final);
memset(pFieldInfo, 0, sizeof(SFieldInfo));
}
@ -1068,7 +1068,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) {
tVariantDestroy(&pExpr->param[i]);
}
taosTFree(pExpr);
tfree(pExpr);
return NULL;
}
@ -1168,11 +1168,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) {
static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) {
for(int32_t i = 0; i < numOfFilters; ++i) {
if (pFilterInfo[i].filterstr) {
taosTFree(pFilterInfo[i].pz);
tfree(pFilterInfo[i].pz);
}
}
taosTFree(pFilterInfo);
tfree(pFilterInfo);
}
SColumn* tscColumnClone(const SColumn* src) {
@ -1238,8 +1238,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
*
*/
static int32_t validateQuoteToken(SStrToken* pToken) {
strdequote(pToken->z);
pToken->n = (uint32_t)strtrim(pToken->z);
tscDequoteAndTrimToken(pToken);
int32_t k = tSQLGetToken(pToken->z, &pToken->type);
@ -1254,8 +1253,6 @@ static int32_t validateQuoteToken(SStrToken* pToken) {
}
void tscDequoteAndTrimToken(SStrToken* pToken) {
assert(pToken->type == TK_STRING);
uint32_t first = 0, last = pToken->n;
// trim leading spaces
@ -1367,7 +1364,8 @@ int32_t tscValidateName(SStrToken* pToken) {
} else {
pStr[firstPartLen] = TS_PATH_DELIMITER[0];
memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n);
pStr[firstPartLen + sizeof(TS_PATH_DELIMITER[0]) + pToken->n] = 0;
uint32_t offset = (uint32_t)(pToken->z - (pStr + firstPartLen + 1));
memset(pToken->z + pToken->n - offset, ' ', offset);
}
pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0]));
pToken->z = pStr;
@ -1460,7 +1458,7 @@ void tscTagCondRelease(STagCond* pTagCond) {
size_t s = taosArrayGetSize(pTagCond->pCond);
for (int32_t i = 0; i < s; ++i) {
SCond* p = taosArrayGet(pTagCond->pCond, i);
taosTFree(p->cond);
tfree(p->cond);
}
taosArrayDestroy(pTagCond->pCond);
@ -1652,7 +1650,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) {
pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf);
taosTFree(pQueryInfo->fillVal);
tfree(pQueryInfo->fillVal);
}
void tscClearSubqueryInfo(SSqlCmd* pCmd) {
@ -1672,7 +1670,7 @@ void tscFreeVgroupTableInfo(SArray* pVgroupTables) {
SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i);
for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
tfree(pInfo->vgInfo.epAddr[j].fqdn);
}
taosArrayDestroy(pInfo->itemList);
@ -1689,7 +1687,7 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) {
SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index);
for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) {
taosTFree(pInfo->vgInfo.epAddr[j].fqdn);
tfree(pInfo->vgInfo.epAddr[j].fqdn);
}
taosArrayDestroy(pInfo->itemList);
@ -1737,7 +1735,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem
free(pTableMetaInfo);
}
taosTFree(pQueryInfo->pTableMetaInfo);
tfree(pQueryInfo->pTableMetaInfo);
}
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta,
@ -2336,7 +2334,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) {
pRes->numOfTotal = num;
taosTFree(pSql->pSubs);
tfree(pSql->pSubs);
pSql->subState.numOfSub = 0;
pSql->fp = fp;
@ -2461,11 +2459,11 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) {
SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i];
for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) {
taosTFree(pVgroupInfo->epAddr[j].fqdn);
tfree(pVgroupInfo->epAddr[j].fqdn);
}
}
taosTFree(vgroupList);
tfree(vgroupList);
return NULL;
}
@ -2473,7 +2471,7 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) {
dst->vgId = src->vgId;
dst->numOfEps = src->numOfEps;
for(int32_t i = 0; i < dst->numOfEps; ++i) {
taosTFree(dst->epAddr[i].fqdn);
tfree(dst->epAddr[i].fqdn);
dst->epAddr[i].port = src->epAddr[i].port;
dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn);
}

4
src/common/inc/tdataformat.h
View File

@ -80,7 +80,7 @@ typedef struct {
#define schemaFLen(s) ((s)->flen)
#define schemaVLen(s) ((s)->vlen)
#define schemaColAt(s, i) ((s)->columns + i)
#define tdFreeSchema(s) taosTFree((s))
#define tdFreeSchema(s) tfree((s))
STSchema *tdDupSchema(STSchema *pSchema);
int tdEncodeSchema(void **buf, STSchema *pSchema);
@ -308,7 +308,7 @@ typedef struct {
#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
#define kvRowFree(r) taosTFree(r)
#define kvRowFree(r) tfree(r)
#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))
SKVRow tdKVRowDup(SKVRow row);

1
src/common/inc/tglobal.h
View File

@ -44,6 +44,7 @@ extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryThreads; // todo remove it
extern int8_t tsDaylight;
extern char tsTimezone[];

10
src/common/src/tdataformat.c
View File

@ -97,7 +97,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
if (pBuilder) {
taosTFree(pBuilder->columns);
tfree(pBuilder->columns);
}
}
@ -339,8 +339,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
void tdFreeDataCols(SDataCols *pCols) {
if (pCols) {
taosTFree(pCols->buf);
taosTFree(pCols->cols);
tfree(pCols->buf);
tfree(pCols->cols);
free(pCols);
}
}
@ -669,8 +669,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) {
}
void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) {
taosTFree(pBuilder->pColIdx);
taosTFree(pBuilder->buf);
tfree(pBuilder->pColIdx);
tfree(pBuilder->buf);
}
void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) {

13
src/common/src/tglobal.c
View File

@ -51,6 +51,7 @@ int32_t tsMaxShellConns = 5000;
int32_t tsMaxConnections = 5000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 1;
float tsRatioOfQueryThreads = 0.5f;
int8_t tsDaylight = 0;
char tsTimezone[TSDB_TIMEZONE_LEN] = {0};
@ -218,6 +219,8 @@ int32_t (*monitorStartSystemFp)() = NULL;
void (*monitorStopSystemFp)() = NULL;
void (*monitorExecuteSQLFp)(char *sql) = NULL;
char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() {
@ -424,6 +427,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "numOfCommitThreads";
cfg.ptr = &tsNumOfCommitThreads;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 1;
cfg.maxValue = 100;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "ratioOfQueryThreads";
cfg.ptr = &tsRatioOfQueryThreads;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;

2
src/common/src/tvariant.c
View File

@ -125,7 +125,7 @@ void tVariantDestroy(tVariant *pVar) {
if (pVar == NULL) return;
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
taosTFree(pVar->pz);
tfree(pVar->pz);
pVar->nLen = 0;
}

154
src/connector/C#/TDengineDriver.cs Normal file
View File

@ -0,0 +1,154 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
namespace TDengineDriver
{
enum TDengineDataType {
TSDB_DATA_TYPE_NULL = 0, // 1 bytes
TSDB_DATA_TYPE_BOOL = 1, // 1 bytes
TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes
TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes
TSDB_DATA_TYPE_INT = 4, // 4 bytes
TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes
TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes
TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes
TSDB_DATA_TYPE_BINARY = 8, // string
TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes
TSDB_DATA_TYPE_NCHAR = 10 // unicode string
}
enum TDengineInitOption
{
TSDB_OPTION_LOCALE = 0,
TSDB_OPTION_CHARSET = 1,
TSDB_OPTION_TIMEZONE = 2,
TDDB_OPTION_CONFIGDIR = 3,
TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4
}
class TDengineMeta
{
public string name;
public short size;
public byte type;
public string TypeName()
{
switch ((TDengineDataType)type)
{
case TDengineDataType.TSDB_DATA_TYPE_BOOL:
return "BOOLEAN";
case TDengineDataType.TSDB_DATA_TYPE_TINYINT:
return "BYTE";
case TDengineDataType.TSDB_DATA_TYPE_SMALLINT:
return "SHORT";
case TDengineDataType.TSDB_DATA_TYPE_INT:
return "INT";
case TDengineDataType.TSDB_DATA_TYPE_BIGINT:
return "LONG";
case TDengineDataType.TSDB_DATA_TYPE_FLOAT:
return "FLOAT";
case TDengineDataType.TSDB_DATA_TYPE_DOUBLE:
return "DOUBLE";
case TDengineDataType.TSDB_DATA_TYPE_BINARY:
return "STRING";
case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP:
return "TIMESTAMP";
case TDengineDataType.TSDB_DATA_TYPE_NCHAR:
return "NCHAR";
default:
return "undefine";
}
}
}
class TDengine
{
public const int TSDB_CODE_SUCCESS = 0;
[DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)]
static extern public void Init();
[DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)]
static extern public void Cleanup();
[DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)]
static extern public void Options(int option, string value);
[DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)]
static extern public IntPtr Connect(string ip, string user, string password, string db, short port);
[DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)]
static extern private IntPtr taos_errstr(IntPtr res);
static public string Error(IntPtr res)
{
IntPtr errPtr = taos_errstr(res);
return Marshal.PtrToStringAnsi(errPtr);
}
[DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)]
static extern public int ErrorNo(IntPtr res);
[DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)]
static extern public IntPtr Query(IntPtr conn, string sqlstr);
[DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)]
static extern public int AffectRows(IntPtr res);
[DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)]
static extern public int FieldCount(IntPtr res);
[DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)]
static extern private IntPtr taos_fetch_fields(IntPtr res);
static public List<TDengineMeta> FetchFields(IntPtr res)
{
const int fieldSize = 68;
List<TDengineMeta> metas = new List<TDengineMeta>();
if (res == IntPtr.Zero)
{
return metas;
}
int fieldCount = FieldCount(res);
IntPtr fieldsPtr = taos_fetch_fields(res);
for (int i = 0; i < fieldCount; ++i)
{
int offset = i * fieldSize;
TDengineMeta meta = new TDengineMeta();
meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset);
meta.type = Marshal.ReadByte(fieldsPtr + offset + 65);
meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66);
metas.Add(meta);
}
return metas;
}
[DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)]
static extern public IntPtr FetchRows(IntPtr res);
[DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)]
static extern public IntPtr FreeResult(IntPtr res);
[DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)]
static extern public int Close(IntPtr taos);
}
}

2
src/connector/go

@ -1 +1 @@
Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766
Subproject commit 66df175edf467df6f80d07789f1a35a6d2551310

3
src/connector/jdbc/deploy-pom.xml
View File

@ -5,14 +5,13 @@
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.6</version>
<version>2.0.10</version>
<packaging>jar</packaging>
<name>JDBCDriver</name>
<url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
<description>TDengine JDBC Driver</description>
<licenses>
<license>
<name>GNU AFFERO GENERAL PUBLIC LICENSE Version 3</name>

24
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
View File

@ -111,8 +111,8 @@ public class TSDBJNIConnector {
* @throws SQLException
*/
public long executeQuery(String sql) throws SQLException {
// close previous result set if the user forgets to invoke the
// free method to close previous result set.
// close previous result set if the user forgets to invoke the
// free method to close previous result set.
if (!this.isResultsetClosed) {
freeResultSet(taosResultSetPointer);
}
@ -122,23 +122,23 @@ public class TSDBJNIConnector {
pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
} catch (Exception e) {
e.printStackTrace();
this.freeResultSet(pSql);
this.freeResultSetImp(this.taos, pSql);
throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding"));
}
int code = this.getErrCode(pSql);
if (code != 0) {
affectedRows = -1;
String msg = this.getErrMsg(pSql);
this.freeResultSet(pSql);
this.freeResultSetImp(this.taos, pSql);
throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code);
}
// Try retrieving result set for the executed SQL using the current connection pointer.
taosResultSetPointer = this.getResultSetImp(this.taos, pSql);
isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER);
return pSql;
}
@ -171,11 +171,11 @@ public class TSDBJNIConnector {
}
private native long getResultSetImp(long connection, long pSql);
public boolean isUpdateQuery(long pSql) {
return isUpdateQueryImp(this.taos, pSql) == 1? true:false;
return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false;
}
private native long isUpdateQueryImp(long connection, long pSql);
/**
@ -191,7 +191,7 @@ public class TSDBJNIConnector {
res = this.freeResultSetImp(this.taos, result);
taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER;
}
isResultsetClosed = true;
return res;
}
@ -274,7 +274,7 @@ public class TSDBJNIConnector {
* Consume a subscription
*/
long consume(long subscription) {
return this.consumeImp(subscription);
return this.consumeImp(subscription);
}
private native long consumeImp(long subscription);

2
src/cq/src/cqMain.c
View File

@ -115,7 +115,7 @@ void cqClose(void *handle) {
SCqObj *pTemp = pObj;
pObj = pObj->next;
tdFreeSchema(pTemp->pSchema);
taosTFree(pTemp->sqlStr);
tfree(pTemp->sqlStr);
free(pTemp);
}

6
src/dnode/src/dnodeMInfos.c
View File

@ -77,15 +77,15 @@ void dnodeUpdateMInfos(SMnodeInfos *minfos) {
void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) {
if (ep->numOfEps <= 0) {
dError("mnode EP list for peer is changed, but content is invalid, discard it");
dError("minfos is changed, but content is invalid, discard it");
return;
}
pthread_mutex_lock(&tsMInfosMutex);
dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
for (int i = 0; i < ep->numOfEps; ++i) {
ep->port[i] -= TSDB_PORT_DNODEDNODE;
dInfo("mnode index:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
}
tsMEpSet = *ep;
pthread_mutex_unlock(&tsMInfosMutex);

2
src/dnode/src/dnodeMPeer.c
View File

@ -84,7 +84,7 @@ void dnodeCleanupMPeer() {
taosCloseQset(tsMPeerQset);
tsMPeerQset = NULL;
taosTFree(tsMPeerWP.worker);
tfree(tsMPeerWP.worker);
}
int32_t dnodeAllocateMPeerQueue() {

2
src/dnode/src/dnodeMWrite.c
View File

@ -86,7 +86,7 @@ void dnodeCleanupMWrite() {
taosCloseQset(tsMWriteQset);
tsMWriteQset = NULL;
taosTFree(tsMWriteWP.worker);
tfree(tsMWriteWP.worker);
}
int32_t dnodeAllocMWritequeue() {

4
src/dnode/src/dnodeVWrite.c
View File

@ -207,8 +207,8 @@ static void *dnodeProcessVWriteQueue(void *param) {
bool forceFsync = false;
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%d version:%" PRIu64, pWrite->rpcAhandle, pWrite,
taosMsg[pWrite->pHead->msgType], qtype, pWrite->pHead->version);
dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite,
taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);
pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
if (pWrite->code <= 0) pWrite->processedCount = 1;

1
src/inc/mnode.h
View File

@ -47,6 +47,7 @@ typedef struct SMnodeMsg {
int8_t successed;
int8_t expected;
int8_t retry;
int32_t incomingTs;
int32_t code;
void * pObj;
SRpcMsg rpcMsg;

6
src/inc/taosdef.h
View File

@ -334,7 +334,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MIN_DAYS_PER_FILE 1
#define TSDB_MAX_DAYS_PER_FILE 3650
#define TSDB_DEFAULT_DAYS_PER_FILE 2
#define TSDB_DEFAULT_DAYS_PER_FILE 10
#define TSDB_MIN_KEEP 1 // data in db to be reserved.
#define TSDB_MAX_KEEP 365000 // data in db to be reserved.
@ -431,6 +431,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_PORT_HTTP 11
#define TSDB_PORT_ARBITRATOR 12
#define TSDB_MAX_WAL_SIZE (1024*1024)
typedef enum {
TAOS_QTYPE_RPC = 0,
TAOS_QTYPE_FWD = 1,
@ -467,6 +469,8 @@ typedef enum {
TSDB_CHECK_ITEM_MAX
} ECheckItemType;
extern char *qtypeStr[];
#ifdef __cplusplus
}
#endif

4
src/inc/taoserror.h
View File

@ -181,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid da
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced")
// dnode
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed")
@ -236,7 +237,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached")
// grant
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired")
@ -260,6 +261,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sy
// wal
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit")
// http
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin")

6
src/inc/tsdb.h
View File

@ -321,6 +321,12 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
*/
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);
int tsdbInitCommitQueue(int nthreads);
void tsdbDestroyCommitQueue();
int tsdbSyncCommit(TSDB_REPO_T *repo);
void tsdbIncCommitRef(int vgId);
void tsdbDecCommitRef(int vgId);
#ifdef __cplusplus
}
#endif

57
src/inc/tsync.h
View File

@ -51,9 +51,9 @@ typedef struct {
} SSyncCfg;
typedef struct {
int selfIndex;
uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
int role[TAOS_SYNC_MAX_REPLICA];
int32_t selfIndex;
uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
int32_t role[TAOS_SYNC_MAX_REPLICA];
} SNodesRole;
/*
@ -83,25 +83,24 @@ typedef void (*FNotifyRole)(void *ahandle, int8_t role);
typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);
// when data file is synced successfully, notity app
typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
typedef struct {
int32_t vgId; // vgroup ID
uint64_t version; // initial version
SSyncCfg syncCfg; // configuration from mgmt
char path[128]; // path to the file
void *ahandle; // handle provided by APP
FGetFileInfo getFileInfo;
FGetWalInfo getWalInfo;
FWriteToCache writeToCache;
FConfirmForward confirmForward;
FNotifyRole notifyRole;
FNotifyFlowCtrl notifyFlowCtrl;
int32_t vgId; // vgroup ID
uint64_t version; // initial version
SSyncCfg syncCfg; // configuration from mgmt
char path[128]; // path to the file
void * ahandle; // handle provided by APP
FGetFileInfo getFileInfo;
FGetWalInfo getWalInfo;
FWriteToCache writeToCache;
FConfirmForward confirmForward;
FNotifyRole notifyRole;
FNotifyFlowCtrl notifyFlowCtrl;
FNotifyFileSynced notifyFileSynced;
} SSyncInfo;
typedef void* tsync_h;
typedef void *tsync_h;
int32_t syncInit();
void syncCleanUp();
@ -109,22 +108,22 @@ void syncCleanUp();
int64_t syncStart(const SSyncInfo *);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg *);
int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype);
int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype);
void syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
void syncRecover(int64_t rid); // recover from other nodes:
int syncGetNodesRole(int64_t rid, SNodesRole *);
void syncRecover(int64_t rid); // recover from other nodes:
int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
extern char *syncRole[];
extern char *syncRole[];
//global configurable parameters
extern int tsMaxSyncNum;
extern int tsSyncTcpThreads;
extern int tsMaxWatchFiles;
extern int tsSyncTimer;
extern int tsMaxFwdInfo;
extern int sDebugFlag;
extern char tsArbitrator[];
extern uint16_t tsSyncPort;
extern int32_t tsMaxSyncNum;
extern int32_t tsSyncTcpThreads;
extern int32_t tsMaxWatchFiles;
extern int32_t tsSyncTimer;
extern int32_t tsMaxFwdInfo;
extern int32_t sDebugFlag;
extern char tsArbitrator[];
extern uint16_t tsSyncPort;
#ifdef __cplusplus
}

20
src/inc/twal.h
View File

@ -54,15 +54,17 @@ typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg)
int32_t walInit();
void walCleanUp();
twalh walOpen(char *path, SWalCfg *pCfg);
int32_t walAlter(twalh pWal, SWalCfg *pCfg);
void walStop(twalh);
void walClose(twalh);
int32_t walRenew(twalh);
int32_t walWrite(twalh, SWalHead *);
void walFsync(twalh, bool forceFsync);
int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
twalh walOpen(char *path, SWalCfg *pCfg);
int32_t walAlter(twalh pWal, SWalCfg *pCfg);
void walStop(twalh);
void walClose(twalh);
int32_t walRenew(twalh);
void walRemoveOneOldFile(twalh);
void walRemoveAllOldFiles(twalh);
int32_t walWrite(twalh, SWalHead *);
void walFsync(twalh, bool forceFsync);
int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
uint64_t walGetVersion(twalh);
#ifdef __cplusplus

2
src/kit/shell/inc/shell.h
View File

@ -60,7 +60,7 @@ typedef struct SShellArguments {
extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
extern TAOS* shellInit(SShellArguments* args);
extern void* shellLoopQuery(void* arg);
extern void taos_error(TAOS_RES* tres);
extern void taos_error(TAOS_RES* tres, int64_t st);
extern int regex_match(const char* s, const char* reg, int cflags);
void shellReadCommand(TAOS* con, char command[]);
int32_t shellRunCommand(TAOS* con, char* command);

4
src/kit/shell/src/shellDarwin.c
View File

@ -229,8 +229,8 @@ void shellReadCommand(TAOS *con, char *command) {
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosTFree(cmd.buffer);
taosTFree(cmd.command);
tfree(cmd.buffer);
tfree(cmd.command);
return;
} else {
updateBuffer(&cmd);

15
src/kit/shell/src/shellEngine.c
View File

@ -193,7 +193,7 @@ int32_t shellRunCommand(TAOS* con, char* command) {
history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL ||
strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) {
if (history.hist[history.hend] != NULL) {
taosTFree(history.hist[history.hend]);
tfree(history.hist[history.hend]);
}
history.hist[history.hend] = strdup(command);
@ -296,7 +296,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
TAOS_RES* pSql = taos_query_h(con, command, &result);
if (taos_errno(pSql)) {
taos_error(pSql);
taos_error(pSql, st);
return;
}
@ -770,7 +770,7 @@ void read_history() {
return;
}
while ((read_size = taosGetline(&line, &line_size, f)) != -1) {
while ((read_size = tgetline(&line, &line_size, f)) != -1) {
line[read_size - 1] = '\0';
history.hist[history.hend] = strdup(line);
@ -800,16 +800,17 @@ void write_history() {
for (int i = history.hstart; i != history.hend;) {
if (history.hist[i] != NULL) {
fprintf(f, "%s\n", history.hist[i]);
taosTFree(history.hist[i]);
tfree(history.hist[i]);
}
i = (i + 1) % MAX_HISTORY_SIZE;
}
fclose(f);
}
void taos_error(TAOS_RES *tres) {
void taos_error(TAOS_RES *tres, int64_t st) {
int64_t et = taosGetTimestampUs();
atomic_store_ptr(&result, 0);
fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres));
fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6);
taos_free_result(tres);
}
@ -853,7 +854,7 @@ void source_file(TAOS *con, char *fptr) {
return;
}
while ((read_len = taosGetline(&line, &line_len, f)) != -1) {
while ((read_len = tgetline(&line, &line_len, f)) != -1) {
if (read_len >= tsMaxSQLStringLen) continue;
line[--read_len] = '\0';

6
src/kit/shell/src/shellLinux.c
View File

@ -232,8 +232,8 @@ void shellReadCommand(TAOS *con, char *command) {
printf("\n");
if (isReadyGo(&cmd)) {
sprintf(command, "%s%s", cmd.buffer, cmd.command);
taosTFree(cmd.buffer);
taosTFree(cmd.command);
tfree(cmd.buffer);
tfree(cmd.command);
return;
} else {
updateBuffer(&cmd);
@ -351,7 +351,7 @@ void *shellLoopQuery(void *arg) {
reset_terminal_mode();
} while (shellRunCommand(con, command) == 0);
taosTFree(command);
tfree(command);
exitShell();
pthread_cleanup_pop(1);

98
src/kit/taosdemo/taosdemo.c
View File

@ -475,6 +475,7 @@ typedef struct {
tsem_t mutex_sem;
int notFinished;
tsem_t lock_sem;
int counter;
} info;
typedef struct {
@ -766,6 +767,7 @@ int main(int argc, char *argv[]) {
t_info->data_of_rate = rate;
t_info->end_table_id = i < b ? last + a : last + a - 1;
last = t_info->end_table_id + 1;
t_info->counter = 0;
tsem_init(&(t_info->mutex_sem), 0, 1);
t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1;
@ -788,14 +790,14 @@ int main(int argc, char *argv[]) {
printf("ASYNC Insert with %d connections:\n", threads);
}
fprintf(fp, "|%10.d | %10.2f | %10.2f | %10.4f |\n\n",
ntables * nrecords_per_table, ntables * nrecords_per_table / t,
(ntables * nrecords_per_table) / (t * nrecords_per_request),
fprintf(fp, "|%"PRIu64" | %10.2f | %10.2f | %10.4f |\n\n",
(int64_t)ntables * nrecords_per_table, ntables * nrecords_per_table / t,
((int64_t)ntables * nrecords_per_table) / (t * nrecords_per_request),
t * 1000);
printf("Spent %.4f seconds to insert %lld records with %d record(s) per request: %.2f records/second\n",
t, (long long int)ntables * nrecords_per_table, nrecords_per_request,
((long long int)ntables * nrecords_per_table) / t);
printf("Spent %.4f seconds to insert %"PRIu64" records with %d record(s) per request: %.2f records/second\n",
t, (int64_t)ntables * nrecords_per_table, nrecords_per_request,
(int64_t)ntables * nrecords_per_table / t);
for (int i = 0; i < threads; i++) {
info *t_info = infos + i;
@ -879,6 +881,7 @@ int main(int argc, char *argv[]) {
taos_close(rInfo->taos);
}
taos_cleanup();
return 0;
}
@ -955,7 +958,7 @@ void querySqlFile(TAOS* taos, char* sqlFile)
double t = getCurrentTime();
while ((read_len = taosGetline(&line, &line_len, fp)) != -1) {
while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
if (read_len >= MAX_SQL_SIZE) continue;
line[--read_len] = '\0';
@ -1283,68 +1286,39 @@ void *syncWrite(void *sarg) {
void *asyncWrite(void *sarg) {
info *winfo = (info *)sarg;
sTable *tb_infos = (sTable *)malloc(sizeof(sTable) * (winfo->end_table_id - winfo->start_table_id + 1));
for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) {
sTable *tb_info = tb_infos + tID - winfo->start_table_id;
tb_info->data_type = winfo->datatype;
tb_info->ncols_per_record = winfo->ncols_per_record;
tb_info->taos = winfo->taos;
sprintf(tb_info->tb_name, "%s.%s%d", winfo->db_name, winfo->tb_prefix, tID);
tb_info->timestamp = winfo->start_time;
tb_info->counter = 0;
tb_info->target = winfo->nrecords_per_table;
tb_info->len_of_binary = winfo->len_of_binary;
tb_info->nrecords_per_request = winfo->nrecords_per_request;
tb_info->mutex_sem = &(winfo->mutex_sem);
tb_info->notFinished = &(winfo->notFinished);
tb_info->lock_sem = &(winfo->lock_sem);
tb_info->data_of_order = winfo->data_of_order;
tb_info->data_of_rate = winfo->data_of_rate;
/* char buff[BUFFER_SIZE] = "\0"; */
/* sprintf(buff, "insert into %s values (0, 0)", tb_info->tb_name); */
/* queryDB(tb_info->taos,buff); */
taos_query_a(winfo->taos, "show databases", callBack, tb_info);
}
taos_query_a(winfo->taos, "show databases", callBack, winfo);
tsem_wait(&(winfo->lock_sem));
free(tb_infos);
return NULL;
}
void callBack(void *param, TAOS_RES *res, int code) {
sTable *tb_info = (sTable *)param;
char **datatype = tb_info->data_type;
int ncols_per_record = tb_info->ncols_per_record;
int len_of_binary = tb_info->len_of_binary;
int64_t tmp_time = tb_info->timestamp;
info* winfo = (info*)param;
char **datatype = winfo->datatype;
int ncols_per_record = winfo->ncols_per_record;
int len_of_binary = winfo->len_of_binary;
if (code < 0) {
fprintf(stderr, "failed to insert data %d:reason; %s\n", code, taos_errstr(res));
exit(EXIT_FAILURE);
int64_t tmp_time = winfo->start_time;
char *buffer = calloc(1, BUFFER_SIZE);
char *data = calloc(1, MAX_DATA_SIZE);
char *pstr = buffer;
pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id);
if (winfo->counter >= winfo->nrecords_per_table) {
winfo->start_table_id++;
winfo->counter = 0;
}
// If finished;
if (tb_info->counter >= tb_info->target) {
tsem_wait(tb_info->mutex_sem);
(*(tb_info->notFinished))--;
if (*(tb_info->notFinished) == 0) tsem_post(tb_info->lock_sem);
tsem_post(tb_info->mutex_sem);
if (winfo->start_table_id > winfo->end_table_id) {
tsem_post(&winfo->lock_sem);
free(buffer);
free(data);
taos_free_result(res);
return;
}
char buffer[BUFFER_SIZE] = "\0";
char data[MAX_DATA_SIZE];
char *pstr = buffer;
pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name);
for (int i = 0; i < tb_info->nrecords_per_request; i++) {
for (int i = 0; i < winfo->nrecords_per_request; i++) {
int rand_num = rand() % 100;
if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate)
if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate)
{
int64_t d = tmp_time - rand() % 1000000 + rand_num;
generateData(data, datatype, ncols_per_record, d, len_of_binary);
@ -1353,15 +1327,15 @@ void callBack(void *param, TAOS_RES *res, int code) {
generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary);
}
pstr += sprintf(pstr, "%s", data);
tb_info->counter++;
winfo->counter++;
if (tb_info->counter >= tb_info->target) {
if (winfo->counter >= winfo->nrecords_per_table) {
break;
}
}
tb_info->timestamp = tmp_time;
taos_query_a(tb_info->taos, buffer, callBack, tb_info);
taos_query_a(winfo->taos, buffer, callBack, winfo);
free(buffer);
free(data);
taos_free_result(res);
}

2
src/mnode/src/mnodeAcct.c
View File

@ -34,7 +34,7 @@ static int32_t mnodeCreateRootAcct();
static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
SAcctObj *pAcct = pOper->pObj;
pthread_mutex_destroy(&pAcct->mutex);
taosTFree(pOper->pObj);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}

2
src/mnode/src/mnodeCluster.c
View File

@ -33,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) {
taosTFree(pOper->pObj);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}

6
src/mnode/src/mnodeDb.c
View File

@ -52,8 +52,8 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg);
static void mnodeDestroyDb(SDbObj *pDb) {
pthread_mutex_destroy(&pDb->mutex);
taosTFree(pDb->vgList);
taosTFree(pDb);
tfree(pDb->vgList);
tfree(pDb);
}
static int32_t mnodeDbActionDestroy(SSdbOper *pOper) {
@ -405,7 +405,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
code = mnodeCheckDbCfg(&pDb->cfg);
if (code != TSDB_CODE_SUCCESS) {
taosTFree(pDb);
tfree(pDb);
return code;
}

6
src/mnode/src/mnodeDnode.c
View File

@ -88,7 +88,7 @@ static char* offlineReason[] = {
};
static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) {
taosTFree(pOper->pObj);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@ -584,7 +584,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
}
mDebug("dnode:%d, from offline to online", pDnode->dnodeId);
mInfo("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
balanceSyncNotify();
@ -655,7 +655,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
int32_t code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
int dnodeId = pDnode->dnodeId;
taosTFree(pDnode);
tfree(pDnode);
mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code));
} else {
mLInfo("dnode:%d is created", pDnode->dnodeId);

1
src/mnode/src/mnodeInt.c
View File

@ -40,6 +40,7 @@ void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
pMsg->rpcMsg = *pRpcMsg;
pMsg->rpcMsg.pCont = pMsg->pCont;
pMsg->incomingTs = taosGetTimestampSec();
memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen);
return pMsg;

6
src/mnode/src/mnodeMnode.c
View File

@ -59,7 +59,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
#endif
static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) {
taosTFree(pOper->pObj);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@ -342,14 +342,14 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
}
if (code != TSDB_CODE_SUCCESS) {
taosTFree(pMnode);
tfree(pMnode);
return;
}
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code));
taosTFree(pMnode);
tfree(pMnode);
}
}

4
src/mnode/src/mnodePeer.c
View File

@ -63,9 +63,9 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
epSet->inUse = (i + 1) % epSet->numOfEps;
mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}

4
src/mnode/src/mnodeProfile.c
View File

@ -131,8 +131,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
static void mnodeFreeConn(void *data) {
SConnObj *pConn = data;
taosTFree(pConn->pQueries);
taosTFree(pConn->pStreams);
tfree(pConn->pQueries);
tfree(pConn->pStreams);
mDebug("connId:%d, is destroyed", pConn->connId);
}

2
src/mnode/src/mnodeSdb.c
View File

@ -984,7 +984,7 @@ void sdbCleanupWriteWorker() {
}
sdbFreeWritequeue();
taosTFree(tsSdbPool.writeWorker);
tfree(tsSdbPool.writeWorker);
mInfo("sdb write is closed");
}

2
src/mnode/src/mnodeShow.c
View File

@ -415,7 +415,7 @@ static void mnodeFreeShowObj(void *data) {
sdbFreeIter(pShow->pIter);
mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
taosTFree(pShow);
tfree(pShow);
}
static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) {

33
src/mnode/src/mnodeTable.c
View File

@ -45,6 +45,10 @@
#include "mnodeRead.h"
#include "mnodePeer.h"
#define ALTER_CTABLE_RETRY_TIMES 3
#define CREATE_CTABLE_RETRY_TIMES 10
#define CREATE_CTABLE_RETRY_SEC 14
static void * tsChildTableSdb;
static void * tsSuperTableSdb;
static int32_t tsChildTableUpdateSize;
@ -89,10 +93,10 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg);
static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName);
static void mnodeDestroyChildTable(SCTableObj *pTable) {
taosTFree(pTable->info.tableId);
taosTFree(pTable->schema);
taosTFree(pTable->sql);
taosTFree(pTable);
tfree(pTable->info.tableId);
tfree(pTable->schema);
tfree(pTable->sql);
tfree(pTable);
}
static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) {
@ -421,9 +425,9 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) {
taosHashCleanup(pStable->vgHash);
pStable->vgHash = NULL;
}
taosTFree(pStable->info.tableId);
taosTFree(pStable->schema);
taosTFree(pStable);
tfree(pStable->info.tableId);
tfree(pStable->schema);
tfree(pStable);
}
static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
@ -2421,7 +2425,9 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
dnodeSendRpcMWriteRsp(mnodeMsg, code);
}
} else {
if (mnodeMsg->retry++ < 10) {
mnodeMsg->retry++;
int32_t sec = taosGetTimestampSec();
if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) {
mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
" result:%s thandle:%p",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid,
@ -2429,13 +2435,18 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
dnodeDelayReprocessMWriteMsg(mnodeMsg);
} else {
mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p",
mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64
", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry);
SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
sdbDeleteRow(&oper);
if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) {
//Avoid retry again in client
rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY;
}
dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
}
}
@ -2456,7 +2467,7 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
} else {
if (mnodeMsg->retry++ < 3) {
if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) {
mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
mnodeMsg->rpcMsg.handle);

4
src/mnode/src/mnodeUser.c
View File

@ -43,7 +43,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg);
static int32_t mnodeUserActionDestroy(SSdbOper *pOper) {
taosTFree(pOper->pObj);
tfree(pOper->pObj);
return TSDB_CODE_SUCCESS;
}
@ -270,7 +270,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
code = sdbInsertRow(&oper);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
taosTFree(pUser);
tfree(pUser);
} else {
mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg));
}

2
src/mnode/src/mnodeVgroup.c
View File

@ -69,7 +69,7 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) {
pVgroup->idPool = NULL;
}
taosTFree(pVgroup);
tfree(pVgroup);
}
static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) {

1
src/os/inc/os.h
View File

@ -53,7 +53,6 @@ extern "C" {
#endif
#include "osDef.h"
#include "osAlloc.h"
#include "osAtomic.h"
#include "osCommon.h"
#include "osDir.h"

43
src/os/inc/osAlloc.h
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_OS_ALLOC_H
#define TDENGINE_OS_ALLOC_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef TAOS_OS_FUNC_ALLOC
#define tmalloc(size) malloc(size)
#define tcalloc(nmemb, size) calloc(nmemb, size)
#define trealloc(p, size) realloc(p, size)
#define tmemalign(alignment, size) malloc(size)
#define tfree(p) free(p)
#define tmemzero(p, size) memset(p, 0, size)
#else
void *tmalloc(int32_t size);
void *tcalloc(int32_t nmemb, int32_t size);
void *trealloc(void *p, int32_t size);
void *tmemalign(int32_t alignment, int32_t size);
void tfree(void *p);
void tmemzero(void *p, int32_t size);
#endif
#ifdef __cplusplus
}
#endif
#endif

61
src/os/inc/osMemory.h
View File

@ -31,6 +31,7 @@ typedef enum {
void taosSetAllocMode(int mode, const char *path, bool autoDump);
void taosDumpMemoryLeak();
// used in tsdb module
void * taosTMalloc(size_t size);
void * taosTCalloc(size_t nmemb, size_t size);
void * taosTRealloc(void *ptr, size_t size);
@ -38,7 +39,14 @@ void taosTZfree(void *ptr);
size_t taosTSizeof(void *ptr);
void taosTMemset(void *ptr, int c);
#define taosTFree(x) \
// used in other module
#define tmalloc(size) malloc(size)
#define tcalloc(num, size) calloc(num, size)
#define trealloc(ptr, size) realloc(ptr, size)
#define tstrdup(str) taosStrdupImp(str)
#define tstrndup(str, size) taosStrndupImp(str, size)
#define tgetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream)
#define tfree(x) \
do { \
if (x) { \
free((void *)(x)); \
@ -46,37 +54,30 @@ void taosTMemset(void *ptr, int c);
} \
} while (0);
#define taosMalloc(size) malloc(size)
#define taosCalloc(num, size) calloc(num, size)
#define taosRealloc(ptr, size) realloc(ptr, size)
#define taosFree(ptr) free(ptr)
#define taosStrdup(str) taosStrdupImp(str)
#define taosStrndup(str, size) taosStrndupImp(str, size)
#define taosGetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream)
#ifdef TAOS_MEM_CHECK
#ifdef TAOS_MEM_CHECK_TEST
void * taos_malloc(size_t size, const char *file, uint32_t line);
void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
void taos_free(void *ptr, const char *file, uint32_t line);
char * taos_strdup(const char *str, const char *file, uint32_t line);
char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
#undef taosMalloc
#undef taosCalloc
#undef taosRealloc
#undef taosFree
#undef taosStrdup
#undef taosStrndup
#undef taosGetline
#define taosMalloc(size) taos_malloc(size, __FILE__, __LINE__)
#define taosCalloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
#define taosRealloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
#define taosFree(ptr) taos_free(ptr, __FILE__, __LINE__)
//#define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__)
//#define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
//#define taosGetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
void * taosMallocMem(size_t size, const char *file, uint32_t line);
void * taosCallocMem(size_t num, size_t size, const char *file, uint32_t line);
void * taosReallocMem(void *ptr, size_t size, const char *file, uint32_t line);
void taosFreeMem(void *ptr, const char *file, uint32_t line);
char * taosStrdupMem(const char *str, const char *file, uint32_t line);
char * taosStrndupMem(const char *str, size_t size, const char *file, uint32_t line);
ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
#undef tmalloc
#undef tcalloc
#undef trealloc
#undef tfree
#define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__)
#define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__)
#define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__)
#define tfree(ptr) taosFreeMem(ptr, __FILE__, __LINE__)
// #undef tstrdup
// #undef tstrndup
// #undef tgetline
// #define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__)
// #define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
// #define tgetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
#endif
#endif

79
src/os/src/detail/osAlloc.c
View File

@ -1,79 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tulog.h"
#include "osAlloc.h"
#define TSDB_HAVE_MEMALIGN
#ifdef TAOS_OS_FUNC_ALLOC
void *tmalloc(int32_t size) {
void *p = malloc(size);
if (p == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to malloc memory, size:%d reason:%s", size, strerror(errno));
}
return p;
}
void *tcalloc(int32_t nmemb, int32_t size) {
void *p = calloc(nmemb, size);
if (p == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to calloc memory, nmemb:%d size:%d reason:%s", nmemb, size, strerror(errno));
}
return p;
}
void *trealloc(void *p, int32_t size) {
p = realloc(p, size);
if (p == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to realloc memory, size:%d reason:%s", size, strerror(errno));
}
return p;
}
void tfree(void *p) { free(p); }
void tmemzero(void *p, int32_t size) { memset(p, 0, size); }
#ifdef TSDB_HAVE_MEMALIGN
void *tmemalign(int32_t alignment, int32_t size) {
void *p;
int err = posix_memalign(&p, alignment, size);
if (err) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to memalign memory, alignment:%d size:%d reason:%s", alignment, size, strerror(err));
p = NULL;
}
return p;
}
#else
void *tmemalign(int32_t alignment, int32_t size) { return tmalloc(size); }
#endif
#endif

2
src/os/src/detail/osFile.c
View File

@ -132,7 +132,7 @@ int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) {
// if (leftbytes > 1000000000) leftbytes = 1000000000;
sentbytes = sendfile(dfd, sfd, offset, leftbytes);
if (sentbytes == -1) {
if (errno == EINTR) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
} else {
return -1;

124
src/os/src/detail/osMemory.c
View File

@ -28,7 +28,7 @@ static FILE* fpAllocLog = NULL;
extern int32_t taosGetTimestampSec();
static int32_t startTime = INT32_MAX;
static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
static bool taosRandomAllocFail(size_t size, const char* file, uint32_t line) {
if (taosGetTimestampSec() < startTime) {
return false;
}
@ -48,33 +48,33 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) {
return true;
}
static void* malloc_random(size_t size, const char* file, uint32_t line) {
return random_alloc_fail(size, file, line) ? NULL : malloc(size);
static void* taosRandmoMalloc(size_t size, const char* file, uint32_t line) {
return taosRandomAllocFail(size, file, line) ? NULL : malloc(size);
}
static void* calloc_random(size_t num, size_t size, const char* file, uint32_t line) {
return random_alloc_fail(num * size, file, line) ? NULL : calloc(num, size);
static void* taosRandomCalloc(size_t num, size_t size, const char* file, uint32_t line) {
return taosRandomAllocFail(num * size, file, line) ? NULL : calloc(num, size);
}
static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t line) {
return random_alloc_fail(size, file, line) ? NULL : realloc(ptr, size);
static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_t line) {
return taosRandomAllocFail(size, file, line) ? NULL : realloc(ptr, size);
}
static char* strdup_random(const char* str, const char* file, uint32_t line) {
static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) {
size_t len = strlen(str);
return random_alloc_fail(len + 1, file, line) ? NULL : taosStrdupImp(str);
return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str);
}
static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) {
static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) {
size_t len = strlen(str);
if (len > size) {
len = size;
}
return random_alloc_fail(len + 1, file, line) ? NULL : taosStrndupImp(str, len);
return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len);
}
static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
return random_alloc_fail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream);
static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream);
}
////////////////////////////////////////////////////////////////////////////////
@ -96,7 +96,7 @@ typedef struct SMemBlock {
static SMemBlock *blocks = NULL;
static uintptr_t lock = 0;
static void add_mem_block(SMemBlock* blk) {
static void taosAddMemBlock(SMemBlock* blk) {
blk->prev = NULL;
while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0);
blk->next = blocks;
@ -107,7 +107,7 @@ static void add_mem_block(SMemBlock* blk) {
atomic_store_ptr(&lock, 0);
}
static void remove_mem_block(SMemBlock* blk) {
static void taosRemoveMemBlock(SMemBlock* blk) {
while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0);
if (blocks == blk) {
@ -126,7 +126,7 @@ static void remove_mem_block(SMemBlock* blk) {
blk->next = NULL;
}
static void free_detect_leak(void* ptr, const char* file, uint32_t line) {
static void taosFreeDetectLeak(void* ptr, const char* file, uint32_t line) {
if (ptr == NULL) {
return;
}
@ -140,11 +140,11 @@ static void free_detect_leak(void* ptr, const char* file, uint32_t line) {
return;
}
remove_mem_block(blk);
taosRemoveMemBlock(blk);
free(blk);
}
static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) {
static void* taosMallocDetectLeak(size_t size, const char* file, uint32_t line) {
if (size == 0) {
return NULL;
}
@ -166,28 +166,28 @@ static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) {
blk->line = (uint16_t)line;
blk->magic = MEMBLK_MAGIC;
blk->size = size;
add_mem_block(blk);
taosAddMemBlock(blk);
return blk->data;
}
static void* calloc_detect_leak(size_t num, size_t size, const char* file, uint32_t line) {
static void* taosCallocDetectLeak(size_t num, size_t size, const char* file, uint32_t line) {
size *= num;
void* p = malloc_detect_leak(size, file, line);
void* p = taosMallocDetectLeak(size, file, line);
if (p != NULL) {
memset(p, 0, size);
}
return p;
}
static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint32_t line) {
static void* taosReallocDetectLeak(void* ptr, size_t size, const char* file, uint32_t line) {
if (size == 0) {
free_detect_leak(ptr, file, line);
taosFreeDetectLeak(ptr, file, line);
return NULL;
}
if (ptr == NULL) {
return malloc_detect_leak(size, file, line);
return taosMallocDetectLeak(size, file, line);
}
SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock);
@ -198,11 +198,11 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3
return realloc(ptr, size);
}
remove_mem_block(blk);
taosRemoveMemBlock(blk);
void* p = realloc(blk, size + sizeof(SMemBlock));
if (p == NULL) {
add_mem_block(blk);
taosAddMemBlock(blk);
return NULL;
}
@ -212,13 +212,13 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3
blk = (SMemBlock*)p;
blk->size = size;
add_mem_block(blk);
taosAddMemBlock(blk);
return blk->data;
}
static char* strdup_detect_leak(const char* str, const char* file, uint32_t line) {
static char* taosStrdupDetectLeak(const char* str, const char* file, uint32_t line) {
size_t len = strlen(str);
char *p = malloc_detect_leak(len + 1, file, line);
char *p = taosMallocDetectLeak(len + 1, file, line);
if (p != NULL) {
memcpy(p, str, len);
p[len] = 0;
@ -226,12 +226,12 @@ static char* strdup_detect_leak(const char* str, const char* file, uint32_t line
return p;
}
static char* strndup_detect_leak(const char* str, size_t size, const char* file, uint32_t line) {
static char* taosStrndupDetectLeak(const char* str, size_t size, const char* file, uint32_t line) {
size_t len = strlen(str);
if (len > size) {
len = size;
}
char *p = malloc_detect_leak(len + 1, file, line);
char *p = taosMallocDetectLeak(len + 1, file, line);
if (p != NULL) {
memcpy(p, str, len);
p[len] = 0;
@ -239,13 +239,13 @@ static char* strndup_detect_leak(const char* str, size_t size, const char* file,
return p;
}
static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
char* buf = NULL;
size_t bufSize = 0;
ssize_t size = taosGetlineImp(&buf, &bufSize, stream);
if (size != -1) {
if (*n < size + 1) {
void* p = realloc_detect_leak(*lineptr, size + 1, file, line);
void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line);
if (p == NULL) {
free(buf);
return -1;
@ -260,7 +260,7 @@ static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, cons
return size;
}
static void dump_memory_leak() {
static void taosDumpMemoryLeakImp() {
const char* hex = "0123456789ABCDEF";
const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)=";
size_t numOfBlk = 0, totalSize = 0;
@ -299,7 +299,7 @@ static void dump_memory_leak() {
fflush(fpAllocLog);
}
static void dump_memory_leak_on_sig(int sig) {
static void taosDumpMemoryLeakOnSig(int sig) {
fprintf(fpAllocLog, "signal %d received.\n", sig);
// restore default signal handler
@ -307,55 +307,55 @@ static void dump_memory_leak_on_sig(int sig) {
act.sa_handler = SIG_DFL;
sigaction(sig, &act, NULL);
dump_memory_leak();
taosDumpMemoryLeakImp();
}
////////////////////////////////////////////////////////////////////////////////
// interface functions
void* taos_malloc(size_t size, const char* file, uint32_t line) {
void* taosMallocMem(size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return malloc(size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
return malloc_random(size, file, line);
return taosRandmoMalloc(size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return malloc_detect_leak(size, file, line);
return taosMallocDetectLeak(size, file, line);
}
return malloc(size);
}
void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) {
void* taosCallocMem(size_t num, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return calloc(num, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
return calloc_random(num, size, file, line);
return taosRandomCalloc(num, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return calloc_detect_leak(num, size, file, line);
return taosCallocDetectLeak(num, size, file, line);
}
return calloc(num, size);
}
void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) {
void* taosReallocMem(void* ptr, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return realloc(ptr, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
return realloc_random(ptr, size, file, line);
return taosRandomRealloc(ptr, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return realloc_detect_leak(ptr, size, file, line);
return taosReallocDetectLeak(ptr, size, file, line);
}
return realloc(ptr, size);
}
void taos_free(void* ptr, const char* file, uint32_t line) {
void taosFreeMem(void* ptr, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return free(ptr);
@ -364,54 +364,54 @@ void taos_free(void* ptr, const char* file, uint32_t line) {
return free(ptr);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return free_detect_leak(ptr, file, line);
return taosFreeDetectLeak(ptr, file, line);
}
return free(ptr);
}
char* taos_strdup(const char* str, const char* file, uint32_t line) {
char* taosStrdupMem(const char* str, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosStrdupImp(str);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
return strdup_random(str, file, line);
return taosRandomStrdup(str, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return strdup_detect_leak(str, file, line);
return taosStrdupDetectLeak(str, file, line);
}
return taosStrdupImp(str);
}
char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) {
char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosStrndupImp(str, size);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
return strndup_random(str, size, file, line);
return taosRandomStrndup(str, size, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return strndup_detect_leak(str, size, file, line);
return taosStrndupDetectLeak(str, size, file, line);
}
return taosStrndupImp(str, size);
}
ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) {
switch (allocMode) {
case TAOS_ALLOC_MODE_DEFAULT:
return taosGetlineImp(lineptr, n, stream);
case TAOS_ALLOC_MODE_RANDOM_FAIL:
return getline_random(lineptr, n, stream, file, line);
return taosRandomGetline(lineptr, n, stream, file, line);
case TAOS_ALLOC_MODE_DETECT_LEAK:
return getline_detect_leak(lineptr, n, stream, file, line);
return taosGetlineDetectLeak(lineptr, n, stream, file, line);
}
return taosGetlineImp(lineptr, n, stream);
}
static void close_alloc_log() {
static void taosCloseAllocLog() {
if (fpAllocLog != NULL) {
if (fpAllocLog != stdout) {
fclose(fpAllocLog);
@ -432,7 +432,7 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
if (path == NULL || path[0] == 0) {
fpAllocLog = stdout;
} else if ((fpAllocLog = fopen(path, "w")) != NULL) {
atexit(close_alloc_log);
atexit(taosCloseAllocLog);
} else {
printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno);
return;
@ -446,10 +446,10 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
}
if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) {
atexit(dump_memory_leak);
atexit(taosDumpMemoryLeakImp);
struct sigaction act = {0};
act.sa_handler = dump_memory_leak_on_sig;
act.sa_handler = taosDumpMemoryLeakOnSig;
sigaction(SIGFPE, &act, NULL);
sigaction(SIGSEGV, &act, NULL);
sigaction(SIGILL, &act, NULL);
@ -457,8 +457,8 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) {
}
void taosDumpMemoryLeak() {
dump_memory_leak();
close_alloc_log();
taosDumpMemoryLeakImp();
taosCloseAllocLog();
}
#else // 'TAOS_MEM_CHECK' not defined

14
src/os/src/detail/osSysinfo.c
View File

@ -61,7 +61,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
size_t len;
char * line = NULL;
while (!feof(fp)) {
taosTFree(line);
tfree(line);
len = 0;
getline(&line, &len, fp);
if (line == NULL) {
@ -83,7 +83,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
sscanf(line, "%s %" PRId64, tmp, &memKB);
*memoryUsedMB = (float)((double)memKB / 1024);
taosTFree(line);
tfree(line);
fclose(fp);
return true;
}
@ -107,7 +107,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
char cpu[10] = {0};
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle);
taosTFree(line);
tfree(line);
fclose(fp);
return true;
}
@ -136,7 +136,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
}
}
taosTFree(line);
tfree(line);
fclose(fp);
return true;
}
@ -377,7 +377,7 @@ static bool taosGetCardInfo(int64_t *bytes) {
*bytes += (rbytes + tbytes);
}
taosTFree(line);
tfree(line);
fclose(fp);
return true;
@ -432,7 +432,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
int readIndex = 0;
while (!feof(fp)) {
taosTFree(line);
tfree(line);
len = 0;
getline(&line, &len, fp);
if (line == NULL) {
@ -450,7 +450,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
if (readIndex >= 2) break;
}
taosTFree(line);
tfree(line);
fclose(fp);
if (readIndex < 2) {

4
src/plugins/http/inc/httpQueue.h
View File

@ -22,9 +22,11 @@ extern "C" {
#include <stdint.h>
typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows);
bool httpInitResultQueue();
void httpCleanupResultQueue();
void httpDispatchToResultQueue();
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp);
#ifdef __cplusplus
}

2
src/plugins/http/src/httpContext.c
View File

@ -63,7 +63,7 @@ static void httpDestroyContext(void *data) {
pContext->parser = NULL;
}
taosTFree(pContext);
tfree(pContext);
}
bool httpInitContexts() {

28
src/plugins/http/src/httpQueue.c
View File

@ -25,6 +25,7 @@
#include "httpResp.h"
#include "httpAuth.h"
#include "httpSession.h"
#include "httpQueue.h"
typedef struct {
pthread_t thread;
@ -37,42 +38,45 @@ typedef struct {
} SHttpWorkerPool;
typedef struct {
void *param;
void *result;
int32_t numOfRows;
void (*fp)(void *param, void *result, int32_t numOfRows);
void * param;
void * result;
int32_t code;
int32_t rows;
FHttpResultFp fp;
} SHttpResult;
static SHttpWorkerPool tsHttpPool;
static taos_qset tsHttpQset;
static taos_queue tsHttpQueue;
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) {
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) {
if (tsHttpQueue != NULL) {
SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult));
pMsg->param = param;
pMsg->result = result;
pMsg->numOfRows = numOfRows;
pMsg->code = code;
pMsg->rows = rows;
pMsg->fp = fp;
taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg);
} else {
(*fp)(param, result, numOfRows);
(*fp)(param, result, code, rows);
}
}
static void *httpProcessResultQueue(void *param) {
SHttpResult *pMsg;
int32_t type;
void *unUsed;
int32_t type;
void * unUsed;
while (1) {
if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) {
httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset);
break;
}
httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result);
(*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows);
httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result,
pMsg->code, pMsg->rows);
(*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows);
taosFreeQitem(pMsg);
}

69
src/plugins/http/src/httpSql.c
View File

@ -29,9 +29,9 @@
void httpProcessMultiSql(HttpContext *pContext);
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows);
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) {
void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
@ -43,7 +43,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
bool isContinue = false;
if (numOfRows > 0) {
if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows);
}
@ -58,9 +58,9 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, numOfRows, sql);
if (numOfRows < 0) {
if (code < 0) {
httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(numOfRows), sql);
pContext->user, multiCmds->pos, tstrerror(code), sql);
}
taos_free_result(result);
@ -73,15 +73,15 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
}
}
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
httpDispatchToResultQueue(param, result, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
int32_t code = taos_errno(result);
httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp);
}
void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
code = taos_errno(result);
HttpSqlCmds *multiCmds = pContext->multiCmds;
HttpEncodeMethod *encode = pContext->encodeMethod;
@ -94,7 +94,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
return;
}
if (code < 0) {
if (code != TSDB_CODE_SUCCESS) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) {
singleCmd->code = code;
httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd,
@ -119,7 +119,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
int affectRows = taos_affected_rows(result);
int32_t affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, affectRows, sql);
@ -156,8 +156,10 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
}
}
void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
httpDispatchToResultQueue(param, result, unUsedCode, httpProcessMultiSqlCallBackImp);
void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
int32_t code = taos_errno(result);
int32_t affectRows = taos_affected_rows(result);
httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp);
}
void httpProcessMultiSql(HttpContext *pContext) {
@ -202,9 +204,9 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) {
httpProcessMultiSql(pContext);
}
void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows);
void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows);
void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) {
void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
@ -212,7 +214,7 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
bool isContinue = false;
if (numOfRows > 0) {
if (code == TSDB_CODE_SUCCESS && numOfRows > 0) {
if (encode->buildQueryJsonFp) {
isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows);
}
@ -227,9 +229,9 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
if (numOfRows < 0) {
if (code < 0) {
httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user,
tstrerror(numOfRows));
tstrerror(code));
}
taos_free_result(result);
@ -242,30 +244,30 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
}
}
void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
httpDispatchToResultQueue(param, result, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) {
int32_t code = taos_errno(result);
httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp);
}
void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCode) {
void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) {
HttpContext *pContext = (HttpContext *)param;
if (pContext == NULL) return;
int32_t code = taos_errno(result);
HttpEncodeMethod *encode = pContext->encodeMethod;
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd,
pContext->user, tstrerror(code), (SSqlObj *)result);
pContext->user, tstrerror(code), result);
return;
}
if (code < 0) {
if (code != TSDB_CODE_SUCCESS) {
SSqlObj *pObj = (SSqlObj *)result;
if (code == TSDB_CODE_TSC_INVALID_SQL) {
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext,
pContext->fd, pContext->user, tstrerror(code), pObj, pObj->cmd.payload);
httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload);
terrno = code;
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd,
pContext->user, tstrerror(code), pObj, taos_errstr(pObj));
httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj));
} else {
httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd,
pContext->user, tstrerror(code), pObj);
@ -278,7 +280,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
bool isUpdate = tscIsUpdateQuery(result);
if (isUpdate) {
// not select or show commands
int affectRows = taos_affected_rows(result);
int32_t affectRows = taos_affected_rows(result);
assert(affectRows == affectRowsInput);
httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd,
pContext->user, affectRows, result);
@ -308,8 +311,10 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
}
}
void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
httpDispatchToResultQueue(param, result, unUsedCode, httpProcessSingleSqlCallBackImp);
void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) {
int32_t code = taos_errno(result);
int32_t affectRows = taos_affected_rows(result);
httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp);
}
void httpProcessSingleSqlCmd(HttpContext *pContext) {
@ -373,7 +378,7 @@ void httpExecCmd(HttpContext *pContext) {
}
}
void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) {
HttpContext *pContext = param;
taos_free_result(result);

2
src/plugins/http/src/httpSystem.c
View File

@ -107,7 +107,7 @@ void httpCleanUpSystem() {
httpCleanupResultQueue();
pthread_mutex_destroy(&tsHttpServer.serverMutex);
taosTFree(tsHttpServer.pThreads);
tfree(tsHttpServer.pThreads);
tsHttpServer.pThreads = NULL;
tsHttpServer.status = HTTP_SERVER_CLOSED;

100
src/query/src/qExecutor.c
View File

@ -1019,7 +1019,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
bool hasTimeWindow = false;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
taosTFree(sasArray);
tfree(sasArray);
return;
}
@ -1081,10 +1081,10 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
continue;
}
taosTFree(sasArray[i].data);
tfree(sasArray[i].data);
}
taosTFree(sasArray);
tfree(sasArray);
}
static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) {
@ -1410,7 +1410,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
continue;
}
taosTFree(sasArray[i].data);
tfree(sasArray[i].data);
}
free(sasArray);
@ -1596,7 +1596,7 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
p->tagInfo.numOfTagCols = num;
p->tagInfo.tagsLen = tagLen;
} else {
taosTFree(pTagCtx);
tfree(pTagCtx);
}
}
@ -1706,7 +1706,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
return TSDB_CODE_SUCCESS;
_clean:
taosTFree(pRuntimeEnv->pCtx);
tfree(pRuntimeEnv->pCtx);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
@ -1744,10 +1744,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
}
tVariantDestroy(&pCtx->tag);
taosTFree(pCtx->tagInfo.pTagCtxList);
tfree(pCtx->tagInfo.pTagCtxList);
}
taosTFree(pRuntimeEnv->pCtx);
tfree(pRuntimeEnv->pCtx);
}
pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo);
@ -1756,8 +1756,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
doFreeQueryHandle(pQInfo);
pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
taosTFree(pRuntimeEnv->keyBuf);
taosTFree(pRuntimeEnv->rowCellInfoOffset);
tfree(pRuntimeEnv->keyBuf);
tfree(pRuntimeEnv->rowCellInfoOffset);
taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
pRuntimeEnv->pResultRowHashTable = NULL;
@ -2985,8 +2985,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
STableQueryInfo **pTableList = malloc(POINTER_BYTES * size);
if (pTableList == NULL || posList == NULL) {
taosTFree(posList);
taosTFree(pTableList);
tfree(posList);
tfree(pTableList);
qError("QInfo:%p failed alloc memory", pQInfo);
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -3010,12 +3010,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
// there is no data in current group
if (numOfTables == 0) {
taosTFree(posList);
taosTFree(pTableList);
tfree(posList);
tfree(pTableList);
return 0;
} else if (numOfTables == 1) { // no need to merge results since only one table in each group
taosTFree(posList);
taosTFree(pTableList);
tfree(posList);
tfree(pTableList);
SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
@ -3045,9 +3045,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
if (IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:%p it is already killed, abort", pQInfo);
taosTFree(pTableList);
taosTFree(posList);
taosTFree(pTree);
tfree(pTableList);
tfree(posList);
tfree(pTree);
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@ -3118,9 +3118,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) {
qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo);
taosTFree(pTree);
taosTFree(pTableList);
taosTFree(posList);
tfree(pTree);
tfree(pTableList);
tfree(posList);
return -1;
}
}
@ -3133,12 +3133,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt);
taosTFree(pTableList);
taosTFree(posList);
taosTFree(pTree);
tfree(pTableList);
tfree(posList);
tfree(pTree);
// taosTFree(pResultInfo);
// taosTFree(buf);
// tfree(pResultInfo);
// tfree(buf);
return pQInfo->groupResInfo.numOfDataPages;
}
@ -5880,13 +5880,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
return TSDB_CODE_SUCCESS;
_cleanup:
taosTFree(*pExpr);
tfree(*pExpr);
taosArrayDestroy(*pTableIdList);
*pTableIdList = NULL;
taosTFree(*tbnameCond);
taosTFree(*groupbyCols);
taosTFree(*tagCols);
taosTFree(*tagCond);
tfree(*tbnameCond);
tfree(*groupbyCols);
tfree(*tagCols);
tfree(*tagCond);
return code;
}
@ -5937,7 +5937,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg);
if (code != TSDB_CODE_SUCCESS) {
taosTFree(pExprs);
tfree(pExprs);
return code;
}
@ -5976,7 +5976,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64;
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes,
&pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
taosTFree(pExprs);
tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@ -6352,7 +6352,7 @@ _cleanup_query:
free(pGroupbyExpr);
}
taosTFree(pTagCols);
tfree(pTagCols);
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExprInfo = &pExprs[i];
if (pExprInfo->pExpr != NULL) {
@ -6360,7 +6360,7 @@ _cleanup_query:
}
}
taosTFree(pExprs);
tfree(pExprs);
_cleanup:
freeQInfo(pQInfo);
@ -6479,19 +6479,19 @@ static void freeQInfo(SQInfo *pQInfo) {
if (pQuery != NULL) {
if (pQuery->sdata != NULL) {
for (int32_t col = 0; col < pQuery->numOfOutput; ++col) {
taosTFree(pQuery->sdata[col]);
tfree(pQuery->sdata[col]);
}
taosTFree(pQuery->sdata);
tfree(pQuery->sdata);
}
if (pQuery->fillVal != NULL) {
taosTFree(pQuery->fillVal);
tfree(pQuery->fillVal);
}
for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i];
if (pColFilter->numOfFilters > 0) {
taosTFree(pColFilter->pFilters);
tfree(pColFilter->pFilters);
}
}
@ -6504,31 +6504,31 @@ static void freeQInfo(SQInfo *pQInfo) {
}
}
taosTFree(pQuery->pSelectExpr);
tfree(pQuery->pSelectExpr);
}
if (pQuery->pGroupbyExpr != NULL) {
taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo);
taosTFree(pQuery->pGroupbyExpr);
tfree(pQuery->pGroupbyExpr);
}
taosTFree(pQuery->tagColList);
taosTFree(pQuery->pFilterInfo);
tfree(pQuery->tagColList);
tfree(pQuery->pFilterInfo);
if (pQuery->colList != NULL) {
for (int32_t i = 0; i < pQuery->numOfCols; i++) {
SColumnInfo *column = pQuery->colList + i;
freeColumnFilterInfo(column->filters, column->numOfFilters);
}
taosTFree(pQuery->colList);
tfree(pQuery->colList);
}
taosTFree(pQuery);
tfree(pQuery);
}
doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
taosTFree(pQInfo->pBuf);
tfree(pQInfo->pBuf);
tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
taosArrayDestroy(pQInfo->arrTableIdInfo);
@ -6536,7 +6536,7 @@ static void freeQInfo(SQInfo *pQInfo) {
qDebug("QInfo:%p QInfo is freed", pQInfo);
taosTFree(pQInfo);
tfree(pQInfo);
}
static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) {
@ -7231,9 +7231,9 @@ void qCleanupQueryMgmt(void* pQMgmt) {
taosCacheCleanup(pqinfoPool);
pthread_mutex_destroy(&pQueryMgmt->lock);
taosTFree(pQueryMgmt);
tfree(pQueryMgmt);
qDebug("vgId:%d queryMgmt cleanup completed", vgId);
qDebug("vgId:%d, queryMgmt cleanup completed", vgId);
}
void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {

16
src/query/src/qExtbuffer.c
View File

@ -64,7 +64,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
// release flush out info link
SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta;
if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) {
taosTFree(pFileMeta->flushoutData.pFlushoutInfo);
tfree(pFileMeta->flushoutData.pFlushoutInfo);
}
// release all in-memory buffer pages
@ -72,7 +72,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
while (pFilePages != NULL) {
tFilePagesItem *pTmp = pFilePages;
pFilePages = pFilePages->pNext;
taosTFree(pTmp);
tfree(pTmp);
}
// close temp file
@ -87,8 +87,8 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) {
destroyColumnModel(pMemBuffer->pColumnModel);
taosTFree(pMemBuffer->path);
taosTFree(pMemBuffer);
tfree(pMemBuffer->path);
tfree(pMemBuffer);
return NULL;
}
@ -275,7 +275,7 @@ int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) {
tFilePagesItem *ptmp = first;
first = first->pNext;
taosTFree(ptmp); // release all data in memory buffer
tfree(ptmp); // release all data in memory buffer
}
fflush(pMemBuffer->file); // flush to disk
@ -300,7 +300,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) {
while (first != NULL) {
tFilePagesItem *ptmp = first;
first = first->pNext;
taosTFree(ptmp);
tfree(ptmp);
}
pMemBuffer->fileMeta.numOfElemsInFile = 0;
@ -802,7 +802,7 @@ void destroyColumnModel(SColumnModel *pModel) {
return;
}
taosTFree(pModel);
tfree(pModel);
}
static void printBinaryData(char *data, int32_t len) {
@ -1087,5 +1087,5 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) {
}
destroyColumnModel(pDesc->pColumnModel);
taosTFree(pDesc);
tfree(pDesc);
}

16
src/query/src/qFill.c
View File

@ -96,18 +96,18 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
return NULL;
}
taosTFree(pFillInfo->prevValues);
taosTFree(pFillInfo->nextValues);
taosTFree(pFillInfo->pTags);
tfree(pFillInfo->prevValues);
tfree(pFillInfo->nextValues);
tfree(pFillInfo->pTags);
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
taosTFree(pFillInfo->pData[i]);
tfree(pFillInfo->pData[i]);
}
taosTFree(pFillInfo->pData);
taosTFree(pFillInfo->pFillCol);
tfree(pFillInfo->pData);
tfree(pFillInfo->pFillCol);
taosTFree(pFillInfo);
tfree(pFillInfo);
return NULL;
}
@ -496,7 +496,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
pFillInfo->numOfRows = 0;
/* the raw data block is exhausted, next value does not exists */
taosTFree(*nextValues);
tfree(*nextValues);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;

8
src/query/src/qParserImpl.c
View File

@ -571,7 +571,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) {
doDestroyQuerySql(pQuerySql);
}
taosTFree(pClause->pClause);
tfree(pClause->pClause);
}
SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName,
@ -641,12 +641,12 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns);
tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals);
taosTFree(pInfo->pCreateTableInfo);
tfree(pInfo->pCreateTableInfo);
} else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
tVariantListDestroy(pInfo->pAlterInfo->varList);
tFieldListDestroy(pInfo->pAlterInfo->pAddColumns);
taosTFree(pInfo->pAlterInfo);
tfree(pInfo->pAlterInfo);
} else {
if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) {
free(pInfo->pDCLInfo->a);
@ -656,7 +656,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep);
}
taosTFree(pInfo->pDCLInfo);
tfree(pInfo->pDCLInfo);
}
}

6
src/query/src/qPercentile.c
View File

@ -361,8 +361,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) {
}
destroyResultBuf(pBucket->pBuffer);
taosTFree(pBucket->pSlots);
taosTFree(pBucket);
tfree(pBucket->pSlots);
tfree(pBucket);
}
void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) {
@ -680,7 +680,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
}
double val = (1 - fraction) * td + fraction * nd;
taosTFree(buffer);
tfree(buffer);
return val;
} else { // incur a second round bucket split

12
src/query/src/qResultbuf.c
View File

@ -267,7 +267,7 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
assert(d->pn == pn);
d->pn = NULL;
taosTFree(pn);
tfree(pn);
bufPage = flushPageToDisk(pResultBuf, d);
}
@ -418,7 +418,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
}
unlink(pResultBuf->path);
taosTFree(pResultBuf->path);
tfree(pResultBuf->path);
SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
while(taosHashIterNext(iter)) {
@ -426,8 +426,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
size_t n = taosArrayGetSize(*p);
for(int32_t i = 0; i < n; ++i) {
SPageInfo* pi = taosArrayGetP(*p, i);
taosTFree(pi->pData);
taosTFree(pi);
tfree(pi->pData);
tfree(pi);
}
taosArrayDestroy(*p);
@ -440,8 +440,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
taosHashCleanup(pResultBuf->groupSet);
taosHashCleanup(pResultBuf->all);
taosTFree(pResultBuf->assistBuf);
taosTFree(pResultBuf);
tfree(pResultBuf->assistBuf);
tfree(pResultBuf);
}
SPageInfo* getLastPageInfo(SIDList pList) {

8
src/query/src/qTsbuf.c
View File

@ -142,11 +142,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
return NULL;
}
taosTFree(pTSBuf->assistBuf);
taosTFree(pTSBuf->tsData.rawBuf);
tfree(pTSBuf->assistBuf);
tfree(pTSBuf->tsData.rawBuf);
taosTFree(pTSBuf->pData);
taosTFree(pTSBuf->block.payload);
tfree(pTSBuf->pData);
tfree(pTSBuf->block.payload);
fclose(pTSBuf->f);

6
src/query/src/qUtil.c
View File

@ -57,7 +57,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
return;
}
taosTFree(pWindowResInfo->pResult);
tfree(pWindowResInfo->pResult);
}
void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) {
@ -358,11 +358,11 @@ void* destroyResultRowPool(SResultRowPool* p) {
size_t size = taosArrayGetSize(p->pData);
for(int32_t i = 0; i < size; ++i) {
void** ptr = taosArrayGet(p->pData, i);
taosTFree(*ptr);
tfree(*ptr);
}
taosArrayDestroy(p->pData);
taosTFree(p);
tfree(p);
return NULL;
}

6
src/rpc/src/rpcCache.c
View File

@ -101,9 +101,9 @@ void rpcCloseConnCache(void *handle) {
if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool);
taosTFree(pCache->connHashList);
taosTFree(pCache->count);
taosTFree(pCache->lockedBy);
tfree(pCache->connHashList);
tfree(pCache->count);
tfree(pCache->lockedBy);
pthread_mutex_unlock(&pCache->mutex);

4
src/rpc/src/rpcMain.c
View File

@ -1620,10 +1620,10 @@ static void rpcDecRef(SRpcInfo *pRpc)
taosTmrCleanUp(pRpc->tmrCtrl);
taosIdPoolCleanUp(pRpc->idPool);
taosTFree(pRpc->connList);
tfree(pRpc->connList);
pthread_mutex_destroy(&pRpc->mutex);
tDebug("%s rpc resources are released", pRpc->label);
taosTFree(pRpc);
tfree(pRpc);
atomic_sub_fetch_32(&tsRpcNum, 1);
}

10
src/rpc/src/rpcTcp.c
View File

@ -236,8 +236,8 @@ void taosCleanUpTcpServer(void *handle) {
tDebug("%s TCP server is cleaned up", pServerObj->label);
taosTFree(pServerObj->pThreadObj);
taosTFree(pServerObj);
tfree(pServerObj->pThreadObj);
tfree(pServerObj);
}
static void *taosAcceptTcpConnection(void *arg) {
@ -535,7 +535,7 @@ static void *taosProcessTcpData(void *param) {
pthread_mutex_destroy(&(pThreadObj->mutex));
tDebug("%s TCP thread exits ...", pThreadObj->label);
taosTFree(pThreadObj);
tfree(pThreadObj);
return NULL;
}
@ -556,7 +556,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
event.events = EPOLLIN | EPOLLRDHUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
taosTFree(pFdObj);
tfree(pFdObj);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
@ -609,5 +609,5 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
taosTFree(pFdObj);
tfree(pFdObj);
}

4
src/rpc/src/rpcUdp.c
View File

@ -147,7 +147,7 @@ void taosStopUdpConnection(void *handle) {
if (taosCheckPthreadValid(pConn->thread)) {
pthread_join(pConn->thread, NULL);
}
taosTFree(pConn->buffer);
tfree(pConn->buffer);
// tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
}
@ -166,7 +166,7 @@ void taosCleanUpUdpConnection(void *handle) {
}
tDebug("%s UDP is cleaned up", pSet->label);
taosTFree(pSet);
tfree(pSet);
}
void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {

48
src/sync/inc/syncInt.h
View File

@ -35,6 +35,8 @@ extern "C" {
#define TAOS_SMSG_SYNC_MUST 6
#define TAOS_SMSG_STATUS 7
#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
#define nodeRole pNode->peerInfo[pNode->selfIndex]->role
#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
@ -89,11 +91,11 @@ typedef struct {
#pragma pack(pop)
typedef struct {
char *buffer;
int bufferSize;
char *offset;
int forwards;
int code;
char * buffer;
int32_t bufferSize;
char * offset;
int32_t forwards;
int32_t code;
} SRecvBuffer;
typedef struct {
@ -107,10 +109,10 @@ typedef struct {
} SFwdInfo;
typedef struct {
int first;
int last;
int fwds; // number of forwards
SFwdInfo fwdInfo[];
int32_t first;
int32_t last;
int32_t fwds; // number of forwards
SFwdInfo fwdInfo[];
} SSyncFwds;
typedef struct SsyncPeer {
@ -123,15 +125,15 @@ typedef struct SsyncPeer {
int8_t sstatus; // sync status
uint64_t version;
uint64_t sversion; // track the peer version in retrieve process
int syncFd;
int peerFd; // forward FD
int numOfRetrieves; // number of retrieves tried
int fileChanged; // a flag to indicate file is changed during retrieving process
int32_t syncFd;
int32_t peerFd; // forward FD
int32_t numOfRetrieves; // number of retrieves tried
int32_t fileChanged; // a flag to indicate file is changed during retrieving process
void * timer;
void * pConn;
int notifyFd;
int watchNum;
int * watchFd;
int32_t notifyFd;
int32_t watchNum;
int32_t *watchFd;
int8_t refCount; // reference count
struct SSyncNode *pSyncNode;
} SSyncPeer;
@ -161,16 +163,16 @@ typedef struct SSyncNode {
} SSyncNode;
// sync module global
extern int tsSyncNum;
extern char tsNodeFqdn[TSDB_FQDN_LEN];
extern int32_t tsSyncNum;
extern char tsNodeFqdn[TSDB_FQDN_LEN];
void *syncRetrieveData(void *param);
void *syncRestoreData(void *param);
int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
void syncRestartConnection(SSyncPeer *pPeer);
void syncBroadcastStatus(SSyncNode *pNode);
void syncAddPeerRef(SSyncPeer *pPeer);
int syncDecPeerRef(SSyncPeer *pPeer);
int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
void syncRestartConnection(SSyncPeer *pPeer);
void syncBroadcastStatus(SSyncNode *pNode);
void syncAddPeerRef(SSyncPeer *pPeer);
int32_t syncDecPeerRef(SSyncPeer *pPeer);
#ifdef __cplusplus
}

22
src/sync/inc/taosTcpPool.h
View File

@ -20,23 +20,23 @@
extern "C" {
#endif
typedef void* ttpool_h;
typedef void* tthread_h;
typedef void *ttpool_h;
typedef void *tthread_h;
typedef struct {
int numOfThreads;
int32_t numOfThreads;
uint32_t serverIp;
int16_t port;
int bufferSize;
void (*processBrokenLink)(void *ahandle);
int (*processIncomingMsg)(void *ahandle, void *buffer);
void (*processIncomingConn)(int fd, uint32_t ip);
int32_t bufferSize;
void (*processBrokenLink)(void *ahandle);
int32_t (*processIncomingMsg)(void *ahandle, void *buffer);
void (*processIncomingConn)(int32_t fd, uint32_t ip);
} SPoolInfo;
ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo);
void taosCloseTcpThreadPool(ttpool_h);
void *taosAllocateTcpConn(void *, void *ahandle, int connFd);
void taosFreeTcpConn(void *);
ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo);
void taosCloseTcpThreadPool(ttpool_h);
void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd);
void taosFreeTcpConn(void *);
#ifdef __cplusplus
}

238
src/sync/src/syncMain.c
View File

@ -31,38 +31,38 @@
#include "syncInt.h"
// global configurable
int tsMaxSyncNum = 2;
int tsSyncTcpThreads = 2;
int tsMaxWatchFiles = 500;
int tsMaxFwdInfo = 200;
int tsSyncTimer = 1;
int32_t tsMaxSyncNum = 2;
int32_t tsSyncTcpThreads = 2;
int32_t tsMaxWatchFiles = 500;
int32_t tsMaxFwdInfo = 200;
int32_t tsSyncTimer = 1;
// module global, not configurable
int tsSyncNum; // number of sync in process in whole system
char tsNodeFqdn[TSDB_FQDN_LEN];
int32_t tsSyncNum; // number of sync in process in whole system
char tsNodeFqdn[TSDB_FQDN_LEN];
static ttpool_h tsTcpPool;
static void * syncTmrCtrl = NULL;
static void * vgIdHash;
static int tsSyncRefId = -1;
static void * tsSyncTmrCtrl = NULL;
static void * tsVgIdHash;
static int32_t tsSyncRefId = -1;
// local functions
static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
static void syncRecoverFromMaster(SSyncPeer *pPeer);
static void syncCheckPeerConnection(void *param, void *tmrId);
static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
static void syncProcessBrokenLink(void *param);
static int syncProcessPeerMsg(void *param, void *buffer);
static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp);
static void syncRemovePeer(SSyncPeer *pPeer);
static void syncAddArbitrator(SSyncNode *pNode);
static void syncFreeNode(void *);
static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
static void syncMonitorFwdInfos(void *param, void *tmrId);
static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
static void syncRestartPeer(SSyncPeer *pPeer);
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtyp);
static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
static void syncRecoverFromMaster(SSyncPeer *pPeer);
static void syncCheckPeerConnection(void *param, void *tmrId);
static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
static void syncProcessBrokenLink(void *param);
static int32_t syncProcessPeerMsg(void *param, void *buffer);
static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
static void syncRemovePeer(SSyncPeer *pPeer);
static void syncAddArbitrator(SSyncNode *pNode);
static void syncFreeNode(void *);
static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
static void syncMonitorFwdInfos(void *param, void *tmrId);
static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
static void syncRestartPeer(SSyncPeer *pPeer);
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp);
static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
char* syncRole[] = {
@ -79,7 +79,7 @@ int32_t syncInit() {
info.numOfThreads = tsSyncTcpThreads;
info.serverIp = 0;
info.port = tsSyncPort;
info.bufferSize = 640000;
info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = syncProcessBrokenLink;
info.processIncomingMsg = syncProcessPeerMsg;
info.processIncomingConn = syncProcessIncommingConnection;
@ -90,21 +90,21 @@ int32_t syncInit() {
return -1;
}
syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
if (syncTmrCtrl == NULL) {
tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
if (tsSyncTmrCtrl == NULL) {
sError("failed to init tmrCtrl");
taosCloseTcpThreadPool(tsTcpPool);
tsTcpPool = NULL;
return -1;
}
vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
if (vgIdHash == NULL) {
sError("failed to init vgIdHash");
taosTmrCleanUp(syncTmrCtrl);
tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
if (tsVgIdHash == NULL) {
sError("failed to init tsVgIdHash");
taosTmrCleanUp(tsSyncTmrCtrl);
taosCloseTcpThreadPool(tsTcpPool);
tsTcpPool = NULL;
syncTmrCtrl = NULL;
tsSyncTmrCtrl = NULL;
return -1;
}
@ -126,14 +126,14 @@ void syncCleanUp() {
tsTcpPool = NULL;
}
if (syncTmrCtrl) {
taosTmrCleanUp(syncTmrCtrl);
syncTmrCtrl = NULL;
if (tsSyncTmrCtrl) {
taosTmrCleanUp(tsSyncTmrCtrl);
tsSyncTmrCtrl = NULL;
}
if (vgIdHash) {
taosHashCleanup(vgIdHash);
vgIdHash = NULL;
if (tsVgIdHash) {
taosHashCleanup(tsVgIdHash);
tsVgIdHash = NULL;
}
taosCloseRef(tsSyncRefId);
@ -176,7 +176,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
return -1;
}
for (int i = 0; i < pCfg->replica; ++i) {
for (int32_t i = 0; i < pCfg->replica; ++i) {
const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) {
@ -204,7 +204,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
return -1;
}
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
if (pNode->pFwdTimer == NULL) {
sError("vgId:%d, failed to allocate timer", pNode->vgId);
syncStop(pNode->rid);
@ -212,7 +212,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
}
syncAddArbitrator(pNode);
taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
if (pNode->notifyRole) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
@ -231,10 +231,10 @@ void syncStop(int64_t rid) {
pthread_mutex_lock(&(pNode->mutex));
if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
if (pPeer) syncRemovePeer(pPeer);
}
@ -249,7 +249,7 @@ void syncStop(int64_t rid) {
}
int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
int i, j;
int32_t i, j;
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
@ -321,7 +321,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
return 0;
}
int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int qtype) {
int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) {
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return 0;
@ -348,11 +348,11 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
pFwdRsp->version = version;
pFwdRsp->code = code;
int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
int retLen = write(pPeer->peerFd, msg, msgLen);
int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
int32_t retLen = write(pPeer->peerFd, msg, msgLen);
if (retLen == msgLen) {
sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
sDebug("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version);
} else {
sDebug("%s, failed to send forward ack, restart", pPeer->id);
syncRestartConnection(pPeer);
@ -377,7 +377,7 @@ void syncRecover(int64_t rid) {
pthread_mutex_lock(&(pNode->mutex));
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = (SSyncPeer *)pNode->peerInfo[i];
if (pPeer->peerFd >= 0) {
syncRestartConnection(pPeer);
@ -389,12 +389,12 @@ void syncRecover(int64_t rid) {
taosReleaseRef(tsSyncRefId, rid);
}
int syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return -1;
pNodesRole->selfIndex = pNode->selfIndex;
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
pNodesRole->role[i] = pNode->peerInfo[i]->role;
}
@ -416,7 +416,7 @@ static void syncAddArbitrator(SSyncNode *pNode) {
SNodeInfo nodeInfo;
nodeInfo.nodeId = 0;
int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
int32_t ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
if (-1 == ret) {
nodeInfo.nodePort = tsArbitratorPort;
}
@ -437,20 +437,20 @@ static void syncFreeNode(void *param) {
SSyncNode *pNode = param;
pthread_mutex_destroy(&pNode->mutex);
taosTFree(pNode->pRecv);
taosTFree(pNode->pSyncFwds);
taosTFree(pNode);
tfree(pNode->pRecv);
tfree(pNode->pSyncFwds);
tfree(pNode);
}
void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); }
int syncDecPeerRef(SSyncPeer *pPeer) {
int32_t syncDecPeerRef(SSyncPeer *pPeer) {
if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
sDebug("%s, resource is freed", pPeer->id);
taosTFree(pPeer->watchFd);
taosTFree(pPeer);
tfree(pPeer->watchFd);
tfree(pPeer);
return 0;
}
@ -486,7 +486,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->ip = ip;
pPeer->port = pInfo->nodePort;
pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
pPeer->peerFd = -1;
pPeer->syncFd = -1;
@ -495,12 +495,12 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->refCount = 1;
sInfo("%s, it is configured", pPeer->id);
int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs;
sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs);
taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
taosAcquireRef(tsSyncRefId, pNode->rid);
@ -510,7 +510,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
void syncBroadcastStatus(SSyncNode *pNode) {
SSyncPeer *pPeer;
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == pNode->selfIndex) continue;
pPeer = pNode->peerInfo[i];
syncSendPeersStatusMsgToPeer(pPeer, 1);
@ -518,7 +518,7 @@ void syncBroadcastStatus(SSyncNode *pNode) {
}
static void syncResetFlowCtrl(SSyncNode *pNode) {
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pNode->peerInfo[i]->numOfRetrieves = 0;
}
@ -529,13 +529,13 @@ static void syncResetFlowCtrl(SSyncNode *pNode) {
static void syncChooseMaster(SSyncNode *pNode) {
SSyncPeer *pPeer;
int onlineNum = 0;
int index = -1;
int replica = pNode->replica;
int32_t onlineNum = 0;
int32_t index = -1;
int32_t replica = pNode->replica;
sDebug("vgId:%d, choose master", pNode->vgId);
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
}
@ -544,7 +544,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (onlineNum == pNode->replica) {
// if all peers are online, peer with highest version shall be master
index = 0;
for (int i = 1; i < pNode->replica; ++i) {
for (int32_t i = 1; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) {
index = i;
}
@ -560,7 +560,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (index < 0 && onlineNum > replica / 2.0) {
// over half of nodes are online
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
// slave with highest version shall be master
pPeer = pNode->peerInfo[i];
if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
@ -587,11 +587,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
}
static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
int onlineNum = 0;
int index = -1;
int replica = pNode->replica;
int32_t onlineNum = 0;
int32_t index = -1;
int32_t replica = pNode->replica;
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
onlineNum++;
}
@ -612,7 +612,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
}
} else {
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
SSyncPeer *pTemp = pNode->peerInfo[i];
if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue;
if (index < 0) {
@ -631,9 +631,9 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
return pMaster;
}
static int syncValidateMaster(SSyncPeer *pPeer) {
static int32_t syncValidateMaster(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
int code = 0;
int32_t code = 0;
if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id);
@ -641,7 +641,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
code = -1;
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == pNode->selfIndex) continue;
syncRestartPeer(pNode->peerInfo[i]);
}
@ -683,7 +683,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
}
} else {
// master not there, if all peer's state and version are consistent, choose the master
int consistent = 0;
int32_t consistent = 0;
if (peersStatus) {
for (i = 0; i < pNode->replica; ++i) {
SSyncPeer *pTemp = pNode->peerInfo[i];
@ -721,9 +721,9 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
}
@ -757,7 +757,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
pthread_t thread;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
pthread_attr_destroy(&thattr);
if (ret != 0) {
@ -802,7 +802,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
// Ensure the sync of mnode not interrupted
if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) {
sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, syncTmrCtrl, &pPeer->timer);
taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer);
return;
}
@ -815,7 +815,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
firstPkt.port = tsSyncPort;
taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
sError("%s, failed to send sync-req to peer", pPeer->id);
@ -831,12 +831,12 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) {
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
SFwdInfo * pFwdInfo;
sDebug("%s, forward-rsp is received, ver:%" PRIu64, pPeer->id, pFwdRsp->version);
sDebug("%s, forward-rsp is received, code:%x ver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version);
SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first;
if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
// find the forwardInfo from first
for (int i = 0; i < pSyncFwds->fwds; ++i) {
for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo;
if (pFwdRsp->version == pFwdInfo->version) break;
}
@ -850,7 +850,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SWalHead * pHead = (SWalHead *)cont;
sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version);
sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len);
if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
// nodeVersion = pHead->version;
@ -859,7 +859,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
syncSaveIntoBuffer(pPeer, pHead);
} else {
sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version);
}
}
}
@ -879,10 +879,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
}
}
static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
if (pPeer->peerFd < 0) return -1;
int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
if (hlen != sizeof(SSyncHead)) {
sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen);
return -1;
@ -890,11 +890,12 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
// head.len = htonl(head.len);
if (pHead->len < 0) {
sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len);
sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len);
return -1;
}
int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
assert(pHead->len <= TSDB_MAX_WAL_SIZE);
int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
if (bytes != pHead->len) {
sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len);
return -1;
@ -903,7 +904,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
return 0;
}
static int syncProcessPeerMsg(void *param, void *buffer) {
static int32_t syncProcessPeerMsg(void *param, void *buffer) {
SSyncPeer *pPeer = param;
SSyncHead head;
char * cont = buffer;
@ -911,7 +912,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) {
SSyncNode *pNode = pPeer->pSyncNode;
pthread_mutex_lock(&(pNode->mutex));
int code = syncReadPeerMsg(pPeer, &head, cont);
int32_t code = syncReadPeerMsg(pPeer, &head, cont);
if (code == 0) {
if (head.type == TAOS_SMSG_FORWARD) {
@ -948,12 +949,12 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
pPeersStatus->role = nodeRole;
pPeersStatus->ack = ack;
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
}
int retLen = write(pPeer->peerFd, msg, statusMsgLen);
int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen);
if (retLen == statusMsgLen) {
sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
pPeersStatus->version, pPeersStatus->ack);
@ -975,10 +976,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
return;
}
int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
if (connFd < 0) {
sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
return;
}
@ -999,7 +1000,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
} else {
sDebug("try later");
close(connFd);
taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer);
taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
}
@ -1024,7 +1025,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
syncAddPeerRef(pPeer);
int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
pthread_attr_destroy(&thattr);
if (ret < 0) {
@ -1036,9 +1037,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
}
}
static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
char ipstr[24];
int i;
static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
char ipstr[24];
int32_t i;
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@ -1051,7 +1052,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) {
}
int32_t vgId = firstPkt.syncHead.vgId;
SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t));
SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t));
if (ppNode == NULL || *ppNode == NULL) {
sError("vgId:%d, vgId could not be found", vgId);
taosCloseSocket(connFd);
@ -1124,10 +1125,9 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
}
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last;
memset(pFwdInfo, 0, sizeof(SFwdInfo));
pFwdInfo->version = version;
pFwdInfo->mhandle = mhandle;
pFwdInfo->acks = 0;
pFwdInfo->confirmed = 0;
pFwdInfo->time = time;
pSyncFwds->fwds++;
@ -1137,8 +1137,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
SSyncFwds *pSyncFwds = pNode->pSyncFwds;
int fwds = pSyncFwds->fwds;
for (int i = 0; i < fwds; ++i) {
int32_t fwds = pSyncFwds->fwds;
for (int32_t i = 0; i < fwds; ++i) {
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first;
if (pFwdInfo->confirmed == 0) break;
@ -1152,7 +1152,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
}
static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) {
int confirm = 0;
int32_t confirm = 0;
if (pFwdInfo->code == 0) pFwdInfo->code = code;
if (code == 0) {
@ -1186,7 +1186,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
if (pSyncFwds->fwds > 0) {
pthread_mutex_lock(&(pNode->mutex));
for (int i = 0; i < pSyncFwds->fwds; ++i) {
for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
if (time - pFwdInfo->time < 2000) break;
syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
@ -1196,23 +1196,23 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
pthread_mutex_unlock(&(pNode->mutex));
}
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
}
taosReleaseRef(tsSyncRefId, rid);
}
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) {
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) {
SSyncPeer *pPeer;
SSyncHead *pSyncHead;
SWalHead * pWalHead = data;
int fwdLen;
int32_t fwdLen;
int32_t code = 0;
if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
pWalHead->version, nodeVersion);
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
syncRestartConnection(pPeer);
}
@ -1221,8 +1221,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
// always update version
nodeVersion = pWalHead->version;
sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole],
qtype, pWalHead->version);
sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
@ -1238,7 +1238,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
pthread_mutex_lock(&(pNode->mutex));
for (int i = 0; i < pNode->replica; ++i) {
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
if (pPeer == NULL || pPeer->peerFd < 0) continue;
if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue;
@ -1248,7 +1248,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
code = 1;
}
int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
if (retLen == fwdLen) {
sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
} else {

32
src/sync/src/syncRestore.c
View File

@ -48,12 +48,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
}
}
static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info
SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info
SFileAck fileAck;
int code = -1;
int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
uint32_t pindex = 0; // index in last restore
bool fileChanged = false;
@ -62,7 +62,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
sinfo.index = 0;
while (1) {
// read file info
int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
if (ret < 0) break;
// if no more file from master, break;
@ -104,7 +104,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
minfo.name[sizeof(minfo.name) - 1] = 0;
snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
if (dfd < 0) {
sError("%s, failed to open file:%s", pPeer->id, name);
break;
@ -132,9 +132,9 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
return code;
}
static int syncRestoreWal(SSyncPeer *pPeer) {
static int32_t syncRestoreWal(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
int ret, code = -1;
int32_t ret, code = -1;
void *buffer = calloc(1024000, 1); // size for one record
if (buffer == NULL) return -1;
@ -153,7 +153,7 @@ static int syncRestoreWal(SSyncPeer *pPeer) {
ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
if (ret < 0) break;
sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version);
(*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL);
}
@ -175,10 +175,10 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
return offset;
}
static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
int forwards = 0;
int32_t forwards = 0;
sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
@ -203,12 +203,12 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) {
return pRecv->code;
}
int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
SSyncNode * pNode = pPeer->pSyncNode;
SRecvBuffer *pRecv = pNode->pRecv;
if (pRecv == NULL) return -1;
int len = pHead->len + sizeof(SWalHead);
int32_t len = pHead->len + sizeof(SWalHead);
if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
memcpy(pRecv->offset, pHead, len);
@ -225,13 +225,13 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
static void syncCloseRecvBuffer(SSyncNode *pNode) {
if (pNode->pRecv) {
taosTFree(pNode->pRecv->buffer);
tfree(pNode->pRecv->buffer);
}
taosTFree(pNode->pRecv);
tfree(pNode->pRecv);
}
static int syncOpenRecvBuffer(SSyncNode *pNode) {
static int32_t syncOpenRecvBuffer(SSyncNode *pNode) {
syncCloseRecvBuffer(pNode);
SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
@ -252,13 +252,13 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) {
return 0;
}
static int syncRestoreDataStepByStep(SSyncPeer *pPeer) {
static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
nodeSStatus = TAOS_SYNC_STATUS_FILE;
uint64_t fversion = 0;
sDebug("%s, start to restore file", pPeer->id);
int code = syncRestoreFile(pPeer, &fversion);
int32_t code = syncRestoreFile(pPeer, &fversion);
if (code < 0) {
sError("%s, failed to restore file", pPeer->id);
return -1;

66
src/sync/src/syncRetrieve.c
View File

@ -27,7 +27,7 @@
#include "tsync.h"
#include "syncInt.h"
static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
sDebug("%s, start to monitor:%s", pPeer->id, name);
if (pPeer->notifyFd <= 0) {
@ -38,16 +38,16 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return -1;
}
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
}
int *wd = pPeer->watchFd + pPeer->watchNum;
int32_t *wd = pPeer->watchFd + pPeer->watchNum;
if (*wd >= 0) {
if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
@ -69,17 +69,17 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
return 0;
}
static int syncAreFilesModified(SSyncPeer *pPeer) {
static int32_t syncAreFilesModified(SSyncPeer *pPeer) {
if (pPeer->notifyFd <= 0) return 0;
char buf[2048];
int len = read(pPeer->notifyFd, buf, sizeof(buf));
char buf[2048];
int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
return -1;
}
int code = 0;
int32_t code = 0;
if (len > 0) {
const struct inotify_event *event;
char *ptr;
@ -97,11 +97,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) {
return code;
}
static int syncRetrieveFile(SSyncPeer *pPeer) {
static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo fileInfo;
SFileAck fileAck;
int code = -1;
int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
memset(&fileInfo, 0, sizeof(fileInfo));
@ -146,7 +146,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
}
// send the file to peer
int sfd = open(name, O_RDONLY);
int32_t sfd = open(name, O_RDONLY);
if (sfd < 0) break;
ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
@ -169,8 +169,8 @@ static int syncRetrieveFile(SSyncPeer *pPeer) {
/* if only a partial record is read out, set the IN_MODIFY flag in event,
so upper layer will reload the file to get a complete record */
static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
int ret;
static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) {
int32_t ret;
ret = read(sfd, pHead, sizeof(SWalHead));
if (ret < 0) return -1;
@ -194,7 +194,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) {
return sizeof(SWalHead) + pHead->len;
}
static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
pPeer->watchNum = 0;
taosClose(pPeer->notifyFd);
pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
@ -203,14 +203,14 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
return -1;
}
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles);
int *wd = pPeer->watchFd;
memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
int32_t *wd = pPeer->watchFd;
*wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
if (*wd == -1) {
@ -222,8 +222,8 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
}
static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
char buf[2048];
int len = read(pPeer->notifyFd, buf, sizeof(buf));
char buf[2048];
int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
return -1;
@ -243,11 +243,11 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
return 0;
}
static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
SWalHead *pHead = malloc(640000);
int code = -1;
static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
SWalHead *pHead = malloc(SYNC_MAX_SIZE);
int32_t code = -1;
int32_t bytes = 0;
int sfd;
int32_t sfd;
sfd = open(name, O_RDONLY);
if (sfd < 0) {
@ -256,10 +256,10 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
(void)lseek(sfd, offset, SEEK_SET);
sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion);
while (1) {
int wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
if (wsize < 0) break;
if (wsize == 0) {
code = 0;
@ -267,7 +267,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
}
sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if (ret != wsize) break;
pPeer->sversion = pHead->version;
@ -287,9 +287,9 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion,
return -1;
}
static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
SSyncNode *pNode = pPeer->pSyncNode;
int code = -1;
int32_t code = -1;
char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file
if (syncAreFilesModified(pPeer) != 0) return -1;
@ -325,7 +325,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
// if all data up to fversion is read out, it is over
if (pPeer->sversion >= fversion && fversion > 0) {
code = 0;
sDebug("%s, data up to fversion:%" PRId64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
break;
}
@ -370,13 +370,13 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
return code;
}
static int syncRetrieveWal(SSyncPeer *pPeer) {
static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
SSyncNode * pNode = pPeer->pSyncNode;
char fname[TSDB_FILENAME_LEN * 3];
char wname[TSDB_FILENAME_LEN * 2];
int32_t size;
struct stat fstat;
int code = -1;
int32_t code = -1;
int64_t index = 0;
while (1) {
@ -403,7 +403,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
size = fstat.st_size;
sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);
int sfd = open(fname, O_RDONLY);
int32_t sfd = open(fname, O_RDONLY);
if (sfd < 0) break;
code = taosSendFile(pPeer->syncFd, sfd, NULL, size);
@ -428,7 +428,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) {
return code;
}
static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFirstPkt firstPkt;

105
src/sync/src/taosTcpPool.c
View File

@ -13,18 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
#include "tutil.h"
#include "tsocket.h"
#include "taoserror.h"
#include "taosTcpPool.h"
#include "twal.h"
#include "tsync.h"
#include "syncInt.h"
typedef struct SThreadObj {
pthread_t thread;
bool stop;
int pollFd;
int numOfFds;
int32_t pollFd;
int32_t numOfFds;
struct SPoolObj *pPool;
} SThreadObj;
@ -32,15 +36,15 @@ typedef struct SPoolObj {
SPoolInfo info;
SThreadObj **pThread;
pthread_t thread;
int nextId;
int acceptFd; // FD for accept new connection
int32_t nextId;
int32_t acceptFd; // FD for accept new connection
} SPoolObj;
typedef struct {
SThreadObj *pThread;
void *ahandle;
int fd;
int closedByApp;
void * ahandle;
int32_t fd;
int32_t closedByApp;
} SConnObj;
static void *taosAcceptPeerTcpConnection(void *argv);
@ -53,66 +57,66 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) {
SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
if (pPool == NULL) {
uError("TCP server, no enough memory");
sError("failed to alloc pool for TCP server since no enough memory");
return NULL;
}
pPool->info = *pInfo;
pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
if (pPool->pThread == NULL) {
uError("TCP server, no enough memory");
free(pPool);
sError("failed to alloc pool thread for TCP server since no enough memory");
tfree(pPool);
return NULL;
}
pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
if (pPool->acceptFd < 0) {
free(pPool->pThread);
free(pPool);
uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
tfree(pPool->pThread);
tfree(pPool);
sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
return NULL;
}
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) {
uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
sError("failed to create accept thread for TCP server since %s", strerror(errno));
close(pPool->acceptFd);
free(pPool->pThread);
free(pPool);
tfree(pPool->pThread);
tfree(pPool);
return NULL;
}
pthread_attr_destroy(&thattr);
uDebug("%p TCP pool is created", pPool);
sDebug("%p TCP pool is created", pPool);
return pPool;
}
void taosCloseTcpThreadPool(void *param) {
SPoolObj * pPool = (SPoolObj *)param;
SPoolObj * pPool = param;
SThreadObj *pThread;
shutdown(pPool->acceptFd, SHUT_RD);
pthread_join(pPool->thread, NULL);
for (int i = 0; i < pPool->info.numOfThreads; ++i) {
for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) {
pThread = pPool->pThread[i];
if (pThread) taosStopPoolThread(pThread);
}
uDebug("%p TCP pool is closed", pPool);
sDebug("%p TCP pool is closed", pPool);
taosTFree(pPool->pThread);
free(pPool);
tfree(pPool->pThread);
tfree(pPool);
}
void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
struct epoll_event event;
SPoolObj *pPool = (SPoolObj *)param;
SPoolObj *pPool = param;
SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1);
SConnObj *pConn = calloc(sizeof(SConnObj), 1);
if (pConn == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@ -120,7 +124,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
SThreadObj *pThread = taosGetTcpThread(pPool);
if (pThread == NULL) {
free(pConn);
tfree(pConn);
return NULL;
}
@ -133,13 +137,13 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) {
event.data.ptr = pConn;
if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
uError("failed to add fd:%d(%s)", connFd, strerror(errno));
sError("failed to add fd:%d since %s", connFd, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
free(pConn);
tfree(pConn);
pConn = NULL;
} else {
pThread->numOfFds++;
uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
}
return pConn;
@ -149,7 +153,7 @@ void taosFreeTcpConn(void *param) {
SConnObj * pConn = (SConnObj *)param;
SThreadObj *pThread = pConn->pThread;
uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
pConn->closedByApp = 1;
shutdown(pConn->fd, SHUT_WR);
}
@ -164,9 +168,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) {
pThread->numOfFds--;
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
taosClose(pConn->fd);
free(pConn);
tfree(pConn);
}
#define maxEvents 10
@ -183,18 +187,18 @@ static void *taosProcessTcpData(void *param) {
while (1) {
if (pThread->stop) break;
int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
if (pThread->stop) {
uDebug("%p TCP epoll thread is exiting...", pThread);
sDebug("%p TCP epoll thread is exiting...", pThread);
break;
}
if (fdNum < 0) {
uError("epoll_wait failed (%s)", strerror(errno));
sError("epoll_wait failed since %s", strerror(errno));
continue;
}
for (int i = 0; i < fdNum; ++i) {
for (int32_t i = 0; i < fdNum; ++i) {
pConn = events[i].data.ptr;
assert(pConn);
@ -219,17 +223,16 @@ static void *taosProcessTcpData(void *param) {
continue;
}
}
}
if (pThread->stop) break;
}
uDebug("%p TCP epoll thread exits", pThread);
sDebug("%p TCP epoll thread exits", pThread);
close(pThread->pollFd);
free(pThread);
free(buffer);
tfree(pThread);
tfree(buffer);
return NULL;
}
@ -242,18 +245,18 @@ static void *taosAcceptPeerTcpConnection(void *argv) {
while (1) {
struct sockaddr_in clientAddr;
socklen_t addrlen = sizeof(clientAddr);
int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd < 0) {
if (errno == EINVAL) {
uDebug("%p TCP server accept is exiting...", pPool);
sDebug("%p TCP server accept is exiting...", pPool);
break;
} else {
uError("TCP accept failure, reason:%s", strerror(errno));
sError("TCP accept failure since %s", strerror(errno));
continue;
}
}
// uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
// sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port);
taosKeepTcpAlive(connFd);
(*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
}
@ -273,23 +276,23 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
pThread->pPool = pPool;
pThread->pollFd = epoll_create(10); // size does not matter
if (pThread->pollFd < 0) {
free(pThread);
tfree(pThread);
return NULL;
}
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread);
pthread_attr_destroy(&thattr);
if (ret != 0) {
close(pThread->pollFd);
free(pThread);
tfree(pThread);
return NULL;
}
uDebug("%p TCP epoll thread is created", pThread);
sDebug("%p TCP epoll thread is created", pThread);
pPool->pThread[pPool->nextId] = pThread;
pPool->nextId++;
pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
@ -314,12 +317,12 @@ static void taosStopPoolThread(SThreadObj *pThread) {
eventfd_t fd = eventfd(1, 0);
if (fd == -1) {
// failed to create eventfd, call pthread_cancel instead, which may result in data corruption
uError("failed to create eventfd(%s)", strerror(errno));
sError("failed to create eventfd since %s", strerror(errno));
pthread_cancel(pThread->thread);
pThread->stop = true;
} else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
// failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption
uError("failed to call epoll_ctl(%s)", strerror(errno));
sError("failed to call epoll_ctl since %s", strerror(errno));
pthread_cancel(pThread->thread);
}

30
src/sync/src/tarbitrator.c
View File

@ -28,22 +28,22 @@
#include "syncInt.h"
static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp);
static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
static void arbProcessBrokenLink(void *param);
static int arbProcessPeerMsg(void *param, void *buffer);
static int32_t arbProcessPeerMsg(void *param, void *buffer);
static tsem_t tsArbSem;
static ttpool_h tsArbTcpPool;
typedef struct {
char id[TSDB_EP_LEN + 24];
int nodeFd;
void *pConn;
char id[TSDB_EP_LEN + 24];
int32_t nodeFd;
void * pConn;
} SNodeConn;
int main(int argc, char *argv[]) {
int32_t main(int32_t argc, char *argv[]) {
char arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
for (int i = 1; i < argc; ++i) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
tsArbitratorPort = atoi(argv[++i]);
} else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) {
@ -86,7 +86,7 @@ int main(int argc, char *argv[]) {
info.numOfThreads = 1;
info.serverIp = 0;
info.port = tsArbitratorPort;
info.bufferSize = 640000;
info.bufferSize = SYNC_MAX_SIZE;
info.processBrokenLink = arbProcessBrokenLink;
info.processIncomingMsg = arbProcessPeerMsg;
info.processIncomingConn = arbProcessIncommingConnection;
@ -108,7 +108,7 @@ int main(int argc, char *argv[]) {
return 0;
}
static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
char ipstr[24];
tinet_ntoa(ipstr, sourceIp);
sDebug("peer TCP connection from ip:%s", ipstr);
@ -128,10 +128,10 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) {
}
firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0;
snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port);
if (firstPkt.syncHead.vgId) {
sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
taosTFree(pNode);
tfree(pNode);
taosCloseSocket(connFd);
return;
}
@ -147,16 +147,16 @@ static void arbProcessBrokenLink(void *param) {
SNodeConn *pNode = param;
sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
taosTFree(pNode);
tfree(pNode);
}
static int arbProcessPeerMsg(void *param, void *buffer) {
static int32_t arbProcessPeerMsg(void *param, void *buffer) {
SNodeConn *pNode = param;
SSyncHead head;
int bytes = 0;
int32_t bytes = 0;
char * cont = (char *)buffer;
int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
if (hlen != sizeof(head)) {
sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
return -1;

7
src/tsdb/inc/tsdbMain.h
View File

@ -220,8 +220,7 @@ typedef struct {
SMemTable* mem;
SMemTable* imem;
STsdbFileH* tsdbFileH;
int commit;
pthread_t commitThread;
sem_t readyToCommit;
pthread_mutex_t mutex;
bool repoLocked;
} STsdbRepo;
@ -440,6 +439,7 @@ void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
void* tsdbCommitData(STsdbRepo* pRepo);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL;
@ -588,6 +588,9 @@ int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx);
int tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);
// ------------------ tsdbCommitQueue.c
int tsdbScheduleCommit(STsdbRepo *pRepo);
#ifdef __cplusplus
}
#endif

6
src/tsdb/src/tsdbBuffer.c
View File

@ -110,7 +110,7 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) {
}
}
tsdbDebug("vgId:%d buffer pool is closed", REPO_ID(pRepo));
tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo));
}
SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
@ -134,7 +134,7 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
pBufBlock->offset = 0;
pBufBlock->remain = pBufPool->bufBlockSize;
tsdbDebug("vgId:%d buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
return pNode;
}
@ -157,4 +157,4 @@ _err:
return NULL;
}
static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { taosTFree(pBufBlock); }
static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }

160
src/tsdb/src/tsdbCommitQueue.c Normal file
View File

@ -0,0 +1,160 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tlist.h"
#include "tref.h"
#include "tsdbMain.h"
typedef struct {
bool stop;
pthread_mutex_t lock;
pthread_cond_t queueNotEmpty;
int nthreads;
int refCount;
SList * queue;
pthread_t * threads;
} SCommitQueue;
typedef struct {
STsdbRepo *pRepo;
} SCommitReq;
static void *tsdbLoopCommit(void *arg);
SCommitQueue tsCommitQueue = {0};
int tsdbInitCommitQueue(int nthreads) {
SCommitQueue *pQueue = &tsCommitQueue;
if (nthreads < 1) nthreads = 1;
pQueue->stop = false;
pQueue->nthreads = nthreads;
pQueue->queue = tdListNew(0);
if (pQueue->queue == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t));
if (pQueue->threads == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tdListFree(pQueue->queue);
return -1;
}
pthread_mutex_init(&(pQueue->lock), NULL);
pthread_cond_init(&(pQueue->queueNotEmpty), NULL);
for (int i = 0; i < nthreads; i++) {
pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL);
}
return 0;
}
void tsdbDestroyCommitQueue() {
SCommitQueue *pQueue = &tsCommitQueue;
pthread_mutex_lock(&(pQueue->lock));
if (pQueue->stop) {
pthread_mutex_unlock(&(pQueue->lock));
return;
}
pQueue->stop = true;
pthread_cond_broadcast(&(pQueue->queueNotEmpty));
pthread_mutex_unlock(&(pQueue->lock));
for (size_t i = 0; i < pQueue->nthreads; i++) {
pthread_join(pQueue->threads[i], NULL);
}
free(pQueue->threads);
tdListFree(pQueue->queue);
pthread_cond_destroy(&(pQueue->queueNotEmpty));
pthread_mutex_destroy(&(pQueue->lock));
}
int tsdbScheduleCommit(STsdbRepo *pRepo) {
SCommitQueue *pQueue = &tsCommitQueue;
SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SCommitReq));
if (pNode == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
((SCommitReq *)pNode->data)->pRepo = pRepo;
pthread_mutex_lock(&(pQueue->lock));
// ASSERT(pQueue->stop);
tdListAppendNode(pQueue->queue, pNode);
pthread_cond_signal(&(pQueue->queueNotEmpty));
pthread_mutex_unlock(&(pQueue->lock));
return 0;
}
static void *tsdbLoopCommit(void *arg) {
SCommitQueue *pQueue = &tsCommitQueue;
SListNode * pNode = NULL;
STsdbRepo * pRepo = NULL;
while (true) {
pthread_mutex_lock(&(pQueue->lock));
while (true) {
pNode = tdListPopHead(pQueue->queue);
if (pNode == NULL) {
if (pQueue->stop && pQueue->refCount <= 0) {
pthread_mutex_unlock(&(pQueue->lock));
goto _exit;
} else {
pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock));
}
} else {
break;
}
}
pthread_mutex_unlock(&(pQueue->lock));
pRepo = ((SCommitReq *)pNode->data)->pRepo;
tsdbCommitData(pRepo);
listNodeFree(pNode);
}
_exit:
return NULL;
}
void tsdbIncCommitRef(int vgId) {
int refCount = atomic_add_fetch_32(&tsCommitQueue.refCount, 1);
tsdbDebug("vgId:%d, inc commit queue ref to %d", vgId, refCount);
}
void tsdbDecCommitRef(int vgId) {
int refCount = atomic_sub_fetch_32(&tsCommitQueue.refCount, 1);
pthread_cond_broadcast(&(tsCommitQueue.queueNotEmpty));
tsdbDebug("vgId:%d, dec commit queue ref to %d", vgId, refCount);
}

6
src/tsdb/src/tsdbFile.c
View File

@ -65,7 +65,7 @@ _err:
void tsdbFreeFileH(STsdbFileH *pFileH) {
if (pFileH) {
pthread_rwlock_destroy(&pFileH->fhlock);
taosTFree(pFileH->pFGroup);
tfree(pFileH->pFGroup);
free(pFileH);
}
}
@ -199,7 +199,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
regfree(&regex1);
regfree(&regex2);
taosTFree(tDataDir);
tfree(tDataDir);
closedir(dir);
return 0;
@ -209,7 +209,7 @@ _err:
regfree(&regex1);
regfree(&regex2);
taosTFree(tDataDir);
tfree(tDataDir);
if (dir != NULL) closedir(dir);
tsdbCloseFileH(pRepo);
return -1;

31
src/tsdb/src/tsdbMain.c
View File

@ -163,7 +163,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
if (toCommit) {
tsdbAsyncCommit(pRepo);
if (pRepo->commit) pthread_join(pRepo->commitThread, NULL);
sem_wait(&(pRepo->readyToCommit));
}
tsdbUnRefMemTable(pRepo, pRepo->mem);
tsdbUnRefMemTable(pRepo, pRepo->imem);
@ -228,7 +228,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
int prefixLen = (int)strlen(prefix);
if (name[0] == 0) { // get the file from index or after, but not larger than eindex
taosTFree(sdup);
tfree(sdup);
int fid = (*index) / TSDB_FILE_TYPE_MAX;
if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
@ -260,8 +260,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
fname = malloc(prefixLen + strlen(name) + 2);
sprintf(fname, "%s/%s", prefix, name);
if (access(fname, F_OK) != 0) {
taosFree(fname);
taosFree(sdup);
tfree(fname);
tfree(sdup);
return 0;
}
if (*index == TSDB_META_FILE_INDEX) { // get meta file
@ -269,20 +269,20 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
} else {
tsdbGetFileInfoImpl(fname, &magic, size);
}
taosFree(fname);
taosFree(sdup);
tfree(fname);
tfree(sdup);
return magic;
}
if (stat(fname, &fState) < 0) {
taosTFree(fname);
tfree(fname);
return 0;
}
*size = fState.st_size;
// magic = *size;
taosTFree(fname);
tfree(fname);
return magic;
}
@ -600,7 +600,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
return 0;
_err:
taosTFree(fname);
tfree(fname);
if (fd >= 0) close(fd);
return -1;
}
@ -637,13 +637,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
tsdbDecodeCfg(buf, pCfg);
taosTFree(fname);
tfree(fname);
close(fd);
return 0;
_err:
taosTFree(fname);
tfree(fname);
if (fd >= 0) close(fd);
return -1;
}
@ -675,6 +675,12 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
goto _err;
}
code = sem_init(&(pRepo->readyToCommit), 0, 1);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
goto _err;
}
pRepo->repoLocked = false;
pRepo->rootDir = strdup(rootDir);
@ -718,7 +724,8 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) {
tsdbFreeMeta(pRepo->tsdbMeta);
// tsdbFreeMemTable(pRepo->mem);
// tsdbFreeMemTable(pRepo->imem);
taosTFree(pRepo->rootDir);
tfree(pRepo->rootDir);
sem_destroy(&(pRepo->readyToCommit));
pthread_mutex_destroy(&pRepo->mutex);
free(pRepo);
}

172
src/tsdb/src/tsdbMemTable.c
View File

@ -24,7 +24,6 @@ static void tsdbFreeMemTable(SMemTable *pMemTable);
static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
static void tsdbFreeTableData(STableData *pTableData);
static char * tsdbGetTsTupleKey(const void *data);
static void * tsdbCommitData(void *arg);
static int tsdbCommitMeta(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo);
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
@ -262,43 +261,31 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
int tsdbAsyncCommit(STsdbRepo *pRepo) {
SMemTable *pIMem = pRepo->imem;
int code = 0;
if (pIMem != NULL) {
ASSERT(pRepo->commit);
tsdbDebug("vgId:%d waiting for the commit thread", REPO_ID(pRepo));
code = pthread_join(pRepo->commitThread, NULL);
tsdbDebug("vgId:%d commit thread is finished", REPO_ID(pRepo));
if (code != 0) {
tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
pRepo->commit = 0;
}
ASSERT(pRepo->commit == 0);
if (pRepo->mem != NULL) {
sem_wait(&(pRepo->readyToCommit));
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
if (tsdbLockRepo(pRepo) < 0) return -1;
pRepo->imem = pRepo->mem;
pRepo->mem = NULL;
pRepo->commit = 1;
code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo);
if (code != 0) {
tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
tsdbUnlockRepo(pRepo);
return -1;
}
tsdbScheduleCommit(pRepo);
if (tsdbUnlockRepo(pRepo) < 0) return -1;
}
if (pIMem && tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
return 0;
}
int tsdbSyncCommit(TSDB_REPO_T *repo) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
tsdbAsyncCommit(pRepo);
sem_wait(&(pRepo->readyToCommit));
sem_post(&(pRepo->readyToCommit));
return 0;
}
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
@ -419,6 +406,68 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
return 0;
}
void *tsdbCommitData(STsdbRepo *pRepo) {
SMemTable * pMem = pRepo->imem;
STsdbCfg * pCfg = &pRepo->config;
SDataCols * pDataCols = NULL;
STsdbMeta * pMeta = pRepo->tsdbMeta;
SCommitIter *iters = NULL;
SRWHelper whelper = {0};
ASSERT(pMem != NULL);
tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
// Create the iterator to read from cache
if (pMem->numOfRows > 0) {
iters = tsdbCreateCommitIters(pRepo);
if (iters == NULL) {
tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
goto _exit;
}
int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
// Loop to commit to each file
for (int fid = sfid; fid <= efid; fid++) {
if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _exit;
}
}
}
// Commit to update meta file
if (tsdbCommitMeta(pRepo) < 0) {
tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
tsdbFitRetention(pRepo);
_exit:
tdFreeDataCols(pDataCols);
tsdbDestroyCommitIters(iters, pMem->maxTables);
tsdbDestroyHelper(&whelper);
tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
tsdbEndCommit(pRepo);
return NULL;
}
// ---------------- LOCAL FUNCTIONS ----------------
static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) {
ASSERT(pRepo->mem != NULL);
@ -488,7 +537,7 @@ static void tsdbFreeMemTable(SMemTable* pMemTable) {
tdListFree(pMemTable->extraBuffList);
tdListFree(pMemTable->bufBlockList);
tdListFree(pMemTable->actList);
taosTFree(pMemTable->tData);
tfree(pMemTable->tData);
free(pMemTable);
}
}
@ -529,69 +578,6 @@ static void tsdbFreeTableData(STableData *pTableData) {
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
static void *tsdbCommitData(void *arg) {
STsdbRepo * pRepo = (STsdbRepo *)arg;
SMemTable * pMem = pRepo->imem;
STsdbCfg * pCfg = &pRepo->config;
SDataCols * pDataCols = NULL;
STsdbMeta * pMeta = pRepo->tsdbMeta;
SCommitIter *iters = NULL;
SRWHelper whelper = {0};
ASSERT(pRepo->commit == 1);
ASSERT(pMem != NULL);
tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
// Create the iterator to read from cache
if (pMem->numOfRows > 0) {
iters = tsdbCreateCommitIters(pRepo);
if (iters == NULL) {
tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
goto _exit;
}
int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
// Loop to commit to each file
for (int fid = sfid; fid <= efid; fid++) {
if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _exit;
}
}
}
// Commit to update meta file
if (tsdbCommitMeta(pRepo) < 0) {
tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
tsdbFitRetention(pRepo);
_exit:
tdFreeDataCols(pDataCols);
tsdbDestroyCommitIters(iters, pMem->maxTables);
tsdbDestroyHelper(&whelper);
tsdbEndCommit(pRepo);
tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
return NULL;
}
static int tsdbCommitMeta(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
@ -642,8 +628,8 @@ _err:
}
static void tsdbEndCommit(STsdbRepo *pRepo) {
ASSERT(pRepo->commit == 1);
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
sem_post(&(pRepo->readyToCommit));
}
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
@ -746,7 +732,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
goto _err;
}
taosTFree(dataDir);
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 0, pGroup);
pthread_rwlock_wrlock(&(pFileH->fhlock));
@ -768,7 +754,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
return 0;
_err:
taosTFree(dataDir);
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 1, NULL);
return -1;
}
@ -843,7 +829,7 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
pMemTable->tData = pTableData;
taosWUnLockLatch(&(pMemTable->latch));
taosTFree(tData);
tfree(tData);
return 0;
}

20
src/tsdb/src/tsdbMeta.c
View File

@ -192,7 +192,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
return 0;
_err:
taosTFree(tbname);
tfree(tbname);
return -1;
}
@ -462,7 +462,7 @@ void tsdbFreeMeta(STsdbMeta *pMeta) {
if (pMeta) {
taosHashCleanup(pMeta->uidMap);
tdListFree(pMeta->superList);
taosTFree(pMeta->tables);
tfree(pMeta->tables);
pthread_rwlock_destroy(&pMeta->rwLock);
free(pMeta);
}
@ -486,11 +486,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
}
tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo));
taosTFree(fname);
tfree(fname);
return 0;
_err:
taosTFree(fname);
tfree(fname);
return -1;
}
@ -761,7 +761,7 @@ static void tsdbFreeTable(STable *pTable) {
if (pTable->name != NULL)
tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable),
TABLE_UID(pTable));
taosTFree(TABLE_NAME(pTable));
tfree(TABLE_NAME(pTable));
if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) {
for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) {
tdFreeSchema(pTable->schema[i]);
@ -775,7 +775,7 @@ static void tsdbFreeTable(STable *pTable) {
kvRowFree(pTable->tagVal);
tSkipListDestroy(pTable->pIndex);
taosTFree(pTable->sql);
tfree(pTable->sql);
free(pTable);
}
}
@ -1066,9 +1066,9 @@ void tsdbClearTableCfg(STableCfg *config) {
if (config->schema) tdFreeSchema(config->schema);
if (config->tagSchema) tdFreeSchema(config->tagSchema);
if (config->tagValues) kvRowFree(config->tagValues);
taosTFree(config->name);
taosTFree(config->sname);
taosTFree(config->sql);
tfree(config->name);
tfree(config->sname);
tfree(config->sql);
free(config);
}
}
@ -1290,7 +1290,7 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) {
STable **tTables = pMeta->tables;
pMeta->tables = tables;
taosTFree(tTables);
tfree(tTables);
tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables);
return 0;

25
src/tsdb/src/tsdbRead.c
View File

@ -1544,15 +1544,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
}
static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) {
taosTFree(pSupporter->numOfBlocksPerTable);
taosTFree(pSupporter->blockIndexArray);
tfree(pSupporter->numOfBlocksPerTable);
tfree(pSupporter->blockIndexArray);
for (int32_t i = 0; i < numOfTables; ++i) {
STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i];
taosTFree(pBlockInfo);
tfree(pBlockInfo);
}
taosTFree(pSupporter->pDataBlockInfo);
tfree(pSupporter->pDataBlockInfo);
}
static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {
@ -1970,7 +1970,7 @@ static void destroyHelper(void* param) {
tQueryInfo* pInfo = (tQueryInfo*)param;
if (pInfo->optr != TSDB_RELATION_IN) {
taosTFree(pInfo->q);
tfree(pInfo->q);
}
free(param);
@ -2035,7 +2035,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
taosTFree(cond.colList);
tfree(cond.colList);
pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
if (pSecQueryHandle->pTableCheckInfo == NULL) {
@ -2348,7 +2348,8 @@ void filterPrepare(void* expr, void* param) {
if (size < (uint32_t)pSchema->bytes) {
size = pSchema->bytes;
}
pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE); // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
// to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(wchar_t) space.
pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
tVariantDump(pCond, pInfo->q, pSchema->type, true);
}
}
@ -2750,7 +2751,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
destroyTableMemIterator(pTableCheckInfo);
taosTFree(pTableCheckInfo->pCompInfo);
tfree(pTableCheckInfo->pCompInfo);
}
taosArrayDestroy(pQueryHandle->pTableCheckInfo);
}
@ -2759,14 +2760,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
size_t cols = taosArrayGetSize(pQueryHandle->pColumns);
for (int32_t i = 0; i < cols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
taosTFree(pColInfo->pData);
tfree(pColInfo->pData);
}
taosArrayDestroy(pQueryHandle->pColumns);
}
taosArrayDestroy(pQueryHandle->defaultLoadColumn);
taosTFree(pQueryHandle->pDataBlockInfo);
taosTFree(pQueryHandle->statis);
tfree(pQueryHandle->pDataBlockInfo);
tfree(pQueryHandle->statis);
// todo check error
tsdbMayUnTakeMemSnapshot(pQueryHandle);
@ -2780,7 +2781,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p",
pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
taosTFree(pQueryHandle);
tfree(pQueryHandle);
}
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) {

4
src/tsdb/tests/tsdbTests.cpp
View File

@ -80,7 +80,7 @@ static int insertData(SInsertInfo *pInfo) {
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
taosTFree(pMsg);
tfree(pMsg);
return -1;
}
}
@ -88,7 +88,7 @@ static int insertData(SInsertInfo *pInfo) {
double etime = getCurTime();
printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows);
taosTFree(pMsg);
tfree(pMsg);
return 0;
}

17
src/util/inc/tqueue.h
View File

@ -20,6 +20,23 @@
extern "C" {
#endif
/*
This set of API for queue is designed specially for vnode/mnode. The main purpose is to
consume all the items instead of one item from a queue by one single read. Also, it can
combine multiple queues into a queue set, a consumer thread can consume a queue set via
a single API instead of looping every queue by itself.
Notes:
1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe
2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe.
3: read/write operation APIs are multi-thread safe
To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c)
shall be used to set up the protection.
*/
typedef void* taos_queue;
typedef void* taos_qset;
typedef void* taos_qall;

4
src/util/src/hash.c
View File

@ -22,7 +22,7 @@
#define DO_FREE_HASH_NODE(_n) \
do { \
taosTFree(_n); \
tfree(_n); \
} while (0)
#define FREE_HASH_NODE(_h, _n) \
@ -522,7 +522,7 @@ void taosHashCleanup(SHashObj *pHashObj) {
size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock);
for (int32_t i = 0; i < memBlock; ++i) {
void *p = taosArrayGetP(pHashObj->pMemBlock, i);
taosTFree(p);
tfree(p);
}
taosArrayDestroy(pHashObj->pMemBlock);

2
src/util/src/talgo.c
View File

@ -153,7 +153,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const
void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) {
char *buf = calloc(1, size); // prepare the swap buffer
tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf);
taosTFree(buf);
tfree(buf);
}
void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) {

4
src/util/src/tcache.c
View File

@ -228,7 +228,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
pCacheObj->freeFp(p->data);
}
taosTFree(p);
tfree(p);
} else {
taosAddToTrashcan(pCacheObj, p);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
@ -614,7 +614,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
__cache_lock_destroy(pCacheObj);
taosTFree(pCacheObj->name);
tfree(pCacheObj->name);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}

8
src/util/src/tconfig.c
View File

@ -288,7 +288,7 @@ void taosReadGlobalLogCfg() {
option = value = NULL;
olen = vlen = 0;
taosGetline(&line, &len, fp);
tgetline(&line, &len, fp);
line[len - 1] = 0;
paGetToken(line, &option, &olen);
@ -302,7 +302,7 @@ void taosReadGlobalLogCfg() {
taosReadLogOption(option, value);
}
taosTFree(line);
tfree(line);
fclose(fp);
}
@ -334,7 +334,7 @@ bool taosReadGlobalCfg() {
option = value = NULL;
olen = vlen = 0;
taosGetline(&line, &len, fp);
tgetline(&line, &len, fp);
line[len - 1] = 0;
paGetToken(line, &option, &olen);
@ -354,7 +354,7 @@ bool taosReadGlobalCfg() {
fclose(fp);
taosTFree(line);
tfree(line);
if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) {
taosSetAllDebugFlag();

Some files were not shown because too many files have changed in this diff Show More