xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into hotfix/TD-2257

This commit is contained in:
haojun Liao 2020-12-04 10:45:36 +08:00 committed by GitHub
parent f349c8af87 8c9c40508e
commit 77e50deed8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
144 changed files with 4842 additions and 2275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
Jenkinsfile vendored
View File

@ -14,10 +14,12 @@ pipeline {
sh '''
date
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
@ -39,11 +41,13 @@ pipeline {
steps {
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
@ -65,11 +69,13 @@ pipeline {
steps {
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
@ -108,11 +114,13 @@ pipeline {
steps {
sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
@ -167,7 +175,47 @@ pipeline {
}
}
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
}
}

1
documentation20/webdocs/markdowndocs/administrator-ch.md
View File

@ -95,6 +95,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修
- logKeepDays：日志文件的最长保存时间。大于0时，日志文件会被重命名为taosdlog.xxx，其中xxx为日志文件最后修改的时间戳，单位为秒。默认值：0天。
- maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。
- telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。
- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。
**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。

2
documentation20/webdocs/markdowndocs/connector-ch.md
View File

@ -142,7 +142,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine
获取最近一次API调用失败的原因，返回值为错误代码。
**注意**：对于单个数据库连接，在同一时刻只能有一个线程使用该连接调用API，否则会有未定义的行为出现并可能导致客户端crash。客户端应用可以通过建立多个连接进行多线程的数据写入或查询处理。
**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。
### 异步查询API

7
packaging/cfg/taos.cfg
View File

@ -236,7 +236,7 @@
# httpDebugFlag 131
# debug flag for monitor
# monitorDebugFlag 131
# monDebugFlag 131
# debug flag for query
# qDebugflag 131
@ -260,4 +260,7 @@
# maxBinaryDisplayWidth 30
# enable/disable telemetry reporting
# telemetryReporting 1
# telemetryReporting 1
# enable/disable stream (continuous query)
# stream 1

2
packaging/tools/install.sh
View File

@ -172,6 +172,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -182,6 +183,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
[ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || :
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
[ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :

6
packaging/tools/install_client.sh
View File

@ -84,8 +84,9 @@ function install_main_path() {
function install_bin() {
# Remove links
${csudo} rm -f ${bin_link_dir}/taos || :
if [ "$osType" == "Darwin" ]; then
if [ "$osType" != "Darwin" ]; then
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
fi
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -94,8 +95,9 @@ function install_bin() {
#Make link
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
if [ "$osType" == "Darwin" ]; then
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
fi
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

6
packaging/tools/install_client_power.sh
View File

@ -84,8 +84,9 @@ function install_main_path() {
function install_bin() {
# Remove links
${csudo} rm -f ${bin_link_dir}/power || :
if [ "$osType" == "Darwin" ]; then
if [ "$osType" != "Darwin" ]; then
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
fi
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -94,8 +95,9 @@ function install_bin() {
#Make link
[ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || :
if [ "$osType" == "Darwin" ]; then
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || :
[ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || :
fi
[ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :

2
packaging/tools/install_power.sh
View File

@ -172,6 +172,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/power || :
${csudo} rm -f ${bin_link_dir}/powerd || :
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -182,6 +183,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || :
[ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || :
[ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || :
[ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || :
[ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
[ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :

2
packaging/tools/post.sh
View File

@ -92,6 +92,7 @@ function install_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -101,6 +102,7 @@ function install_bin() {
[ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos || :
[ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || :
[ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || :
[ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || :
}

3
packaging/tools/remove.sh
View File

@ -72,6 +72,7 @@ function clean_bin() {
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -222,4 +223,4 @@ elif echo $osinfo | grep -qwi "centos" ; then
fi
echo -e "${GREEN}TDengine is removed successfully!${NC}"
echo
echo

1
packaging/tools/remove_client.sh
View File

@ -38,6 +38,7 @@ function clean_bin() {
# Remove link
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} rm -f ${bin_link_dir}/set_core || :
}

1
packaging/tools/remove_client_power.sh
View File

@ -38,6 +38,7 @@ function clean_bin() {
# Remove link
${csudo} rm -f ${bin_link_dir}/power || :
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/set_core || :
}

3
packaging/tools/remove_power.sh
View File

@ -72,6 +72,7 @@ function clean_bin() {
${csudo} rm -f ${bin_link_dir}/power || :
${csudo} rm -f ${bin_link_dir}/powerd || :
${csudo} rm -f ${bin_link_dir}/powerdemo || :
${csudo} rm -f ${bin_link_dir}/powerdump || :
${csudo} rm -f ${bin_link_dir}/rmpower || :
${csudo} rm -f ${bin_link_dir}/tarbitrator || :
${csudo} rm -f ${bin_link_dir}/set_core || :
@ -223,4 +224,4 @@ fi
#fi
echo -e "${GREEN}PowerDB is removed successfully!${NC}"
echo
echo

57
src/balance/inc/bnInt.h Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_BALANCE_INT_H
#define TDENGINE_BALANCE_INT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "mnodeInt.h"
#include "mnodeDef.h"
#include "mnodeDnode.h"
typedef struct {
int32_t size;
int32_t maxSize;
SDnodeObj **list;
} SBnDnodes;
typedef struct {
void * timer;
bool stop;
pthread_mutex_t mutex;
pthread_cond_t cond;
pthread_t thread;
} SBnThread;
typedef struct {
pthread_mutex_t mutex;
} SBnMgmt;
int32_t bnInit();
void bnCleanUp();
bool bnStart();
void bnCheckStatus();
void bnCheckModules();
extern SBnDnodes tsBnDnodes;
extern void *tsMnodeTmr;
#ifdef __cplusplus
}
#endif
#endif

34
src/balance/inc/bnScore.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_BALANCE_SCORE_H
#define TDENGINE_BALANCE_SCORE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "bnInt.h"
void bnInitDnodes();
void bnCleanupDnodes();
void bnAccquireDnodes();
void bnReleaseDnodes();
float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
#ifdef __cplusplus
}
#endif
#endif

33
src/balance/inc/bnThread.h Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_BALANCE_THREAD_H
#define TDENGINE_BALANCE_THREAD_H
#ifdef __cplusplus
extern "C" {
#endif
#include "bnInt.h"
int32_t bnInitThread();
void bnCleanupThread();
void bnNotify();
void bnStartTimer(int64_t mseconds);
#ifdef __cplusplus
}
#endif
#endif

592
src/balance/src/balance.c → src/balance/src/bnMain.c
View File

@ -15,17 +15,12 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tutil.h"
#include "tbalance.h"
#include "tsync.h"
#include "ttimer.h"
#include "tglobal.h"
#include "tdataformat.h"
#include "dnode.h"
#include "mnode.h"
#include "mnodeDef.h"
#include "mnodeInt.h"
#include "mnodeDnode.h"
#include "bnInt.h"
#include "bnScore.h"
#include "bnThread.h"
#include "mnodeDb.h"
#include "mnodeMnode.h"
#include "mnodeSdb.h"
@ -33,36 +28,18 @@
#include "mnodeUser.h"
#include "mnodeVgroup.h"
/*
* once sdb work as mater, then tsAccessSquence reset to zero
* increase tsAccessSquence every balance interval
*/
extern void * tsMnodeTmr;
static void * tsBalanceTimer = NULL;
static int32_t tsBalanceDnodeListSize = 0;
static SDnodeObj ** tsBalanceDnodeList = NULL;
static int32_t tsBalanceDnodeListMallocSize = 16;
static pthread_mutex_t tsBalanceMutex;
static SBnMgmt tsBnMgmt;;
static void bnMonitorDnodeModule();
static void balanceStartTimer(int64_t mseconds);
static void balanceInitDnodeList();
static void balanceCleanupDnodeList();
static void balanceAccquireDnodeList();
static void balanceReleaseDnodeList();
static void balanceMonitorDnodeModule();
static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void balanceLock() {
pthread_mutex_lock(&tsBalanceMutex);
static void bnLock() {
pthread_mutex_lock(&tsBnMgmt.mutex);
}
static void balanceUnLock() {
pthread_mutex_unlock(&tsBalanceMutex);
static void bnUnLock() {
pthread_mutex_unlock(&tsBnMgmt.mutex);
}
static bool balanceCheckFree(SDnodeObj *pDnode) {
static bool bnCheckFree(SDnodeObj *pDnode) {
if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
return false;
@ -86,7 +63,7 @@ static bool balanceCheckFree(SDnodeObj *pDnode) {
return true;
}
static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
@ -111,27 +88,26 @@ static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
mnodeUpdateVgroup(pVgroup);
}
static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
// SVnodeGid tmp = *pVnodeGid1;
// *pVnodeGid1 = *pVnodeGid2;
// *pVnodeGid2 = tmp;
}
int32_t balanceAllocVnodes(SVgObj *pVgroup) {
int32_t bnAllocVnodes(SVgObj *pVgroup) {
static int32_t randIndex = 0;
int32_t dnode = 0;
int32_t vnodes = 0;
balanceLock();
balanceAccquireDnodeList();
bnLock();
bnAccquireDnodes();
mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
mnodeGetDnodesNum(), tsBalanceDnodeListSize);
mnodeGetDnodesNum(), tsBnDnodes.size);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
for (; dnode < tsBalanceDnodeListSize; ++dnode) {
SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
if (balanceCheckFree(pDnode)) {
for (; dnode < tsBnDnodes.size; ++dnode) {
SDnodeObj *pDnode = tsBnDnodes.list[dnode];
if (bnCheckFree(pDnode)) {
SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
pVnodeGid->dnodeId = pDnode->dnodeId;
pVnodeGid->pDnode = pDnode;
@ -148,8 +124,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
}
if (vnodes != pVgroup->numOfVnodes) {
balanceReleaseDnodeList();
balanceUnLock();
bnReleaseDnodes();
bnUnLock();
mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
@ -162,7 +138,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
if (mnodeGetOnlineDnodesNum() == 0) {
return TSDB_CODE_MND_NOT_READY;
@ -180,33 +155,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
if (pVgroup->numOfVnodes == 1) {
} else if (pVgroup->numOfVnodes == 2) {
if (randIndex++ % 2 == 0) {
balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
}
} else {
int32_t randVal = randIndex++ % 6;
if (randVal == 1) { // 1, 0, 2
balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
} else if (randVal == 2) { // 1, 2, 0
balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
} else if (randVal == 3) { // 2, 1, 0
balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
} else if (randVal == 4) { // 2, 0, 1
balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
}
if (randVal == 5) { // 0, 2, 1
balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
} else {
} // 0, 1, 2
}
balanceReleaseDnodeList();
balanceUnLock();
bnReleaseDnodes();
bnUnLock();
return TSDB_CODE_SUCCESS;
}
static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
return false;
}
@ -233,7 +208,7 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
* desc: remove one vnode from vgroup
* all vnodes in vgroup should in ready state, except the balancing one
**/
static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
static int32_t bnRemoveVnode(SVgObj *pVgroup) {
if (pVgroup->numOfVnodes <= 1) return -1;
SVnodeGid *pRmVnode = NULL;
@ -275,17 +250,17 @@ static int32_t balanceRemoveVnode(SVgObj *pVgroup) {
pSelVnode = pRmVnode;
}
if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
mDebug("vgId:%d, is not ready", pVgroup->vgId);
return -1;
} else {
mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
balanceDiscardVnode(pVgroup, pSelVnode);
bnDiscardVnode(pVgroup, pSelVnode);
return TSDB_CODE_SUCCESS;
}
}
static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *pGid = &pVgroup->vnodeGid[i];
if (pGid->dnodeId == 0) break;
@ -300,13 +275,13 @@ static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
/**
* desc: add vnode to vgroup, find a new one if dest dnode is null
**/
static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
if (pDestDnode == NULL) {
for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
SDnodeObj *pDnode = tsBalanceDnodeList[i];
for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
SDnodeObj *pDnode = tsBnDnodes.list[i];
if (pDnode == pSrcDnode) continue;
if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
if (!balanceCheckFree(pDnode)) continue;
if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
if (!bnCheckFree(pDnode)) continue;
pDestDnode = pDnode;
mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
@ -334,25 +309,25 @@ static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj
return TSDB_CODE_SUCCESS;
}
static bool balanceMonitorBalance() {
if (tsBalanceDnodeListSize < 2) return false;
static bool bnMonitorBalance() {
if (tsBnDnodes.size < 2) return false;
for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
SDnodeObj *pDnode = tsBalanceDnodeList[src];
mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
SDnodeObj *pDnode = tsBnDnodes.list[src];
mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1,
pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
pDnode->openVnodes);
}
float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
if (scoresDiff < 0.01) {
mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff);
return false;
}
for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) {
SDnodeObj *pSrcDnode = tsBnDnodes.list[src];
float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1);
if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
continue;
}
@ -363,29 +338,27 @@ static bool balanceMonitorBalance() {
pIter = mnodeGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break;
if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
for (int32_t dest = 0; dest < src; dest++) {
SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
SDnodeObj *pDestDnode = tsBnDnodes.list[dest];
if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
float destScore = bnTryCalcDnodeScore(pDestDnode, 1);
if (srcScore + 0.0001 < destScore) continue;
if (!balanceCheckFree(pDestDnode)) continue;
if (!bnCheckFree(pDestDnode)) continue;
mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
srcScore, pDestDnode->score, destScore);
balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
mnodeDecVgroupRef(pVgroup);
sdbFreeIter(pIter);
mnodeCancelGetNextVgroup(pIter);
return true;
}
}
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
}
return false;
@ -395,7 +368,7 @@ static bool balanceMonitorBalance() {
// 1. reset balanceAccessSquence to zero
// 2. reset state of dnodes to offline
// 3. reset lastAccess of dnodes to zero
void balanceReset() {
void bnReset() {
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
while (1) {
@ -413,12 +386,10 @@ void balanceReset() {
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
tsAccessSquence = 0;
}
static int32_t balanceMonitorVgroups() {
static int32_t bnMonitorVgroups() {
void * pIter = NULL;
SVgObj *pVgroup = NULL;
bool hasUpdatingVgroup = false;
@ -434,25 +405,24 @@ static int32_t balanceMonitorVgroups() {
if (vgReplica > dbReplica) {
mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
hasUpdatingVgroup = true;
code = balanceRemoveVnode(pVgroup);
code = bnRemoveVnode(pVgroup);
} else if (vgReplica < dbReplica) {
mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
hasUpdatingVgroup = true;
code = balanceAddVnode(pVgroup, NULL, NULL);
code = bnAddVnode(pVgroup, NULL, NULL);
}
mnodeDecVgroupRef(pVgroup);
if (code == TSDB_CODE_SUCCESS) {
mnodeCancelGetNextVgroup(pIter);
break;
}
}
sdbFreeIter(pIter);
return hasUpdatingVgroup;
}
static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) {
mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
void * pIter = NULL;
@ -462,14 +432,15 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
pIter = mnodeGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break;
hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup);
mnodeDecVgroupRef(pVgroup);
if (hasThisDnode) break;
if (hasThisDnode) {
mnodeCancelGetNextVgroup(pIter);
break;
}
}
sdbFreeIter(pIter);
if (!hasThisDnode) {
mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
mnodeDropDnode(pDnode, NULL);
@ -479,7 +450,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
return false;
}
static bool balanceMontiorDropping() {
static bool bnMontiorDropping() {
void *pIter = NULL;
SDnodeObj *pDnode = NULL;
@ -499,50 +470,46 @@ static bool balanceMontiorDropping() {
pDnode->status = TAOS_DN_STATUS_DROPPING;
mnodeUpdateDnode(pDnode);
mnodeDecDnodeRef(pDnode);
sdbFreeIter(pIter);
mnodeCancelGetNextDnode(pIter);
return true;
}
if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
bool ret = balanceMonitorDnodeDropping(pDnode);
bool ret = bnMonitorDnodeDropping(pDnode);
mnodeDecDnodeRef(pDnode);
sdbFreeIter(pIter);
mnodeCancelGetNextDnode(pIter);
return ret;
}
}
sdbFreeIter(pIter);
return false;
}
static bool balanceStart() {
bool bnStart() {
if (!sdbIsMaster()) return false;
balanceLock();
bnLock();
bnAccquireDnodes();
balanceAccquireDnodeList();
bnMonitorDnodeModule();
balanceMonitorDnodeModule();
bool updateSoon = balanceMontiorDropping();
bool updateSoon = bnMontiorDropping();
if (!updateSoon) {
updateSoon = balanceMonitorVgroups();
updateSoon = bnMonitorVgroups();
}
if (!updateSoon) {
updateSoon = balanceMonitorBalance();
updateSoon = bnMonitorBalance();
}
balanceReleaseDnodeList();
balanceUnLock();
bnReleaseDnodes();
bnUnLock();
return updateSoon;
}
static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
static void bnSetVgroupOffline(SDnodeObj* pDnode) {
void *pIter = NULL;
while (1) {
SVgObj *pVgroup;
@ -556,11 +523,9 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
}
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
}
static void balanceCheckDnodeAccess() {
void bnCheckStatus() {
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
@ -573,85 +538,39 @@ static void balanceCheckDnodeAccess() {
pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
pDnode->lastAccess, pDnode->status);
balanceSetVgroupOffline(pDnode);
bnSetVgroupOffline(pDnode);
}
}
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
}
static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
if (!sdbIsMaster()) return;
tsBalanceTimer = NULL;
tsAccessSquence ++;
balanceCheckDnodeAccess();
bool updateSoon = false;
if (handle == NULL) {
if (tsAccessSquence % tsBalanceInterval == 0) {
mDebug("balance function is scheduled by timer");
updateSoon = balanceStart();
}
} else {
int64_t mseconds = (int64_t)handle;
mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
updateSoon = balanceStart();
}
if (updateSoon) {
balanceStartTimer(1000);
} else {
taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
}
}
static void balanceStartTimer(int64_t mseconds) {
taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
}
void balanceSyncNotify() {
void bnCheckModules() {
if (sdbIsMaster()) {
balanceLock();
balanceAccquireDnodeList();
balanceMonitorDnodeModule();
balanceReleaseDnodeList();
balanceUnLock();
bnLock();
bnAccquireDnodes();
bnMonitorDnodeModule();
bnReleaseDnodes();
bnUnLock();
}
}
void balanceAsyncNotify() {
balanceStartTimer(500);
}
int32_t balanceInit() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
pthread_mutex_init(&tsBalanceMutex, NULL);
balanceInitDnodeList();
balanceStartTimer(2000);
mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
balanceReset();
int32_t bnInit() {
pthread_mutex_init(&tsBnMgmt.mutex, NULL);
bnInitDnodes();
bnInitThread();
bnReset();
return 0;
}
void balanceCleanUp() {
if (tsBalanceTimer != NULL) {
taosTmrStopA(&tsBalanceTimer);
pthread_mutex_destroy(&tsBalanceMutex);
tsBalanceTimer = NULL;
mDebug("stop balance timer");
}
balanceCleanupDnodeList();
void bnCleanUp() {
bnCleanupThread();
bnCleanupDnodes();
pthread_mutex_destroy(&tsBnMgmt.mutex);
}
int32_t balanceDropDnode(SDnodeObj *pDnode) {
int32_t bnDropDnode(SDnodeObj *pDnode) {
int32_t totalFreeVnodes = 0;
void * pIter = NULL;
SDnodeObj *pTempDnode = NULL;
@ -660,15 +579,13 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
pIter = mnodeGetNextDnode(pIter, &pTempDnode);
if (pTempDnode == NULL) break;
if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) {
totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
}
mnodeDecDnodeRef(pTempDnode);
}
sdbFreeIter(pIter);
if (pDnode->openVnodes > totalFreeVnodes) {
mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
return TSDB_CODE_MND_NO_ENOUGH_DNODES;
@ -677,296 +594,17 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
pDnode->status = TAOS_DN_STATUS_DROPPING;
mnodeUpdateDnode(pDnode);
balanceStartTimer(1100);
bnStartTimer(1100);
return TSDB_CODE_SUCCESS;
}
static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
if (pDnode->cpuAvgUsage < 80)
return 0;
else if (pDnode->cpuAvgUsage < 90)
return 10;
else
return 50;
}
static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
if (pDnode->memoryAvgUsage < 80)
return 0;
else if (pDnode->memoryAvgUsage < 90)
return 10;
else
return 50;
}
static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
if (pDnode->diskAvgUsage < 80)
return 0;
else if (pDnode->diskAvgUsage < 90)
return 10;
else
return 50;
}
static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
if (pDnode->bandwidthUsage < 30)
return 0;
else if (pDnode->bandwidthUsage < 80)
return 10;
else
return 50;
}
static float balanceCalcModuleScore(SDnodeObj *pDnode) {
if (pDnode->numOfCores <= 0) return 0;
if (pDnode->isMgmt) {
return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
}
return 0;
}
static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
if (pDnode->numOfCores <= 0) return 0;
return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
}
/**
* calc singe score, such as cpu/memory/disk/bandwitdh/vnode
* 1. get the score config
* 2. if the value is out of range, use border data
* 3. otherwise use interpolation method
**/
void balanceCalcDnodeScore(SDnodeObj *pDnode) {
pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
}
float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
balanceCalcBandwidthScore(pDnode);
float moduleScore = balanceCalcModuleScore(pDnode);
float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
float score = systemScore + moduleScore + vnodeScore + pDnode->customScore;
return score;
}
static void balanceInitDnodeList() {
tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
}
static void balanceCleanupDnodeList() {
if (tsBalanceDnodeList != NULL) {
free(tsBalanceDnodeList);
tsBalanceDnodeList = NULL;
}
}
static void balanceCheckDnodeListSize(int32_t dnodesNum) {
if (tsBalanceDnodeListMallocSize <= dnodesNum) {
tsBalanceDnodeListMallocSize = dnodesNum * 2;
tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
}
}
void balanceAccquireDnodeList() {
int32_t dnodesNum = mnodeGetDnodesNum();
balanceCheckDnodeListSize(dnodesNum);
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
int32_t dnodeIndex = 0;
while (1) {
if (dnodeIndex >= dnodesNum) break;
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
mnodeDecDnodeRef(pDnode);
continue;
}
balanceCalcDnodeScore(pDnode);
int32_t orderIndex = dnodeIndex;
for (; orderIndex > 0; --orderIndex) {
if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
break;
}
tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
}
tsBalanceDnodeList[orderIndex] = pDnode;
dnodeIndex++;
}
sdbFreeIter(pIter);
tsBalanceDnodeListSize = dnodeIndex;
}
void balanceReleaseDnodeList() {
for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
SDnodeObj *pDnode = tsBalanceDnodeList[i];
if (pDnode != NULL) {
mnodeDecDnodeRef(pDnode);
}
}
}
static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->pAcct->user, "root") != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "id");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "system scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "custom scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "module scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "vnode scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "total scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "open vnodes");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "cpu cores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "balance state");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = mnodeGetDnodesNum();
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pIter = NULL;
mnodeDecUserRef(pUser);
return 0;
}
static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SDnodeObj *pDnode = NULL;
char * pWrite;
int32_t cols = 0;
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
if (pDnode == NULL) break;
int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
balanceCalcBandwidthScore(pDnode);
float moduleScore = balanceCalcModuleScore(pDnode);
float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDnode->dnodeId;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = systemScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = pDnode->customScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)moduleScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)vnodeScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDnode->openVnodes;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDnode->numOfCores;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
cols++;
numOfRows++;
mnodeDecDnodeRef(pDnode);
}
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows;
return numOfRows;
}
static void balanceMonitorDnodeModule() {
static void bnMonitorDnodeModule() {
int32_t numOfMnodes = mnodeGetMnodesNum();
if (numOfMnodes >= tsNumOfMnodes) return;
for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
SDnodeObj *pDnode = tsBalanceDnodeList[i];
for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
SDnodeObj *pDnode = tsBnDnodes.list[i];
if (pDnode == NULL) break;
if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
@ -990,7 +628,7 @@ static void balanceMonitorDnodeModule() {
}
}
int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
if (!sdbIsMaster()) {
mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
return TSDB_CODE_MND_DNODE_NOT_EXIST;
@ -1014,29 +652,29 @@ int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t
return TSDB_CODE_MND_DNODE_NOT_EXIST;
}
balanceLock();
balanceAccquireDnodeList();
bnLock();
bnAccquireDnodes();
int32_t code = TSDB_CODE_SUCCESS;
if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
dnodeId, pSrcDnode->dnodeId);
code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
} else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
} else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) {
mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
dnodeId, dnodeId);
code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
} else if (!balanceCheckFree(pDestDnode)) {
} else if (!bnCheckFree(pDestDnode)) {
mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
dnodeId);
code = TSDB_CODE_MND_DNODE_NOT_FREE;
} else {
code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
}
balanceReleaseDnodeList();
balanceUnLock();
bnReleaseDnodes();
bnUnLock();
mnodeDecVgroupRef(pVgroup);
mnodeDecDnodeRef(pDestDnode);

312
src/balance/src/bnScore.c Normal file
View File

@ -0,0 +1,312 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "mnodeShow.h"
#include "mnodeUser.h"
#include "bnScore.h"
SBnDnodes tsBnDnodes;
static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t bnCalcCpuScore(SDnodeObj *pDnode) {
if (pDnode->cpuAvgUsage < 80)
return 0;
else if (pDnode->cpuAvgUsage < 90)
return 10;
else
return 50;
}
static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) {
if (pDnode->memoryAvgUsage < 80)
return 0;
else if (pDnode->memoryAvgUsage < 90)
return 10;
else
return 50;
}
static int32_t bnCalcDiskScore(SDnodeObj *pDnode) {
if (pDnode->diskAvgUsage < 80)
return 0;
else if (pDnode->diskAvgUsage < 90)
return 10;
else
return 50;
}
static int32_t bnCalcBandScore(SDnodeObj *pDnode) {
if (pDnode->bandwidthUsage < 30)
return 0;
else if (pDnode->bandwidthUsage < 80)
return 10;
else
return 50;
}
static float bnCalcModuleScore(SDnodeObj *pDnode) {
if (pDnode->numOfCores <= 0) return 0;
if (pDnode->isMgmt) {
return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
}
return 0;
}
static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
if (pDnode->numOfCores <= 0) return 0;
return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
}
/**
* calc singe score, such as cpu/memory/disk/bandwitdh/vnode
* 1. get the score config
* 2. if the value is out of range, use border data
* 3. otherwise use interpolation method
**/
static void bnCalcDnodeScore(SDnodeObj *pDnode) {
pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) +
pDnode->customScore;
}
float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
bnCalcBandScore(pDnode);
float moduleScore = bnCalcModuleScore(pDnode);
float vnodeScore = bnCalcVnodeScore(pDnode, extra);
float score = systemScore + moduleScore + vnodeScore + pDnode->customScore;
return score;
}
void bnInitDnodes() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
memset(&tsBnDnodes, 0, sizeof(SBnDnodes));
tsBnDnodes.maxSize = 16;
tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *));
}
void bnCleanupDnodes() {
if (tsBnDnodes.list != NULL) {
free(tsBnDnodes.list);
tsBnDnodes.list = NULL;
}
}
static void bnCheckDnodesSize(int32_t dnodesNum) {
if (tsBnDnodes.maxSize <= dnodesNum) {
tsBnDnodes.maxSize = dnodesNum * 2;
tsBnDnodes.list = realloc(tsBnDnodes.list, tsBnDnodes.maxSize * sizeof(SDnodeObj *));
}
}
void bnAccquireDnodes() {
int32_t dnodesNum = mnodeGetDnodesNum();
bnCheckDnodesSize(dnodesNum);
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
int32_t dnodeIndex = 0;
while (1) {
if (dnodeIndex >= dnodesNum) {
mnodeCancelGetNextDnode(pIter);
break;
}
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
mnodeDecDnodeRef(pDnode);
continue;
}
bnCalcDnodeScore(pDnode);
int32_t orderIndex = dnodeIndex;
for (; orderIndex > 0; --orderIndex) {
if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) {
break;
}
tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1];
}
tsBnDnodes.list[orderIndex] = pDnode;
dnodeIndex++;
}
tsBnDnodes.size = dnodeIndex;
}
void bnReleaseDnodes() {
for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
SDnodeObj *pDnode = tsBnDnodes.list[i];
if (pDnode != NULL) {
mnodeDecDnodeRef(pDnode);
}
}
}
static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;
if (strcmp(pUser->pAcct->user, "root") != 0) {
mnodeDecUserRef(pUser);
return TSDB_CODE_MND_NO_RIGHTS;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "id");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "system scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "custom scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "module scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "vnode scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
strcpy(pSchema[cols].name, "total scores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "open vnodes");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "cpu cores");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "balance state");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = mnodeGetDnodesNum();
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
pShow->pIter = NULL;
mnodeDecUserRef(pUser);
return 0;
}
static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SDnodeObj *pDnode = NULL;
char * pWrite;
int32_t cols = 0;
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
if (pDnode == NULL) break;
int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode);
float moduleScore = bnCalcModuleScore(pDnode);
float vnodeScore = bnCalcVnodeScore(pDnode, 0);
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDnode->dnodeId;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = systemScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = pDnode->customScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)moduleScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)vnodeScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDnode->openVnodes;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDnode->numOfCores;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
cols++;
numOfRows++;
mnodeDecDnodeRef(pDnode);
}
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows;
return numOfRows;
}

132
src/balance/src/bnThread.c Normal file
View File

@ -0,0 +1,132 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "ttimer.h"
#include "tglobal.h"
#include "mnodeSdb.h"
#include "bnThread.h"
static SBnThread tsBnThread;
static void *bnThreadFunc(void *arg) {
while (1) {
pthread_mutex_lock(&tsBnThread.mutex);
if (tsBnThread.stop) {
pthread_mutex_unlock(&tsBnThread.mutex);
break;
}
pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
bool updateSoon = bnStart();
bnStartTimer(updateSoon ? 1000 : -1);
pthread_mutex_unlock(&(tsBnThread.mutex));
}
mDebug("balance thread is stopped");
return NULL;
}
int32_t bnInitThread() {
memset(&tsBnThread, 0, sizeof(SBnThread));
tsBnThread.stop = false;
pthread_mutex_init(&tsBnThread.mutex, NULL);
pthread_cond_init(&tsBnThread.cond, NULL);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL);
pthread_attr_destroy(&thattr);
if (ret != 0) {
mError("failed to create balance thread since %s", strerror(errno));
return -1;
}
bnStartTimer(2000);
mDebug("balance thread is created");
return 0;
}
void bnCleanupThread() {
mDebug("balance thread will be cleanup");
if (tsBnThread.timer != NULL) {
taosTmrStopA(&tsBnThread.timer);
tsBnThread.timer = NULL;
mDebug("stop balance timer");
}
pthread_mutex_lock(&tsBnThread.mutex);
tsBnThread.stop = true;
pthread_cond_signal(&tsBnThread.cond);
pthread_mutex_unlock(&(tsBnThread.mutex));
pthread_join(tsBnThread.thread, NULL);
pthread_cond_destroy(&tsBnThread.cond);
pthread_mutex_destroy(&tsBnThread.mutex);
}
static void bnPostSignal() {
if (tsBnThread.stop) return;
pthread_mutex_lock(&tsBnThread.mutex);
pthread_cond_signal(&tsBnThread.cond);
pthread_mutex_unlock(&(tsBnThread.mutex));
}
/*
* once sdb work as mater, then tsAccessSquence reset to zero
* increase tsAccessSquence every balance interval
*/
static void bnProcessTimer(void *handle, void *tmrId) {
if (!sdbIsMaster()) return;
if (tsBnThread.stop) return;
tsBnThread.timer = NULL;
tsAccessSquence++;
bnCheckStatus();
bnStartTimer(-1);
if (handle == NULL) {
if (tsAccessSquence % tsBalanceInterval == 0) {
mDebug("balance function is scheduled by timer");
bnPostSignal();
}
} else {
int64_t mseconds = (int64_t)handle;
mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
bnPostSignal();
}
}
void bnStartTimer(int64_t mseconds) {
if (tsBnThread.stop) return;
bool updateSoon = (mseconds != -1);
if (updateSoon) {
taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
} else {
taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
}
}
void bnNotify() {
bnStartTimer(500);
}

1
src/client/inc/tscLocalMerge.h
View File

@ -56,7 +56,6 @@ typedef struct SLocalReducer {
tFilePage * pTempBuffer;
struct SQLFunctionCtx *pCtx;
int32_t rowSize; // size of each intermediate result.
int32_t finalRowSize; // final result row size
int32_t status; // denote it is in reduce process, in reduce process, it
bool hasPrevRow; // cannot be released
bool hasUnprocessedRow;

11
src/client/inc/tsclient.h
View File

@ -246,11 +246,14 @@ typedef struct SQueryInfo {
int16_t fillType; // final result fill type
int16_t numOfTables;
STableMetaInfo **pTableMetaInfo;
struct STSBuf * tsBuf;
struct STSBuf *tsBuf;
int64_t * fillVal; // default value for fill
char * msg; // pointer to the pCmd->payload to keep error message temporarily
int64_t clauseLimit; // limit for current sub clause
int64_t prjOffset; // offset value in the original sql expression, only applied at client side
int64_t tableLimit; // table limit in case of super table projection query + global order + limit
int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
int16_t resColumnId; // result column id
} SQueryInfo;
@ -333,7 +336,7 @@ typedef struct STscObj {
char superAuth : 1;
uint32_t connId;
uint64_t rid; // ref ID returned by taosAddRef
struct SSqlObj * pHb;
int64_t hbrid;
struct SSqlObj * sqlList;
struct SSqlStream *streamList;
SRpcCorEpSet *tscCorMgmtEpSet;
@ -374,7 +377,7 @@ typedef struct SSqlObj {
struct SSqlObj **pSubs;
struct SSqlObj *prev, *next;
struct SSqlObj **self;
int64_t self;
} SSqlObj;
typedef struct SSqlStream {
@ -508,7 +511,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
}
extern SCacheObj* tscMetaCache;
extern SCacheObj* tscObjCache;
extern int tscObjRef;
extern void * tscTmr;
extern void * tscQhandle;
extern int tscKeepConn[];

401
src/client/src/tscFunctionImpl.c
View File

@ -64,13 +64,13 @@
} \
} while (0);
#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
do {\
for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \
SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \
aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \
} \
} while(0);
#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
do { \
for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \
SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \
aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \
} \
} while (0);
void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {}
void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {}
@ -426,8 +426,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) {
}
SET_VAL(pCtx, 1, 1);
*((int64_t *)pCtx->aOutputBuf) += 1;
*((int64_t *)pCtx->aOutputBuf) += pCtx->size;
// do not need it actually
SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
@ -3624,94 +3623,158 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes;
STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->lastKey = INT64_MIN;
pInfo->type = pCtx->inputType;
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->lastKey = INT64_MIN;
pInfo->win = TSWINDOW_INITIALIZER;
return true;
}
static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, int32_t i, STwaInfo *pInfo) {
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_INT:
pInfo->iLastValue = GET_INT32_VAL(data + pCtx->inputBytes * i);
break;
case TSDB_DATA_TYPE_TINYINT:
pInfo->iLastValue = GET_INT8_VAL(data + pCtx->inputBytes * i);
break;
case TSDB_DATA_TYPE_SMALLINT:
pInfo->iLastValue = GET_INT16_VAL(data + pCtx->inputBytes * i);
break;
case TSDB_DATA_TYPE_BIGINT:
pInfo->iLastValue = GET_INT64_VAL(data + pCtx->inputBytes * i);
break;
case TSDB_DATA_TYPE_FLOAT:
pInfo->dLastValue = GET_FLOAT_VAL(data + pCtx->inputBytes * i);
break;
case TSDB_DATA_TYPE_DOUBLE:
pInfo->dLastValue = GET_DOUBLE_VAL(data + pCtx->inputBytes * i);
break;
default:
assert(0);
static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) {
int32_t notNullElems = 0;
TSKEY *primaryKey = pCtx->ptsList;
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t i = index;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
if (pCtx->start.key != INT64_MIN) {
assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) ||
(pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC));
assert(pInfo->lastKey == INT64_MIN);
pInfo->lastKey = primaryKey[tsIndex + i];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pCtx->start.key;
notNullElems++;
i += step;
} else if (pInfo->lastKey == INT64_MIN) {
pInfo->lastKey = primaryKey[tsIndex + i];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pInfo->lastKey;
notNullElems++;
i += step;
}
// calculate the value of
switch(pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
for (; i < size && i >= 0; i += step) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + tsIndex];
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
for (; i < size && i >= 0; i += step) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + tsIndex];
}
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
for (; i < size && i >= 0; i += step) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + tsIndex];
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0);
for (; i < size && i >= 0; i += step) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
pInfo->lastValue = (double) val[i];
pInfo->lastKey = primaryKey[i + tsIndex];
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0);
for (; i < size && i >= 0; i += step) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + tsIndex];
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0);
for (; i < size && i >= 0; i += step) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + tsIndex];
}
break;
}
default: assert(0);
}
// the last interpolated time window value
if (pCtx->end.key != INT64_MIN) {
pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
pInfo->lastValue = pCtx->end.val;
pInfo->lastKey = pCtx->end.key;
}
pInfo->win.ekey = pInfo->lastKey;
return notNullElems;
}
static void twa_function(SQLFunctionCtx *pCtx) {
void * data = GET_INPUT_CHAR(pCtx);
TSKEY *primaryKey = pCtx->ptsList;
int32_t notNullElems = 0;
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo);
int32_t i = 0;
// skip null value
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1);
while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
i++;
i += step;
}
if (i >= pCtx->size) {
return;
}
if (pInfo->lastKey == INT64_MIN) {
pInfo->lastKey = pCtx->nStartQueryTimestamp;
setTWALastVal(pCtx, data, i, pInfo);
pInfo->hasResult = DATA_SET_FLAG;
}
notNullElems++;
if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey);
} else {
pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey);
}
pInfo->lastKey = primaryKey[i];
setTWALastVal(pCtx, data, i, pInfo);
for (++i; i < pCtx->size; i++) {
if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
continue;
}
notNullElems++;
if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey);
} else {
pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey);
}
pInfo->lastKey = primaryKey[i];
setTWALastVal(pCtx, data, i, pInfo);
}
int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size);
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
@ -3721,8 +3784,6 @@ static void twa_function(SQLFunctionCtx *pCtx) {
if (pCtx->stableQuery) {
memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo));
}
// pCtx->numOfIteratedElems += notNullElems;
}
static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
@ -3730,34 +3791,136 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) {
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
return;
}
SET_VAL(pCtx, 1, 1);
int32_t notNullElems = 0;
TSKEY *primaryKey = pCtx->ptsList;
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->lastKey == INT64_MIN) {
pInfo->lastKey = pCtx->nStartQueryTimestamp;
setTWALastVal(pCtx, pData, 0, pInfo);
int32_t i = pCtx->startOffset;
int32_t size = pCtx->size;
if (pCtx->start.key != INT64_MIN) {
assert(pInfo->lastKey == INT64_MIN);
pInfo->lastKey = primaryKey[index];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pCtx->start.key;
notNullElems++;
i += 1;
} else if (pInfo->lastKey == INT64_MIN) {
pInfo->lastKey = primaryKey[index];
GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index));
pInfo->hasResult = DATA_SET_FLAG;
pInfo->win.skey = pInfo->lastKey;
notNullElems++;
i += 1;
}
if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey);
} else {
pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey);
// calculate the value of
switch(pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
for (; i < size; i++) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + index];
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
for (; i < size; i++) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + index];
}
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
for (; i < size; i++) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + index];
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index);
for (; i < size; i++) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
pInfo->lastValue = (double) val[i];
pInfo->lastKey = primaryKey[i + index];
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index);
for (; i < size; i++) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + index];
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index);
for (; i < size; i++) {
if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
continue;
}
pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey);
pInfo->lastValue = val[i];
pInfo->lastKey = primaryKey[i + index];
}
break;
}
default: assert(0);
}
// record the last key/value
pInfo->lastKey = primaryKey[index];
setTWALastVal(pCtx, pData, 0, pInfo);
// pCtx->numOfIteratedElems += 1;
pResInfo->hasResult = DATA_SET_FLAG;
// the last interpolated time window value
if (pCtx->end.key != INT64_MIN) {
pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey);
pInfo->lastValue = pCtx->end.val;
pInfo->lastKey = pCtx->end.key;
}
pInfo->win.ekey = pInfo->lastKey;
SET_VAL(pCtx, notNullElems, 1);
if (notNullElems > 0) {
pResInfo->hasResult = DATA_SET_FLAG;
}
if (pCtx->stableQuery) {
memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo));
}
@ -3778,16 +3941,10 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) {
}
numOfNotNull++;
if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
pBuf->iOutput += pInput->iOutput;
} else {
pBuf->dOutput += pInput->dOutput;
}
pBuf->SKey = pInput->SKey;
pBuf->EKey = pInput->EKey;
pBuf->dOutput += pInput->dOutput;
pBuf->win = pInput->win;
pBuf->lastKey = pInput->lastKey;
pBuf->iLastValue = pInput->iLastValue;
}
SET_VAL(pCtx, numOfNotNull, 1);
@ -3814,21 +3971,17 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult);
if (pInfo->hasResult != DATA_SET_FLAG) {
setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
return;
}
if (pInfo->SKey == pInfo->EKey) {
*(double *)pCtx->aOutputBuf = 0;
} else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) {
pInfo->iOutput += pInfo->iLastValue * (pInfo->EKey - pInfo->lastKey);
*(double *)pCtx->aOutputBuf = pInfo->iOutput / (double)(pInfo->EKey - pInfo->SKey);
if (pInfo->win.ekey == pInfo->win.skey) {
*(double *)pCtx->aOutputBuf = pInfo->lastValue;
} else {
pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey);
*(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey);
*(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey);
}
GET_RES_INFO(pCtx)->numOfRes = 1;

7
src/client/src/tscLocal.c
View File

@ -825,8 +825,11 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) {
static int32_t tscProcessServStatus(SSqlObj *pSql) {
STscObj* pObj = pSql->pTscObj;
if (pObj->pHb != NULL) {
if (pObj->pHb->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
if (pHb != NULL) {
int32_t code = pHb->res.code;
taosReleaseRef(tscObjRef, pObj->hbrid);
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
return pSql->res.code;
}

31
src/client/src/tscLocalMerge.c
View File

@ -198,6 +198,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (numOfFlush == 0 || numOfBuffer == 0) {
tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer);
pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty
tscDebug("%p retrieved no data", pSql);
return;
}
@ -330,22 +331,19 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16;
pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage));
pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList);
pReducer->resColModel = finalmodel;
pReducer->resColModel->capacity = pReducer->nResultBufSize;
pReducer->finalModel = pFFModel;
assert(pReducer->finalRowSize > 0);
if (pReducer->finalRowSize > 0) {
pReducer->resColModel->capacity /= pReducer->finalRowSize;
if (finalmodel->rowSize > 0) {
pReducer->resColModel->capacity /= finalmodel->rowSize;
}
assert(pReducer->finalRowSize <= pReducer->rowSize);
assert(finalmodel->rowSize > 0 && finalmodel->rowSize <= pReducer->rowSize);
pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity);
if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
/*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
tfree(pReducer->pTempBuffer);
tfree(pReducer->discardData);
tfree(pReducer->pResultBuf);
@ -723,10 +721,16 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
// final result depends on the fields number
memset(pSchema, 0, sizeof(SSchema) * size);
for (int32_t i = 0; i < size; ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
SSchema p1 = {0};
if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) {
p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex);
} else {
p1 = tGetTableNameColumnSchema();
}
int32_t inter = 0;
int16_t type = -1;
@ -745,7 +749,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
functionId = TSDB_FUNC_LAST;
}
getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false);
int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false);
assert(ret == TSDB_CODE_SUCCESS);
}
pSchema[i].type = (uint8_t)type;
@ -920,7 +925,7 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo);
}
memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalRowSize));
memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalModel->rowSize));
pRes->numOfClauseTotal += pRes->numOfRows;
pBeforeFillData->num = 0;
@ -1256,7 +1261,7 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
tColModelCompact(pModel, pResBuf, pModel->capacity);
if (tscIsSecondStageQuery(pQueryInfo)) {
pLocalReducer->finalRowSize = doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalRowSize);
doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalModel->rowSize);
}
#ifdef _DEBUG_VIEW
@ -1627,7 +1632,8 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
}
int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
char* pbuf = calloc(1, pOutput->num * rowSize);
int32_t maxRowSize = MAX(rowSize, finalRowSize);
char* pbuf = calloc(1, pOutput->num * maxRowSize);
size_t size = tscNumOfFields(pQueryInfo);
SArithmeticSupport arithSup = {0};
@ -1660,7 +1666,6 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_
offset += pSup->field.bytes;
}
assert(finalRowSize <= rowSize);
memcpy(pOutput->data, pbuf, pOutput->num * offset);
tfree(pbuf);

2
src/client/src/tscProfile.c
View File

@ -39,6 +39,7 @@ void tscInitConnCb(void *param, TAOS_RES *result, int code) {
tscSlowQueryConnInitialized = true;
tscSaveSlowQueryFp(sql, NULL);
}
taos_free_result(result);
}
void tscAddIntoSqlList(SSqlObj *pSql) {
@ -69,6 +70,7 @@ void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) {
} else {
tscDebug("success to save slow query, code:%d", code);
}
taos_free_result(result);
}
void tscSaveSlowQueryFp(void *handle, void *tmrId) {

18
src/client/src/tscSQLParser.c
View File

@ -4248,7 +4248,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
tExprTreeDestroy(&p, NULL);
taosArrayDestroy(colList);
if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table");
}
}
@ -4256,6 +4256,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
pCondExpr->pTagCond = NULL;
return ret;
}
int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) {
if (pExpr == NULL) {
return TSDB_CODE_SUCCESS;
@ -5102,7 +5103,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) {
const int tokenDebugFlagEnd = 20;
const SDNodeDynConfOption cfgOptions[] = {
{"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7},
{"debugFlag", 9}, {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10},
{"debugFlag", 9}, {"monDebugFlag", 12}, {"vDebugFlag", 10}, {"mDebugFlag", 10},
{"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12},
{"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12},
{"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12},
@ -5306,15 +5307,18 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn
// keep original limitation value in globalLimit
pQueryInfo->clauseLimit = pQueryInfo->limit.limit;
pQueryInfo->prjOffset = pQueryInfo->limit.offset;
pQueryInfo->prjOffset = pQueryInfo->limit.offset;
pQueryInfo->tableLimit = -1;
if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
/*
* the limitation/offset value should be removed during retrieve data from virtual node,
* since the global order are done in client side, so the limitation should also
* be done at the client side.
* the offset value should be removed during retrieve data from virtual node, since the
* global order are done in client side, so the offset is applied at the client side
* However, note that the maximum allowed number of result for each table should be less
* than or equal to the value of limit.
*/
if (pQueryInfo->limit.limit > 0) {
pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset;
pQueryInfo->limit.limit = -1;
}
@ -6648,7 +6652,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
return TSDB_CODE_SUCCESS;
} else {
return TSDB_CODE_TSC_INVALID_SQL;
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression");
}
} else {

50
src/client/src/tscServer.c
View File

@ -165,10 +165,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));
}
} else {
tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code));
tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code));
}
if (pObj->pHb != NULL) {
if (pObj->hbrid != 0) {
int32_t waitingDuring = tsShellActivityTimer * 500;
tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
@ -183,20 +183,12 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
STscObj *pObj = taosAcquireRef(tscRefId, rid);
if (pObj == NULL) return;
SSqlObj* pHB = pObj->pHb;
void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
tscWarn("%p HB object has been released already", pHB);
taosReleaseRef(tscRefId, pObj->rid);
return;
}
assert(*pHB->self == pHB);
SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid);
assert(pHB->self == pObj->hbrid);
pHB->retry = 0;
int32_t code = tscProcessSql(pHB);
taosCacheRelease(tscObjCache, (void**) &p, false);
taosReleaseRef(tscObjRef, pObj->hbrid);
if (code != TSDB_CODE_SUCCESS) {
tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
@ -226,7 +218,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.msgType = pSql->cmd.msgType,
.pCont = pMsg,
.contLen = pSql->cmd.payloadLen,
.ahandle = pSql,
.ahandle = (void*)pSql->self,
.handle = NULL,
.code = 0
};
@ -237,26 +229,24 @@ int tscSendMsgToServer(SSqlObj *pSql) {
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle;
void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle);
if (pSql == NULL) {
rpcFreeCont(rpcMsg->pCont);
return;
}
SSqlObj* pSql = *p;
assert(pSql != NULL);
assert(pSql->self == handle);
STscObj *pObj = pSql->pTscObj;
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
assert(*pSql->self == pSql);
pSql->rpcRid = -1;
if (pObj->signature != pObj) {
tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
taosCacheRelease(tscObjCache, (void**) &p, true);
taosRemoveRef(tscObjRef, pSql->self);
taosReleaseRef(tscObjRef, pSql->self);
rpcFreeCont(rpcMsg->pCont);
return;
}
@ -266,10 +256,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
void** p1 = p;
taosCacheRelease(tscObjCache, (void**) &p1, false);
taosCacheRelease(tscObjCache, (void**) &p, true);
taosRemoveRef(tscObjRef, pSql->self);
taosReleaseRef(tscObjRef, pSql->self);
rpcFreeCont(rpcMsg->pCont);
return;
}
@ -312,7 +300,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
// if there is an error occurring, proceed to the following error handling procedure.
if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosCacheRelease(tscObjCache, (void**) &p, false);
taosReleaseRef(tscObjRef, pSql->self);
rpcFreeCont(rpcMsg->pCont);
return;
}
@ -380,11 +368,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
(*pSql->fp)(pSql->param, pSql, rpcMsg->code);
}
void** p1 = p;
taosCacheRelease(tscObjCache, (void**) &p1, false);
taosReleaseRef(tscObjRef, pSql->self);
if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it
taosCacheRelease(tscObjCache, (void **)&p, true);
taosRemoveRef(tscObjRef, pSql->self);
tscDebug("%p sqlObj is automatically freed", pSql);
}
@ -687,6 +674,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
pQueryMsg->numOfTags = htonl(numOfTags);
pQueryMsg->queryType = htonl(pQueryInfo->type);
pQueryMsg->tableLimit = htobe64(pQueryInfo->tableLimit);
size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo);
pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number
@ -2010,7 +1998,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
// TODO multithread problem
static void createHBObj(STscObj* pObj) {
if (pObj->pHb != NULL) {
if (pObj->hbrid != 0) {
return;
}
@ -2042,7 +2030,7 @@ static void createHBObj(STscObj* pObj) {
registerSqlObj(pSql);
tscDebug("%p HB is allocated, pObj:%p", pSql, pObj);
pObj->pHb = pSql;
pObj->hbrid = pSql->self;
}
int tscProcessConnectRsp(SSqlObj *pSql) {

18
src/client/src/tscSql.c
View File

@ -292,8 +292,8 @@ void taos_close(TAOS *taos) {
pObj->signature = NULL;
taosTmrStopA(&(pObj->pTimer));
SSqlObj* pHb = pObj->pHb;
if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
if (pHb != NULL) {
if (pHb->rpcRid > 0) { // wait for rsp from dnode
rpcCancelRequest(pHb->rpcRid);
pHb->rpcRid = -1;
@ -301,6 +301,7 @@ void taos_close(TAOS *taos) {
tscDebug("%p HB is freed", pHb);
taos_free_result(pHb);
taosReleaseRef(tscObjRef, pHb->self);
}
int32_t ref = T_REF_DEC(pObj);
@ -622,8 +623,7 @@ void taos_free_result(TAOS_RES *res) {
bool freeNow = tscKillQueryInDnode(pSql);
if (freeNow) {
tscDebug("%p free sqlObj in cache", pSql);
SSqlObj** p = pSql->self;
taosCacheRelease(tscObjCache, (void**) &p, true);
taosReleaseRef(tscObjRef, pSql->self);
}
}
@ -716,13 +716,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
continue;
}
void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) {
continue;
}
SSqlObj* pSubObj = (SSqlObj*) (*p);
assert(pSubObj->self == (SSqlObj**) p);
SSqlObj* pSubObj = pSub;
pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
if (pSubObj->rpcRid > 0) {
@ -731,7 +725,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
}
tscQueueAsyncRes(pSubObj);
taosCacheRelease(tscObjCache, (void**) &p, false);
taosReleaseRef(tscObjRef, pSubObj->self);
}
tscDebug("%p super table query cancelled", pSql);

95
src/client/src/tscSub.c
View File

@ -157,7 +157,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
registerSqlObj(pSql);
code = tsParseSql(pSql, false);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tsem_wait(&pSub->sem);
code = pSql->res.code;
@ -168,7 +168,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
goto fail;
}
if (pSql->cmd.command != TSDB_SQL_SELECT) {
if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
line = __LINE__;
code = TSDB_CODE_TSC_INVALID_SQL;
goto fail;
@ -179,10 +179,10 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
fail:
tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code));
if (pSql != NULL) {
if (pSql->self != NULL) {
taos_free_result(pSql);
if (pSql->self != 0) {
taosReleaseRef(tscObjRef, pSql->self);
} else {
tscFreeSqlObj(pSql);
tscFreeSqlObj(pSql);
}
pSql = NULL;
@ -401,9 +401,11 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
tscLoadSubscriptionProgress(pSub);
}
if (!tscUpdateSubscription(pObj, pSub)) {
taos_unsubscribe(pSub, 1);
return NULL;
if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) {
if (!tscUpdateSubscription(pObj, pSub)) {
taos_unsubscribe(pSub, 1);
return NULL;
}
}
pSub->interval = interval;
@ -417,10 +419,80 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char
return pSub;
}
SSqlObj* recreateSqlObj(SSub* pSub) {
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
return NULL;
}
pSql->signature = pSql;
pSql->pTscObj = pSub->taos;
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
tscFreeSqlObj(pSql);
return NULL;
}
pSql->param = pSub;
pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = asyncCallback;
pSql->fetchFp = asyncCallback;
pSql->sqlstr = strdup(pSub->pSql->sqlstr);
if (pSql->sqlstr == NULL) {
tscFreeSqlObj(pSql);
return NULL;
}
pRes->qhandle = 0;
pRes->numOfRows = 1;
int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
if (code != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pSql);
return NULL;
}
registerSqlObj(pSql);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
tsem_wait(&pSub->sem);
code = pSql->res.code;
}
if (code != TSDB_CODE_SUCCESS) {
taosReleaseRef(tscObjRef, pSql->self);
return NULL;
}
if (pSql->cmd.command != TSDB_SQL_SELECT) {
taosReleaseRef(tscObjRef, pSql->self);
return NULL;
}
return pSql;
}
TAOS_RES *taos_consume(TAOS_SUB *tsub) {
SSub *pSub = (SSub *)tsub;
if (pSub == NULL) return NULL;
if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
SSqlObj* pSql = recreateSqlObj(pSub);
if (pSql == NULL) {
return NULL;
}
if (pSub->pSql->self != 0) {
taosReleaseRef(tscObjRef, pSub->pSql->self);
} else {
tscFreeSqlObj(pSub->pSql);
}
pSub->pSql = pSql;
pSql->pSubscription = pSub;
}
tscSaveSubscriptionProgress(pSub);
SSqlObj *pSql = pSub->pSql;
@ -512,10 +584,13 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
}
if (pSub->pSql != NULL) {
taos_free_result(pSub->pSql);
if (pSub->pSql->self != 0) {
taosReleaseRef(tscObjRef, pSub->pSql->self);
} else {
tscFreeSqlObj(pSub->pSql);
}
}
tscFreeSqlObj(pSub->pSql);
taosArrayDestroy(pSub->progress);
tsem_destroy(&pSub->sem);
memset(pSub, 0, sizeof(*pSub));

3
src/client/src/tscSubquery.c
View File

@ -2198,6 +2198,9 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index);
int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
// free the data block created from insert sql string
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
tscQueueAsyncRes(pSql);
return code; // here the pSql may have been released already.

12
src/client/src/tscSystem.c
View File

@ -15,7 +15,7 @@
#include "os.h"
#include "taosmsg.h"
#include "tcache.h"
#include "tref.h"
#include "trpc.h"
#include "tsystem.h"
#include "ttimer.h"
@ -31,7 +31,7 @@
// global, not configurable
SCacheObj* tscMetaCache;
SCacheObj* tscObjCache;
int tscObjRef = -1;
void * tscTmr;
void * tscQhandle;
void * tscCheckDiskUsageTmr;
@ -139,7 +139,7 @@ void taos_init_imp(void) {
int64_t refreshTime = 10; // 10 seconds by default
if (tscMetaCache == NULL) {
tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta");
tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj");
tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj);
}
tscRefId = taosOpenRef(200, tscCloseTscObj);
@ -162,9 +162,9 @@ void taos_cleanup(void) {
taosCacheCleanup(m);
}
m = tscObjCache;
if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) {
taosCacheCleanup(m);
int refId = atomic_exchange_32(&tscObjRef, -1);
if (refId != -1) {
taosCloseRef(refId);
}
m = tscQhandle;

35
src/client/src/tscUtil.c
View File

@ -447,20 +447,18 @@ static void tscFreeSubobj(SSqlObj* pSql) {
void tscFreeRegisteredSqlObj(void *pSql) {
assert(pSql != NULL);
SSqlObj** p = (SSqlObj**)pSql;
STscObj* pTscObj = (*p)->pTscObj;
SSqlObj* p = *(SSqlObj**)pSql;
STscObj* pTscObj = p->pTscObj;
assert((*p)->self != 0 && (*p)->self == (p));
SSqlObj* ptr = *p;
tscFreeSqlObj(*p);
assert(p->self != 0);
tscFreeSqlObj(p);
int32_t ref = T_REF_DEC(pTscObj);
assert(ref >= 0);
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", ptr, pTscObj, ref);
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref);
if (ref == 0) {
tscDebug("%p all sqlObj freed, free tscObj:%p", ptr, pTscObj);
tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj);
taosRemoveRef(tscRefId, pTscObj->rid);
}
}
@ -840,7 +838,6 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
// the length does not include the SSubmitBlk structure
pBlocks->dataLen = htonl(finalLen);
dataBuf->numOfTables += 1;
}
@ -1565,19 +1562,6 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) {
}
}
void tscSetFreeHeatBeat(STscObj* pObj) {
if (pObj == NULL || pObj->signature != pObj || pObj->pHb == NULL) {
return;
}
SSqlObj* pHeatBeat = pObj->pHb;
assert(pHeatBeat == pHeatBeat->signature);
// to denote the heart-beat timer close connection and free all allocated resources
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHeatBeat->cmd, 0);
pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
}
/*
* the following four kinds of SqlObj should not be freed
* 1. SqlObj for stream computing
@ -1596,7 +1580,7 @@ bool tscShouldBeFreed(SSqlObj* pSql) {
}
STscObj* pTscObj = pSql->pTscObj;
if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) {
if (pSql->pStream != NULL || pTscObj->hbrid == pSql->self || pSql->pSubscription != NULL) {
return false;
}
@ -1888,13 +1872,10 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
}
void registerSqlObj(SSqlObj* pSql) {
int32_t DEFAULT_LIFE_TIME = 2 * 600 * 1000; // 1200 sec
int32_t ref = T_REF_INC(pSql->pTscObj);
tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref);
TSDB_CACHE_PTR_TYPE p = (TSDB_CACHE_PTR_TYPE) pSql;
pSql->self = taosCachePut(tscObjCache, &p, sizeof(TSDB_CACHE_PTR_TYPE), &p, sizeof(TSDB_CACHE_PTR_TYPE), DEFAULT_LIFE_TIME);
pSql->self = taosAddRef(tscObjRef, pSql);
}
SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {

5
src/common/inc/tglobal.h
View File

@ -125,6 +125,9 @@ extern char tsMonitorDbName[];
extern char tsInternalPass[];
extern int32_t tsMonitorInterval;
// stream
extern int32_t tsEnableStream;
// internal
extern int32_t tsPrintAuth;
extern int32_t tscEmbedded;
@ -176,7 +179,7 @@ extern int32_t tmrDebugFlag;
extern int32_t sdbDebugFlag;
extern int32_t httpDebugFlag;
extern int32_t mqttDebugFlag;
extern int32_t monitorDebugFlag;
extern int32_t monDebugFlag;
extern int32_t uDebugFlag;
extern int32_t rpcDebugFlag;
extern int32_t odbcDebugFlag;

41
src/common/src/tglobal.c
View File

@ -161,6 +161,9 @@ char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
char tsInternalPass[] = "secretkey";
int32_t tsMonitorInterval = 30; // seconds
// stream
int32_t tsEnableStream = 1;
// internal
int32_t tsPrintAuth = 0;
int32_t tscEmbedded = 0;
@ -200,13 +203,13 @@ int32_t tsNumOfLogLines = 10000000;
int32_t mDebugFlag = 135;
int32_t sdbDebugFlag = 135;
int32_t dDebugFlag = 135;
int32_t vDebugFlag = 131;
int32_t vDebugFlag = 135;
int32_t cDebugFlag = 131;
int32_t jniDebugFlag = 131;
int32_t odbcDebugFlag = 131;
int32_t httpDebugFlag = 131;
int32_t mqttDebugFlag = 131;
int32_t monitorDebugFlag = 131;
int32_t monDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 131;
int32_t uDebugFlag = 131;
@ -216,9 +219,9 @@ int32_t wDebugFlag = 135;
int32_t tsdbDebugFlag = 131;
int32_t cqDebugFlag = 135;
int32_t (*monitorStartSystemFp)() = NULL;
void (*monitorStopSystemFp)() = NULL;
void (*monitorExecuteSQLFp)(char *sql) = NULL;
int32_t (*monStartSystemFp)() = NULL;
void (*monStopSystemFp)() = NULL;
void (*monExecuteSQLFp)(char *sql) = NULL;
char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
@ -235,7 +238,7 @@ void taosSetAllDebugFlag() {
odbcDebugFlag = debugFlag;
httpDebugFlag = debugFlag;
mqttDebugFlag = debugFlag;
monitorDebugFlag = debugFlag;
monDebugFlag = debugFlag;
qDebugFlag = debugFlag;
rpcDebugFlag = debugFlag;
uDebugFlag = debugFlag;
@ -276,15 +279,15 @@ bool taosCfgDynamicOptions(char *msg) {
if (strncasecmp(cfg->option, "monitor", olen) == 0) {
if (1 == vint) {
if (monitorStartSystemFp) {
(*monitorStartSystemFp)();
if (monStartSystemFp) {
(*monStartSystemFp)();
uInfo("monitor is enabled");
} else {
uError("monitor can't be updated, for monitor not initialized");
}
} else {
if (monitorStopSystemFp) {
(*monitorStopSystemFp)();
if (monStopSystemFp) {
(*monStopSystemFp)();
uInfo("monitor is disabled");
} else {
uError("monitor can't be updated, for monitor not initialized");
@ -307,8 +310,8 @@ bool taosCfgDynamicOptions(char *msg) {
}
if (strncasecmp(option, "resetQueryCache", 15) == 0) {
if (monitorExecuteSQLFp) {
(*monitorExecuteSQLFp)("resetQueryCache");
if (monExecuteSQLFp) {
(*monExecuteSQLFp)("resetQueryCache");
uInfo("resetquerycache is executed");
} else {
uError("resetquerycache can't be executed, for monitor not started");
@ -1015,6 +1018,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "stream";
cfg.ptr = &tsEnableStream;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "httpEnableRecordSql";
cfg.ptr = &tsHttpEnableRecordSql;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1227,8 +1240,8 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "monitorDebugFlag";
cfg.ptr = &monitorDebugFlag;
cfg.option = "monDebugFlag";
cfg.ptr = &monDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;

2
src/connector/go

@ -1 +1 @@
Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f
Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766

2
src/connector/grafanaplugin

@ -1 +1 @@
Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944
Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c

48
src/cq/src/cqMain.c
View File

@ -40,15 +40,14 @@
typedef struct {
int32_t vgId;
int32_t master;
int32_t num; // number of continuous streams
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_NAME_LEN];
FCqWrite cqWrite;
void *ahandle;
int32_t num; // number of continuous streams
struct SCqObj *pHead;
void *dbConn;
int32_t master;
void *tmrCtrl;
pthread_mutex_t mutex;
} SCqContext;
@ -70,6 +69,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row);
static void cqCreateStream(SCqContext *pContext, SCqObj *pObj);
void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
if (tsEnableStream == 0) {
return NULL;
}
SCqContext *pContext = calloc(sizeof(SCqContext), 1);
if (pContext == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
@ -90,7 +92,6 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
tstrncpy(pContext->db, db, sizeof(pContext->db));
pContext->vgId = pCfg->vgId;
pContext->cqWrite = pCfg->cqWrite;
pContext->ahandle = ahandle;
tscEmbedded = 1;
pthread_mutex_init(&pContext->mutex, NULL);
@ -101,6 +102,9 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
}
void cqClose(void *handle) {
if (tsEnableStream == 0) {
return;
}
SCqContext *pContext = handle;
if (handle == NULL) return;
@ -131,6 +135,9 @@ void cqClose(void *handle) {
}
void cqStart(void *handle) {
if (tsEnableStream == 0) {
return;
}
SCqContext *pContext = handle;
if (pContext->dbConn || pContext->master) return;
@ -149,6 +156,9 @@ void cqStart(void *handle) {
}
void cqStop(void *handle) {
if (tsEnableStream == 0) {
return;
}
SCqContext *pContext = handle;
cInfo("vgId:%d, stop all CQs", pContext->vgId);
if (pContext->dbConn == NULL || pContext->master == 0) return;
@ -176,6 +186,9 @@ void cqStop(void *handle) {
}
void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
if (tsEnableStream == 0) {
return NULL;
}
SCqContext *pContext = handle;
SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@ -205,6 +218,9 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *
}
void cqDrop(void *handle) {
if (tsEnableStream == 0) {
return;
}
SCqObj *pObj = handle;
SCqContext *pContext = pObj->pContext;
@ -241,8 +257,12 @@ static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
SCqObj* pObj = (SCqObj*)param;
SCqContext* pContext = pObj->pContext;
SSqlObj* pSql = (SSqlObj*)result;
pContext->dbConn = pSql->pTscObj;
if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) {
taos_close(pSql->pTscObj);
}
pthread_mutex_lock(&pContext->mutex);
cqCreateStream(pContext, pObj);
pthread_mutex_unlock(&pContext->mutex);
}
static void cqProcessCreateTimer(void *param, void *tmrId) {
@ -253,7 +273,9 @@ static void cqProcessCreateTimer(void *param, void *tmrId) {
cDebug("vgId:%d, try connect to TDengine", pContext->vgId);
taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL);
} else {
pthread_mutex_lock(&pContext->mutex);
cqCreateStream(pContext, pObj);
pthread_mutex_unlock(&pContext->mutex);
}
}
@ -267,12 +289,14 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
}
pObj->tmrId = 0;
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
if (pObj->pStream) {
pContext->num++;
cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
if (pObj->pStream == NULL) {
pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL);
if (pObj->pStream) {
pContext->num++;
cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
} else {
cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
}
}
}
@ -334,7 +358,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
pHead->version = 0;
// write into vnode write queue
pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL);
pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL);
free(buffer);
}

2
src/cq/test/cqtest.c
View File

@ -24,7 +24,7 @@
int64_t ver = 0;
void *pCq = NULL;
int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
int writeToQueue(int32_t vgId, void *data, int type, void *pMsg) {
return 0;
}

2
src/dnode/src/dnodeMain.c
View File

@ -19,6 +19,7 @@
#include "tutil.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tfile.h"
#include "twal.h"
#include "trpc.h"
#include "dnode.h"
@ -55,6 +56,7 @@ typedef struct {
} SDnodeComponent;
static const SDnodeComponent tsDnodeComponents[] = {
{"tfile", tfInit, tfCleanup},
{"rpc", rpcInit, rpcCleanup},
{"storage", dnodeInitStorage, dnodeCleanupStorage},
{"dnodecfg", dnodeInitCfg, dnodeCleanupCfg},

6
src/dnode/src/dnodeMgmt.c
View File

@ -24,7 +24,7 @@
#include "tqueue.h"
#include "tsync.h"
#include "ttimer.h"
#include "tbalance.h"
#include "tbn.h"
#include "tglobal.h"
#include "dnode.h"
#include "vnode.h"
@ -444,12 +444,12 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
SCreateMnodeMsg *pCfg = pMsg->pCont;
pCfg->dnodeId = htonl(pCfg->dnodeId);
if (pCfg->dnodeId != dnodeGetDnodeId()) {
dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId());
return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
}
if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) {
dError("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp);
return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED;
}

8
src/dnode/src/dnodeModule.c
View File

@ -78,10 +78,10 @@ static void dnodeAllocModules() {
tsModule[TSDB_MOD_MONITOR].enable = (tsEnableMonitorModule == 1);
tsModule[TSDB_MOD_MONITOR].name = "monitor";
tsModule[TSDB_MOD_MONITOR].initFp = monitorInitSystem;
tsModule[TSDB_MOD_MONITOR].cleanUpFp = monitorCleanUpSystem;
tsModule[TSDB_MOD_MONITOR].startFp = monitorStartSystem;
tsModule[TSDB_MOD_MONITOR].stopFp = monitorStopSystem;
tsModule[TSDB_MOD_MONITOR].initFp = monInitSystem;
tsModule[TSDB_MOD_MONITOR].cleanUpFp = monCleanupSystem;
tsModule[TSDB_MOD_MONITOR].startFp = monStartSystem;
tsModule[TSDB_MOD_MONITOR].stopFp = monStopSystem;
if (tsEnableMonitorModule) {
dnodeSetModuleStatus(TSDB_MOD_MONITOR);
}

2
src/dnode/src/dnodePeer.c
View File

@ -182,6 +182,8 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
SRpcEpSet epSet = {0};
dnodeGetEpSetForPeer(&epSet);
assert(tsClientRpc != 0);
rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
}

14
src/inc/monitor.h
View File

@ -47,13 +47,13 @@ typedef struct {
int8_t accessState;
} SAcctMonitorObj;
int32_t monitorInitSystem();
int32_t monitorStartSystem();
void monitorStopSystem();
void monitorCleanUpSystem();
void monitorSaveAcctLog(SAcctMonitorObj *pMonObj);
void monitorSaveLog(int32_t level, const char *const format, ...);
void monitorExecuteSQL(char *sql);
int32_t monInitSystem();
int32_t monStartSystem();
void monStopSystem();
void monCleanupSystem();
void monSaveAcctLog(SAcctMonitorObj *pMonObj);
void monSaveLog(int32_t level, const char *const format, ...);
void monExecuteSQL(char *sql);
#ifdef __cplusplus
}

2
src/inc/taosdef.h
View File

@ -257,7 +257,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
#define TSDB_MAX_SQL_SHOW_LEN 512
#define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 8mb
#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024U) // sql length should be less than 1mb
#define TSDB_APPNAME_LEN TSDB_UNI_LEN

5
src/inc/taoserror.h
View File

@ -184,6 +184,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many d
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, 0, 0x0390, "Invalid database option: days out of range")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, 0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days")
// dnode
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out of memory")
@ -367,6 +370,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string")
// odbc
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input")
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined")
@ -390,7 +394,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, 0, 0x2113, "src bad se
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete")
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general")
#ifdef TAOS_ERROR_C
};
#endif

16
src/inc/taosmsg.h
View File

@ -476,19 +476,21 @@ typedef struct {
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx
int64_t tableLimit; // limit the number of rows for each table, used in order by + limit in stable projection query.
int16_t prjOrder; // global order in super table projection query.
int64_t limit;
int64_t offset;
uint32_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t tagNameRelType; // relation of tag criteria and tbname criteria
int16_t fillType; // interpolate type
uint64_t fillVal; // default value array list
int16_t fillType; // interpolate type
uint64_t fillVal; // default value array list
int32_t secondStageOutput;
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
int32_t tsOrder; // ts comp block order
int32_t numOfTags; // number of tags columns involved
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
int32_t tsOrder; // ts comp block order
int32_t numOfTags; // number of tags columns involved
SColumnInfo colList[];
} SQueryTableMsg;

16
src/inc/tbalance.h → src/inc/tbn.h
View File

@ -23,14 +23,14 @@ extern "C" {
struct SVgObj;
struct SDnodeObj;
int32_t balanceInit();
void balanceCleanUp();
void balanceAsyncNotify();
void balanceSyncNotify();
void balanceReset();
int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
int32_t balanceDropDnode(struct SDnodeObj *pDnode);
int32_t bnInit();
void bnCleanUp();
void bnNotify();
void bnCheckModules();
void bnReset();
int32_t bnAllocVnodes(struct SVgObj *pVgroup);
int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
int32_t bnDropDnode(struct SDnodeObj *pDnode);
#ifdef __cplusplus
}

2
src/inc/tcq.h
View File

@ -21,7 +21,7 @@ extern "C" {
#include "tdataformat.h"
typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
typedef struct {
int32_t vgId;

4
src/inc/tsync.h
View File

@ -85,6 +85,9 @@ typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
// when data file is synced successfully, notity app
typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
// get file version
typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
typedef struct {
int32_t vgId; // vgroup ID
uint64_t version; // initial version
@ -97,6 +100,7 @@ typedef struct {
FNotifyRole notifyRole;
FNotifyFlowCtrl notifyFlowCtrl;
FNotifyFileSynced notifyFileSynced;
FGetVersion getVersion;
} SSyncInfo;
typedef void *tsync_h;

5
src/inc/twal.h
View File

@ -51,9 +51,8 @@ typedef struct {
typedef void * twalh; // WAL HANDLE
typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
int32_t walInit();
void walCleanUp();
int32_t walInit();
void walCleanUp();
twalh walOpen(char *path, SWalCfg *pCfg);
int32_t walAlter(twalh pWal, SWalCfg *pCfg);
void walStop(twalh);

11
src/kit/shell/src/shellDarwin.c
View File

@ -407,7 +407,11 @@ void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid
void clearScreen(int ecmd_pos, int cursor_pos) {
struct winsize w;
ioctl(0, TIOCGWINSZ, &w);
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
//fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
@ -425,8 +429,9 @@ void clearScreen(int ecmd_pos, int cursor_pos) {
void showOnScreen(Command *cmd) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
fprintf(stderr, "No stream device\n");
exit(EXIT_FAILURE);
//fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
wchar_t wc;

11
src/kit/shell/src/shellLinux.c
View File

@ -413,7 +413,11 @@ void get_history_path(char *history) { snprintf(history, TSDB_FILENAME_LEN, "%s/
void clearScreen(int ecmd_pos, int cursor_pos) {
struct winsize w;
ioctl(0, TIOCGWINSZ, &w);
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
//fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
w.ws_col = 120;
w.ws_row = 30;
}
int cursor_x = cursor_pos / w.ws_col;
int cursor_y = cursor_pos % w.ws_col;
@ -431,8 +435,9 @@ void clearScreen(int ecmd_pos, int cursor_pos) {
void showOnScreen(Command *cmd) {
struct winsize w;
if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
fprintf(stderr, "No stream device\n");
exit(EXIT_FAILURE);
//fprintf(stderr, "No stream device\n");
w.ws_col = 120;
w.ws_row = 30;
}
wchar_t wc;

1
src/mnode/inc/mnodeAcct.h
View File

@ -27,6 +27,7 @@ void mnodeCleanupAccts();
void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo);
void * mnodeGetAcct(char *acctName);
void * mnodeGetNextAcct(void *pIter, SAcctObj **pAcct);
void mnodeCancelGetNextAcct(void *pIter);
void mnodeIncAcctRef(SAcctObj *pAcct);
void mnodeDecAcctRef(SAcctObj *pAcct);
void mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);

1
src/mnode/inc/mnodeDb.h
View File

@ -34,6 +34,7 @@ int64_t mnodeGetDbNum();
SDbObj *mnodeGetDb(char *db);
SDbObj *mnodeGetDbByTableId(char *db);
void * mnodeGetNextDb(void *pIter, SDbObj **pDb);
void mnodeCancelGetNextDb(void *pIter);
void mnodeIncDbRef(SDbObj *pDb);
void mnodeDecDbRef(SDbObj *pDb);
bool mnodeCheckIsMonitorDB(char *db, char *monitordb);

1
src/mnode/inc/mnodeDnode.h
View File

@ -65,6 +65,7 @@ int32_t mnodeGetDnodesNum();
int32_t mnodeGetOnlinDnodesCpuCoreNum();
int32_t mnodeGetOnlineDnodesNum();
void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
void mnodeCancelGetNextDnode(void *pIter);
void mnodeIncDnodeRef(SDnodeObj *pDnode);
void mnodeDecDnodeRef(SDnodeObj *pDnode);
void * mnodeGetDnode(int32_t dnodeId);

6
src/mnode/inc/mnodeInt.h
View File

@ -41,9 +41,9 @@ extern int32_t sdbDebugFlag;
#define sdbDebug(...) { if (sdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
#define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }}
#define mLError(...) { monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
#define mLWarn(...) { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) }
#define mLInfo(...) { monitorSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) }
#define mLError(...) { monSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) }
#define mLWarn(...) { monSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) }
#define mLInfo(...) { monSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) }
#ifdef __cplusplus
}

1
src/mnode/inc/mnodeMnode.h
View File

@ -38,6 +38,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId);
void * mnodeGetMnode(int32_t mnodeId);
int32_t mnodeGetMnodesNum();
void * mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
void mnodeCancelGetNextMnode(void *pIter);
void mnodeIncMnodeRef(struct SMnodeObj *pMnode);
void mnodeDecMnodeRef(struct SMnodeObj *pMnode);

13
src/mnode/inc/mnodeSdb.h
View File

@ -79,10 +79,13 @@ typedef struct {
int32_t (*fpRestored)();
} SSdbTableDesc;
int32_t sdbInitRef();
void sdbCleanUpRef();
int32_t sdbInit();
void sdbCleanUp();
void * sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(void *handle);
int64_t sdbOpenTable(SSdbTableDesc *desc);
void sdbCloseTable(int64_t rid);
void* sdbGetTableByRid(int64_t rid);
bool sdbIsMaster();
bool sdbIsServing();
void sdbUpdateMnodeRoles();
@ -92,9 +95,9 @@ int32_t sdbDeleteRow(SSdbRow *pRow);
int32_t sdbUpdateRow(SSdbRow *pRow);
int32_t sdbInsertRowToQueue(SSdbRow *pRow);
void *sdbGetRow(void *pTable, void *key);
void *sdbFetchRow(void *pTable, void *pIter, void **ppRow);
void sdbFreeIter(void *pIter);
void * sdbGetRow(void *pTable, void *key);
void * sdbFetchRow(void *pTable, void *pIter, void **ppRow);
void sdbFreeIter(void *pTable, void *pIter);
void sdbIncRef(void *pTable, void *pRow);
void sdbDecRef(void *pTable, void *pRow);
int64_t sdbGetNumOfRows(void *pTable);

2
src/mnode/inc/mnodeShow.h
View File

@ -26,8 +26,10 @@ void mnodeCleanUpShow();
typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
typedef void (*SShowFreeIterFp)(void *pIter);
void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp);
void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp);
void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow);
#ifdef __cplusplus

2
src/mnode/inc/mnodeTable.h
View File

@ -31,6 +31,8 @@ void mnodeIncTableRef(void *pTable);
void mnodeDecTableRef(void *pTable);
void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
void mnodeCancelGetNextChildTable(void *pIter);
void mnodeCancelGetNextSuperTable(void *pIter);
void mnodeDropAllChildTables(SDbObj *pDropDb);
void mnodeDropAllSuperTables(SDbObj *pDropDb);
void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);

1
src/mnode/inc/mnodeUser.h
View File

@ -25,6 +25,7 @@ int32_t mnodeInitUsers();
void mnodeCleanupUsers();
SUserObj *mnodeGetUser(char *name);
void * mnodeGetNextUser(void *pIter, SUserObj **pUser);
void mnodeCancelGetNextUser(void *pIter);
void mnodeIncUserRef(SUserObj *pUser);
void mnodeDecUserRef(SUserObj *pUser);
SUserObj *mnodeGetUserFromConn(void *pConn);

1
src/mnode/inc/mnodeVgroup.h
View File

@ -34,6 +34,7 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode);
//void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
void mnodeCancelGetNextVgroup(void *pIter);
void mnodeUpdateVgroup(SVgObj *pVgroup);
void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload);
void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes);

12
src/mnode/src/mnodeAcct.c
View File

@ -26,6 +26,7 @@
#include "mnodeUser.h"
#include "mnodeVgroup.h"
int64_t tsAcctRid = -1;
void * tsAcctSdb = NULL;
static int32_t tsAcctUpdateSize;
static int32_t mnodeCreateRootAcct();
@ -114,7 +115,8 @@ int32_t mnodeInitAccts() {
.fpRestored = mnodeAcctActionRestored
};
tsAcctSdb = sdbOpenTable(&desc);
tsAcctRid = sdbOpenTable(&desc);
tsAcctSdb = sdbGetTableByRid(tsAcctRid);
if (tsAcctSdb == NULL) {
mError("table:%s, failed to create hash", desc.name);
return -1;
@ -126,7 +128,7 @@ int32_t mnodeInitAccts() {
void mnodeCleanupAccts() {
acctCleanUp();
sdbCloseTable(tsAcctSdb);
sdbCloseTable(tsAcctRid);
tsAcctSdb = NULL;
}
@ -144,7 +146,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries;
mnodeDecAcctRef(pAcct);
}
sdbFreeIter(pIter);
SVgObj *pVgroup = NULL;
pIter = NULL;
@ -158,7 +159,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
pAcctInfo->totalPoints += pVgroup->pointsWritten;
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
}
void *mnodeGetAcct(char *name) {
@ -169,6 +169,10 @@ void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) {
return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct);
}
void mnodeCancelGetNextAcct(void *pIter) {
sdbFreeIter(tsAcctSdb, pIter);
}
void mnodeIncAcctRef(SAcctObj *pAcct) {
sdbIncRef(tsAcctSdb, pAcct);
}

16
src/mnode/src/mnodeCluster.c
View File

@ -24,6 +24,7 @@
#include "mnodeShow.h"
#include "tglobal.h"
int64_t tsClusterRid = -1;
static void * tsClusterSdb = NULL;
static int32_t tsClusterUpdateSize;
static char tsClusterId[TSDB_CLUSTER_ID_LEN];
@ -31,6 +32,7 @@ static int32_t mnodeCreateCluster();
static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mnodeCancelGetNextCluster(void *pIter);
static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) {
tfree(pRow->pObj);
@ -100,21 +102,23 @@ int32_t mnodeInitCluster() {
.fpRestored = mnodeClusterActionRestored
};
tsClusterSdb = sdbOpenTable(&desc);
tsClusterRid = sdbOpenTable(&desc);
tsClusterSdb = sdbGetTableByRid(tsClusterRid);
if (tsClusterSdb == NULL) {
mError("table:%s, failed to create hash", desc.name);
mError("table:%s, rid:%" PRId64 ", failed to create hash", desc.name, tsClusterRid);
return -1;
}
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster);
mDebug("table:%s, hash is created", desc.name);
return TSDB_CODE_SUCCESS;
}
void mnodeCleanupCluster() {
sdbCloseTable(tsClusterSdb);
sdbCloseTable(tsClusterRid);
tsClusterSdb = NULL;
}
@ -122,6 +126,10 @@ void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) {
return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster);
}
void mnodeCancelGetNextCluster(void *pIter) {
sdbFreeIter(tsClusterSdb, pIter);
}
void mnodeIncClusterRef(SClusterObj *pCluster) {
sdbIncRef(tsClusterSdb, pCluster);
}
@ -167,7 +175,7 @@ void mnodeUpdateClusterId() {
}
mnodeDecClusterRef(pCluster);
sdbFreeIter(pIter);
mnodeCancelGetNextCluster(pIter);
}
static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {

36
src/mnode/src/mnodeDb.c
View File

@ -20,7 +20,7 @@
#include "tgrant.h"
#include "tglobal.h"
#include "tname.h"
#include "tbalance.h"
#include "tbn.h"
#include "tdataformat.h"
#include "mnode.h"
#include "mnodeDef.h"
@ -38,6 +38,7 @@
#include "mnodeVgroup.h"
#define VG_LIST_SIZE 8
int64_t tsDbRid = -1;
static void * tsDbSdb = NULL;
static int32_t tsDbUpdateSize;
@ -160,7 +161,8 @@ int32_t mnodeInitDbs() {
.fpRestored = mnodeDbActionRestored
};
tsDbSdb = sdbOpenTable(&desc);
tsDbRid = sdbOpenTable(&desc);
tsDbSdb = sdbGetTableByRid(tsDbRid);
if (tsDbSdb == NULL) {
mError("failed to init db data");
return -1;
@ -171,6 +173,7 @@ int32_t mnodeInitDbs() {
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb);
mDebug("table:dbs table is created");
return 0;
@ -180,6 +183,10 @@ void *mnodeGetNextDb(void *pIter, SDbObj **pDb) {
return sdbFetchRow(tsDbSdb, pIter, (void **)pDb);
}
void mnodeCancelGetNextDb(void *pIter) {
sdbFreeIter(tsDbSdb, pIter);
}
SDbObj *mnodeGetDb(char *db) {
return (SDbObj *)sdbGetRow(tsDbSdb, db);
}
@ -229,30 +236,28 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE,
TSDB_MAX_DAYS_PER_FILE);
return TSDB_CODE_MND_INVALID_DB_OPTION;
return TSDB_CODE_MND_INVALID_DB_OPTION_DAYS;
}
if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) {
mError("invalid db option daysToKeep:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
return TSDB_CODE_MND_INVALID_DB_OPTION;
return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
}
if (pCfg->daysToKeep < pCfg->daysPerFile) {
mError("invalid db option daysToKeep:%d should larger than daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile);
return TSDB_CODE_MND_INVALID_DB_OPTION;
return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
}
#if 0
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > pCfg->daysToKeep) {
mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, pCfg->daysToKeep);
return TSDB_CODE_MND_INVALID_DB_OPTION;
mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep2, TSDB_MIN_KEEP, pCfg->daysToKeep);
return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
}
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) {
mError("invalid db option daysToKeep1:%d valid range: [%d, %d]", pCfg->daysToKeep1, TSDB_MIN_KEEP, pCfg->daysToKeep2);
return TSDB_CODE_MND_INVALID_DB_OPTION;
return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP;
}
#endif
if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock,
@ -491,7 +496,7 @@ void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) {
}
void mnodeCleanupDbs() {
sdbCloseTable(tsDbSdb);
sdbCloseTable(tsDbRid);
tsDbSdb = NULL;
}
@ -986,8 +991,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
SDbObj *pDb = pMsg->pDb;
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
SVgObj *pVgroup = NULL;
while (1) {
pIter = mnodeGetNextVgroup(pIter, &pVgroup);
if (pVgroup == NULL) break;
if (pVgroup->pDb == pDb) {
@ -995,12 +1000,11 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
}
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mDebug("db:%s, all vgroups is altered", pDb->name);
mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
balanceAsyncNotify();
bnNotify();
return TSDB_CODE_SUCCESS;
}
@ -1146,7 +1150,5 @@ void mnodeDropAllDbs(SAcctObj *pAcct) {
mnodeDecDbRef(pDb);
}
sdbFreeIter(pIter);
mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs);
}

57
src/mnode/src/mnodeDnode.c
View File

@ -16,12 +16,12 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tgrant.h"
#include "tbalance.h"
#include "tbn.h"
#include "tglobal.h"
#include "tconfig.h"
#include "tutil.h"
#include "tsocket.h"
#include "tbalance.h"
#include "tbn.h"
#include "tsync.h"
#include "tdataformat.h"
#include "mnode.h"
@ -39,6 +39,7 @@
#include "mnodeCluster.h"
int32_t tsAccessSquence = 0;
int64_t tsDnodeRid = -1;
static void * tsDnodeSdb = NULL;
static int32_t tsDnodeUpdateSize = 0;
extern void * tsMnodeSdb;
@ -114,7 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) {
mnodeDropAllDnodeVgroups(pDnode);
#endif
mnodeDropMnodeLocal(pDnode->dnodeId);
balanceAsyncNotify();
bnNotify();
mnodeUpdateDnodeEps();
mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
@ -187,7 +188,8 @@ int32_t mnodeInitDnodes() {
.fpRestored = mnodeDnodeActionRestored
};
tsDnodeSdb = sdbOpenTable(&desc);
tsDnodeRid = sdbOpenTable(&desc);
tsDnodeSdb = sdbGetTableByRid(tsDnodeRid);
if (tsDnodeSdb == NULL) {
mError("failed to init dnodes data");
return -1;
@ -206,13 +208,14 @@ int32_t mnodeInitDnodes() {
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mnodeRetrieveDnodes);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DNODE, mnodeCancelGetNextDnode);
mDebug("table:dnodes table is created");
return 0;
}
void mnodeCleanupDnodes() {
sdbCloseTable(tsDnodeSdb);
sdbCloseTable(tsDnodeRid);
pthread_mutex_destroy(&tsDnodeEpsMutex);
free(tsDnodeEps);
tsDnodeEps = NULL;
@ -223,6 +226,10 @@ void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) {
return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode);
}
void mnodeCancelGetNextDnode(void *pIter) {
sdbFreeIter(tsDnodeSdb, pIter);
}
int32_t mnodeGetDnodesNum() {
return sdbGetNumOfRows(tsDnodeSdb);
}
@ -241,8 +248,6 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() {
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
if (cpuCores < 2) cpuCores = 2;
return cpuCores;
}
@ -259,8 +264,6 @@ int32_t mnodeGetOnlineDnodesNum() {
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
return onlineDnodes;
}
@ -276,13 +279,12 @@ void *mnodeGetDnodeByEp(char *ep) {
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (strcmp(ep, pDnode->dnodeEp) == 0) {
sdbFreeIter(pIter);
mnodeCancelGetNextDnode(pIter);
return pDnode;
}
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
return NULL;
}
@ -345,7 +347,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION;
}
int32_t code = balanceAlterDnode(pDnode, vnodeId, dnodeId);
int32_t code = bnAlterDnode(pDnode, vnodeId, dnodeId);
mnodeDecDnodeRef(pDnode);
return code;
} else {
@ -464,7 +466,10 @@ static void mnodeUpdateDnodeEps() {
while (1) {
pIter = mnodeGetNextDnode(pIter, &pDnode);
if (pDnode == NULL) break;
if (dnodesNum >= totalDnodes) break;
if (dnodesNum >= totalDnodes) {
mnodeCancelGetNextDnode(pIter);
break;
}
SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum];
dnodesNum++;
@ -474,7 +479,6 @@ static void mnodeUpdateDnodeEps() {
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
pthread_mutex_unlock(&tsDnodeEpsMutex);
}
@ -587,8 +591,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
mInfo("dnode:%d, from offline to online", pDnode->dnodeId);
pDnode->status = TAOS_DN_STATUS_READY;
pDnode->offlineReason = TAOS_DN_OFF_ONLINE;
balanceSyncNotify();
balanceAsyncNotify();
bnCheckModules();
bnNotify();
}
if (openVnodes != pDnode->openVnodes) {
@ -704,7 +708,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
#ifndef _SYNC
int32_t code = mnodeDropDnode(pDnode, pMsg);
#else
int32_t code = balanceDropDnode(pDnode);
int32_t code = bnDropDnode(pDnode);
#endif
mnodeDecDnodeRef(pDnode);
return code;
@ -1100,7 +1104,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
pDnode = mnodeGetDnodeByEp(pShow->payload);
} else {
void *pIter = mnodeGetNextDnode(NULL, (SDnodeObj **)&pDnode);
sdbFreeIter(pIter);
mnodeCancelGetNextDnode(pIter);
}
if (pDnode != NULL) {
@ -1148,7 +1152,6 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
} else {
numOfRows = 0;
}
@ -1179,12 +1182,12 @@ static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) {
#ifndef _SYNC
int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
void balanceCleanUp() {}
void balanceAsyncNotify() {}
void balanceSyncNotify() {}
void balanceReset() {}
int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
int32_t bnInit() { return TSDB_CODE_SUCCESS; }
void bnCleanUp() {}
void bnNotify() {}
void bnCheckModules() {}
void bnReset() {}
int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
char* syncRole[] = {
"offline",
@ -1194,7 +1197,7 @@ char* syncRole[] = {
"master"
};
int32_t balanceAllocVnodes(SVgObj *pVgroup) {
int32_t bnAllocVnodes(SVgObj *pVgroup) {
void * pIter = NULL;
SDnodeObj *pDnode = NULL;
SDnodeObj *pSelDnode = NULL;
@ -1217,8 +1220,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
mnodeDecDnodeRef(pDnode);
}
sdbFreeIter(pIter);
if (pSelDnode == NULL) {
mError("failed to alloc vnode to vgroup");
return TSDB_CODE_MND_NO_ENOUGH_DNODES;

5
src/mnode/src/mnodeMain.c
View File

@ -17,7 +17,7 @@
#include "os.h"
#include "taosdef.h"
#include "tsched.h"
#include "tbalance.h"
#include "tbn.h"
#include "tgrant.h"
#include "ttimer.h"
#include "tglobal.h"
@ -47,6 +47,7 @@ void *tsMnodeTmr = NULL;
static bool tsMgmtIsRunning = false;
static const SMnodeComponent tsMnodeComponents[] = {
{"sdbref", sdbInitRef, sdbCleanUpRef},
{"profile", mnodeInitProfile, mnodeCleanupProfile},
{"cluster", mnodeInitCluster, mnodeCleanupCluster},
{"accts", mnodeInitAccts, mnodeCleanupAccts},
@ -57,7 +58,7 @@ static const SMnodeComponent tsMnodeComponents[] = {
{"tables", mnodeInitTables, mnodeCleanupTables},
{"mnodes", mnodeInitMnodes, mnodeCleanupMnodes},
{"sdb", sdbInit, sdbCleanUp},
{"balance", balanceInit, balanceCleanUp},
{"balance", bnInit, bnCleanUp},
{"grant", grantInit, grantCleanUp},
{"show", mnodeInitShow, mnodeCleanUpShow}
};

35
src/mnode/src/mnodeMnode.c
View File

@ -19,7 +19,7 @@
#include "tglobal.h"
#include "trpc.h"
#include "tsync.h"
#include "tbalance.h"
#include "tbn.h"
#include "tutil.h"
#include "tsocket.h"
#include "tdataformat.h"
@ -34,6 +34,7 @@
#include "mnodeUser.h"
#include "mnodeVgroup.h"
int64_t tsMnodeRid = -1;
static void * tsMnodeSdb = NULL;
static int32_t tsMnodeUpdateSize = 0;
static SRpcEpSet tsMnodeEpSetForShell;
@ -123,7 +124,7 @@ static int32_t mnodeMnodeActionRestored() {
pMnode->role = TAOS_SYNC_ROLE_MASTER;
mnodeDecMnodeRef(pMnode);
}
sdbFreeIter(pIter);
mnodeCancelGetNextMnode(pIter);
}
mnodeUpdateMnodeEpSet();
@ -153,7 +154,8 @@ int32_t mnodeInitMnodes() {
.fpRestored = mnodeMnodeActionRestored
};
tsMnodeSdb = sdbOpenTable(&desc);
tsMnodeRid = sdbOpenTable(&desc);
tsMnodeSdb = sdbGetTableByRid(tsMnodeRid);
if (tsMnodeSdb == NULL) {
mError("failed to init mnodes data");
return -1;
@ -161,13 +163,14 @@ int32_t mnodeInitMnodes() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_MNODE, mnodeCancelGetNextMnode);
mDebug("table:mnodes table is created");
return TSDB_CODE_SUCCESS;
}
void mnodeCleanupMnodes() {
sdbCloseTable(tsMnodeSdb);
sdbCloseTable(tsMnodeRid);
tsMnodeSdb = NULL;
mnodeMnodeDestroyLock();
}
@ -192,6 +195,10 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) {
return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode);
}
void mnodeCancelGetNextMnode(void *pIter) {
sdbFreeIter(tsMnodeSdb, pIter);
}
void mnodeUpdateMnodeEpSet() {
mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum());
@ -239,8 +246,6 @@ void mnodeUpdateMnodeEpSet() {
tsMnodeEpSetForShell.numOfEps = index;
tsMnodeEpSetForPeer.numOfEps = index;
sdbFreeIter(pIter);
mnodeMnodeUnLock();
}
@ -248,12 +253,30 @@ void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) {
mnodeMnodeRdLock();
*epSet = tsMnodeEpSetForPeer;
mnodeMnodeUnLock();
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
epSet->inUse = (i + 1) % epSet->numOfEps;
mTrace("mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
mTrace("mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}
}
void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) {
mnodeMnodeRdLock();
*epSet = tsMnodeEpSetForShell;
mnodeMnodeUnLock();
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
epSet->inUse = (i + 1) % epSet->numOfEps;
mTrace("mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
mTrace("mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}
}
char* mnodeGetMnodeMasterEp() {

14
src/mnode/src/mnodePeer.c
View File

@ -20,7 +20,7 @@
#include "tsystem.h"
#include "tutil.h"
#include "tgrant.h"
#include "tbalance.h"
#include "tbn.h"
#include "tglobal.h"
#include "mnode.h"
#include "dnode.h"
@ -58,16 +58,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcEpSet);
mDebug("msg:%p, ahandle:%p type:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg,
pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
epSet->inUse = (i + 1) % epSet->numOfEps;
mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}
mDebug("msg:%p, ahandle:%p type:%s in mpeer queue is redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
return TSDB_CODE_RPC_REDIRECT;
}

25
src/mnode/src/mnodeProfile.c
View File

@ -34,7 +34,6 @@
#define QUERY_ID_SIZE 20
#define QUERY_STREAM_SAVE_SIZE 20
extern void *tsMnodeTmr;
static SCacheObj *tsMnodeConnCache = NULL;
static int32_t tsConnIndex = 0;
@ -42,6 +41,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mnodeCancelGetNextConn(void *pIter);
static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static void mnodeFreeConn(void *data);
@ -52,10 +52,13 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg);
int32_t mnodeInitProfile() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_QUERIES, mnodeGetQueryMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_QUERIES, mnodeRetrieveQueries);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_QUERIES, mnodeCancelGetNextConn);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONNS, mnodeGetConnsMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONNS, mnodeRetrieveConns);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CONNS, mnodeCancelGetNextConn);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMS, mnodeCancelGetNextConn);
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_QUERY, mnodeProcessKillQueryMsg);
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
@ -137,21 +140,15 @@ static void mnodeFreeConn(void *data) {
mDebug("connId:%d, is destroyed", pConn->connId);
}
static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
static void *mnodeGetNextConn(void *pIter, SConnObj **pConn) {
*pConn = NULL;
if (pIter == NULL) {
pIter = taosHashCreateIter(tsMnodeConnCache->pHashTable);
}
pIter = taosHashIterate(tsMnodeConnCache->pHashTable, pIter);
if (pIter == NULL) return NULL;
if (!taosHashIterNext(pIter)) {
taosHashDestroyIter(pIter);
return NULL;
}
SCacheDataNode **pNode = taosHashIterGet(pIter);
SCacheDataNode **pNode = pIter;
if (pNode == NULL || *pNode == NULL) {
taosHashDestroyIter(pIter);
taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter);
return NULL;
}
@ -159,6 +156,10 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
return pIter;
}
static void mnodeCancelGetNextConn(void *pIter) {
taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter);
}
static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) return 0;

17
src/mnode/src/mnodeRead.c
View File

@ -17,7 +17,7 @@
#include "os.h"
#include "taosdef.h"
#include "tsched.h"
#include "tbalance.h"
#include "tbn.h"
#include "tgrant.h"
#include "ttimer.h"
#include "tglobal.h"
@ -51,21 +51,12 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
mnodeGetMnodeEpSetForShell(epSet);
mDebug("msg:%p, app:%p type:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
epSet->inUse = (i + 1) % epSet->numOfEps;
mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse);
} else {
mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i]));
}
}
rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcEpSet);
mDebug("msg:%p, app:%p type:%s in mread queue is redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
return TSDB_CODE_RPC_REDIRECT;
}

99
src/mnode/src/mnodeSdb.c
View File

@ -18,7 +18,8 @@
#include "taoserror.h"
#include "hash.h"
#include "tutil.h"
#include "tbalance.h"
#include "tref.h"
#include "tbn.h"
#include "tqueue.h"
#include "twal.h"
#include "tsync.h"
@ -98,6 +99,7 @@ typedef struct {
SSdbWorker *worker;
} SSdbWorkerPool;
int32_t tsSdbRid;
extern void * tsMnodeTmr;
static void * tsSdbTmr;
static SSdbMgmt tsSdbMgmt = {0};
@ -118,6 +120,7 @@ static void sdbFreeQueue();
static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow);
static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow);
static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow);
static void sdbCloseTableObj(void *handle);
int32_t sdbGetId(void *pTable) {
return ((SSdbTable *)pTable)->autoIndex;
@ -241,13 +244,23 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]);
if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) {
balanceReset();
bnReset();
}
tsSdbMgmt.role = role;
sdbUpdateMnodeRoles();
}
static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; }
static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {}
static int32_t sdbGetSyncVersion(int32_t vgId, uint64_t *fver, uint64_t *vver) {
*fver = 0;
*vver = 0;
return 0;
}
// failed to forward, need revert insert
static void sdbHandleFailedConfirm(SSdbRow *pRow) {
SWalHead *pHead = pRow->pHead;
@ -325,7 +338,6 @@ void sdbUpdateSync(void *pMnodes) {
mnodeDecDnodeRef(pDnode);
mnodeDecMnodeRef(pMnode);
}
sdbFreeIter(pIter);
syncCfg.replica = index;
mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
} else {
@ -370,11 +382,14 @@ void sdbUpdateSync(void *pMnodes) {
syncInfo.version = sdbGetVersion();
syncInfo.syncCfg = syncCfg;
sprintf(syncInfo.path, "%s", tsMnodeDir);
syncInfo.getWalInfo = sdbGetWalInfo;
syncInfo.getFileInfo = sdbGetFileInfo;
syncInfo.getWalInfo = sdbGetWalInfo;
syncInfo.writeToCache = sdbWriteFwdToQueue;
syncInfo.confirmForward = sdbConfirmForward;
syncInfo.notifyRole = sdbNotifyRole;
syncInfo.notifyFileSynced = sdbNotifyFileSynced;
syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl;
syncInfo.getVersion = sdbGetSyncVersion;
tsSdbMgmt.cfg = syncCfg;
if (tsSdbMgmt.sync) {
@ -386,6 +401,17 @@ void sdbUpdateSync(void *pMnodes) {
sdbUpdateMnodeRoles();
}
int32_t sdbInitRef() {
tsSdbRid = taosOpenRef(10, sdbCloseTableObj);
if (tsSdbRid <= 0) {
sdbError("failed to init sdb ref");
return -1;
}
return 0;
}
void sdbCleanUpRef() { taosCloseRef(tsSdbRid); }
int32_t sdbInit() {
pthread_mutex_init(&tsSdbMgmt.mutex, NULL);
@ -424,7 +450,7 @@ void sdbCleanUp() {
walClose(tsSdbMgmt.wal);
tsSdbMgmt.wal = NULL;
}
pthread_mutex_destroy(&tsSdbMgmt.mutex);
}
@ -507,7 +533,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) {
atomic_add_fetch_32(&pTable->autoIndex, 1);
}
sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name,
sdbTrace("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name,
sdbGetRowStr(pTable, pRow->pObj), pRow->rowSize, pTable->numOfRows, pRow->pMsg);
int32_t code = (*pTable->fpInsert)(pRow);
@ -543,7 +569,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
atomic_sub_fetch_32(&pTable->numOfRows, 1);
sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
sdbTrace("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg);
sdbDecRef(pTable, pRow->pObj);
@ -552,7 +578,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
}
static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) {
sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
sdbTrace("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg);
(*pTable->fpUpdate)(pRow);
@ -650,7 +676,7 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *
return syncCode;
}
sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->name,
sdbTrace("vgId:1, sdb:%s, record from %s is disposed, action:%s key:%s hver:%" PRIu64, pTable->name, qtypeStr[qtype],
actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version);
// even it is WAL/FWD, it shall be called to update version in sync
@ -775,24 +801,17 @@ int32_t sdbUpdateRow(SSdbRow *pRow) {
}
}
void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) {
void *sdbFetchRow(void *tparam, void *pIter, void **ppRow) {
SSdbTable *pTable = tparam;
*ppRow = NULL;
if (pTable == NULL) return NULL;
SHashMutableIterator *pIter = pNode;
if (pIter == NULL) {
pIter = taosHashCreateIter(pTable->iHandle);
}
pIter = taosHashIterate(pTable->iHandle, pIter);
if (pIter == NULL) return NULL;
if (!taosHashIterNext(pIter)) {
taosHashDestroyIter(pIter);
return NULL;
}
void **ppMetaRow = taosHashIterGet(pIter);
void **ppMetaRow = pIter;
if (ppMetaRow == NULL) {
taosHashDestroyIter(pIter);
taosHashCancelIterate(pTable->iHandle, pIter);
return NULL;
}
@ -802,16 +821,17 @@ void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) {
return pIter;
}
void sdbFreeIter(void *pIter) {
if (pIter != NULL) {
taosHashDestroyIter(pIter);
}
void sdbFreeIter(void *tparam, void *pIter) {
SSdbTable *pTable = tparam;
if (pTable == NULL || pIter == NULL) return;
taosHashCancelIterate(pTable->iHandle, pIter);
}
void *sdbOpenTable(SSdbTableDesc *pDesc) {
int64_t sdbOpenTable(SSdbTableDesc *pDesc) {
SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable));
if (pTable == NULL) return NULL;
if (pTable == NULL) return -1;
pthread_mutex_init(&pTable->mutex, NULL);
tstrncpy(pTable->name, pDesc->name, SDB_TABLE_LEN);
@ -836,19 +856,31 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
tsSdbMgmt.numOfTables++;
tsSdbMgmt.tableList[pTable->id] = pTable;
return pTable;
return taosAddRef(tsSdbRid, pTable);
}
void sdbCloseTable(void *handle) {
void sdbCloseTable(int64_t rid) {
taosRemoveRef(tsSdbRid, rid);
}
void *sdbGetTableByRid(int64_t rid) {
void *handle = taosAcquireRef(tsSdbRid, rid);
taosReleaseRef(tsSdbRid, rid);
return handle;
}
static void sdbCloseTableObj(void *handle) {
SSdbTable *pTable = (SSdbTable *)handle;
if (pTable == NULL) return;
tsSdbMgmt.numOfTables--;
tsSdbMgmt.tableList[pTable->id] = NULL;
SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle);
while (taosHashIterNext(pIter)) {
void **ppRow = taosHashIterGet(pIter);
void *pIter = taosHashIterate(pTable->iHandle, NULL);
while (pIter) {
void **ppRow = pIter;
pIter = taosHashIterate(pTable->iHandle, pIter);
if (ppRow == NULL) continue;
SSdbRow row = {
@ -859,8 +891,9 @@ void sdbCloseTable(void *handle) {
(*pTable->fpDestroy)(&row);
}
taosHashDestroyIter(pIter);
taosHashCancelIterate(pTable->iHandle, pIter);
taosHashCleanup(pTable->iHandle);
pTable->iHandle = NULL;
pthread_mutex_destroy(&pTable->mutex);
sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->name, tsSdbMgmt.numOfTables);

10
src/mnode/src/mnodeShow.c
View File

@ -52,11 +52,11 @@ static bool mnodeCheckShowFinished(SShowObj *pShow);
static void *mnodePutShowObj(SShowObj *pShow);
static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove);
extern void *tsMnodeTmr;
static void *tsMnodeShowCache = NULL;
static int32_t tsShowObjIndex = 0;
static SShowMetaFp tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX] = {0};
static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
static SShowFreeIterFp tsMnodeShowFreeIterFp[TSDB_MGMT_TABLE_MAX] = {0};
int32_t mnodeInitShow() {
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mnodeProcessShowMsg);
@ -85,6 +85,10 @@ void mnodeAddShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) {
tsMnodeShowRetrieveFp[msgType] = fp;
}
void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp) {
tsMnodeShowFreeIterFp[msgType] = fp;
}
static char *mnodeGetShowType(int32_t showType) {
switch (showType) {
case TSDB_MGMT_TABLE_ACCT: return "show accounts";
@ -412,7 +416,9 @@ static void* mnodePutShowObj(SShowObj *pShow) {
static void mnodeFreeShowObj(void *data) {
SShowObj *pShow = *(SShowObj **)data;
sdbFreeIter(pShow->pIter);
if (tsMnodeShowFreeIterFp[pShow->type] != NULL && pShow->pIter != NULL) {
(*tsMnodeShowFreeIterFp[pShow->type])(pShow->pIter);
}
mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
tfree(pShow);

63
src/mnode/src/mnodeTable.c
View File

@ -49,7 +49,9 @@
#define CREATE_CTABLE_RETRY_TIMES 10
#define CREATE_CTABLE_RETRY_SEC 14
int64_t tsCTableRid = -1;
static void * tsChildTableSdb;
int64_t tsSTableRid = -1;
static void * tsSuperTableSdb;
static int32_t tsChildTableUpdateSize;
static int32_t tsSuperTableUpdateSize;
@ -342,8 +344,7 @@ static int32_t mnodeChildTableActionRestored() {
mnodeDecTableRef(pTable);
}
sdbFreeIter(pIter);
mnodeCancelGetNextChildTable(pIter);
return 0;
}
@ -351,7 +352,7 @@ static int32_t mnodeInitChildTables() {
SCTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
SSdbTableDesc tableDesc = {
SSdbTableDesc desc = {
.id = SDB_TABLE_CTABLE,
.name = "ctables",
.hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
@ -367,7 +368,8 @@ static int32_t mnodeInitChildTables() {
.fpRestored = mnodeChildTableActionRestored
};
tsChildTableSdb = sdbOpenTable(&tableDesc);
tsCTableRid = sdbOpenTable(&desc);
tsChildTableSdb = sdbGetTableByRid(tsCTableRid);
if (tsChildTableSdb == NULL) {
mError("failed to init child table data");
return -1;
@ -378,7 +380,7 @@ static int32_t mnodeInitChildTables() {
}
static void mnodeCleanupChildTables() {
sdbCloseTable(tsChildTableSdb);
sdbCloseTable(tsCTableRid);
tsChildTableSdb = NULL;
}
@ -544,7 +546,7 @@ static int32_t mnodeInitSuperTables() {
SSTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
SSdbTableDesc tableDesc = {
SSdbTableDesc desc = {
.id = SDB_TABLE_STABLE,
.name = "stables",
.hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
@ -560,7 +562,8 @@ static int32_t mnodeInitSuperTables() {
.fpRestored = mnodeSuperTableActionRestored
};
tsSuperTableSdb = sdbOpenTable(&tableDesc);
tsSTableRid = sdbOpenTable(&desc);
tsSuperTableSdb = sdbGetTableByRid(tsSTableRid);
if (tsSuperTableSdb == NULL) {
mError("failed to init stables data");
return -1;
@ -571,7 +574,7 @@ static int32_t mnodeInitSuperTables() {
}
static void mnodeCleanupSuperTables() {
sdbCloseTable(tsSuperTableSdb);
sdbCloseTable(tsSTableRid);
tsSuperTableSdb = NULL;
}
@ -602,10 +605,13 @@ int32_t mnodeInitTables() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mnodeGetShowTableMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_TABLE, mnodeCancelGetNextChildTable);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_METRIC, mnodeCancelGetNextSuperTable);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeCancelGetNextChildTable);
return TSDB_CODE_SUCCESS;
}
@ -626,14 +632,12 @@ static void *mnodeGetSuperTableByUid(uint64_t uid) {
pIter = mnodeGetNextSuperTable(pIter, &pStable);
if (pStable == NULL) break;
if (pStable->uid == uid) {
sdbFreeIter(pIter);
mnodeCancelGetNextSuperTable(pIter);
return pStable;
}
mnodeDecTableRef(pStable);
}
sdbFreeIter(pIter);
return NULL;
}
@ -655,10 +659,18 @@ void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) {
return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
}
void mnodeCancelGetNextChildTable(void *pIter) {
sdbFreeIter(tsChildTableSdb, pIter);
}
void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) {
return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
}
void mnodeCancelGetNextSuperTable(void *pIter) {
sdbFreeIter(tsSuperTableSdb, pIter);
}
void mnodeIncTableRef(void *p1) {
STableObj *pTable = (STableObj *)p1;
if (pTable->type == TSDB_SUPER_TABLE) {
@ -914,10 +926,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
while (taosHashIterNext(pIter)) {
int32_t *pVgId = taosHashIterGet(pIter);
int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL);
while (pVgId) {
SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
pVgId = taosHashIterate(pStable->vgHash, pVgId);
if (pVgroup == NULL) break;
SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg));
@ -933,7 +945,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
dnodeSendMsgToDnode(&epSet, &rpcMsg);
mnodeDecVgroupRef(pVgroup);
}
taosHashDestroyIter(pIter);
taosHashCancelIterate(pStable->vgHash, pVgId);
mnodeDropAllChildTablesInStable(pStable);
}
@ -1430,8 +1443,6 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) {
mnodeDecTableRef(pTable);
}
sdbFreeIter(pIter);
mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
@ -1523,11 +1534,11 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
} else {
SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg;
SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
int32_t vgSize = 0;
while (taosHashIterNext(pIter)) {
int32_t *pVgId = taosHashIterGet(pIter);
SVgObj * pVgroup = mnodeGetVgroup(*pVgId);
int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL);
int32_t vgSize = 0;
while (pVgId) {
SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
pVgId = taosHashIterate(pTable->vgHash, pVgId);
if (pVgroup == NULL) continue;
pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
@ -1547,7 +1558,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
mnodeDecVgroupRef(pVgroup);
}
taosHashDestroyIter(pIter);
taosHashCancelIterate(pTable->vgHash, pVgId);
mnodeDecTableRef(pTable);
pVgroupMsg->numOfVgroups = htonl(vgSize);
@ -2230,8 +2241,6 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
mnodeDecTableRef(pTable);
}
sdbFreeIter(pIter);
mInfo("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId);
}
@ -2263,8 +2272,6 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) {
mnodeDecTableRef(pTable);
}
sdbFreeIter(pIter);
mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
}
@ -2293,8 +2300,6 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) {
mnodeDecTableRef(pTable);
}
sdbFreeIter(pIter);
mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
}

15
src/mnode/src/mnodeUser.c
View File

@ -33,6 +33,7 @@
#include "mnodeWrite.h"
#include "mnodePeer.h"
int64_t tsUserRid = -1;
static void * tsUserSdb = NULL;
static int32_t tsUserUpdateSize = 0;
static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@ -123,7 +124,6 @@ static void mnodePrintUserAuth() {
}
fflush(fp);
sdbFreeIter(pIter);
fclose(fp);
}
@ -166,7 +166,8 @@ int32_t mnodeInitUsers() {
.fpRestored = mnodeUserActionRestored
};
tsUserSdb = sdbOpenTable(&desc);
tsUserRid = sdbOpenTable(&desc);
tsUserSdb = sdbGetTableByRid(tsUserRid);
if (tsUserSdb == NULL) {
mError("table:%s, failed to create hash", desc.name);
return -1;
@ -177,6 +178,8 @@ int32_t mnodeInitUsers() {
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mnodeProcessDropUserMsg);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_USER, mnodeGetUserMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_USER, mnodeCancelGetNextUser);
mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg);
mDebug("table:%s, hash is created", desc.name);
@ -184,7 +187,7 @@ int32_t mnodeInitUsers() {
}
void mnodeCleanupUsers() {
sdbCloseTable(tsUserSdb);
sdbCloseTable(tsUserRid);
tsUserSdb = NULL;
}
@ -196,6 +199,10 @@ void *mnodeGetNextUser(void *pIter, SUserObj **pUser) {
return sdbFetchRow(tsUserSdb, pIter, (void **)pUser);
}
void mnodeCancelGetNextUser(void *pIter) {
sdbFreeIter(tsUserSdb, pIter);
}
void mnodeIncUserRef(SUserObj *pUser) {
return sdbIncRef(tsUserSdb, pUser);
}
@ -574,8 +581,6 @@ void mnodeDropAllUsers(SAcctObj *pAcct) {
mnodeDecUserRef(pUser);
}
sdbFreeIter(pIter);
mDebug("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
}

25
src/mnode/src/mnodeVgroup.c
View File

@ -20,7 +20,7 @@
#include "tsocket.h"
#include "tidpool.h"
#include "tsync.h"
#include "tbalance.h"
#include "tbn.h"
#include "tglobal.h"
#include "tdataformat.h"
#include "dnode.h"
@ -51,6 +51,7 @@ char* vgroupStatus[] = {
"updating"
};
int64_t tsVgroupRid = -1;
static void *tsVgroupSdb = NULL;
static int32_t tsVgUpdateSize = 0;
@ -222,7 +223,8 @@ int32_t mnodeInitVgroups() {
.fpRestored = mnodeVgroupActionRestored,
};
tsVgroupSdb = sdbOpenTable(&desc);
tsVgroupRid = sdbOpenTable(&desc);
tsVgroupSdb = sdbGetTableByRid(tsVgroupRid);
if (tsVgroupSdb == NULL) {
mError("failed to init vgroups data");
return -1;
@ -230,6 +232,7 @@ int32_t mnodeInitVgroups() {
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup);
mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp);
mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp);
mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp);
@ -304,7 +307,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mnodeCancelGetNextVgroup(pIter);
}
void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
@ -491,6 +494,10 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup);
}
void mnodeCancelGetNextVgroup(void *pIter) {
sdbFreeIter(tsVgroupSdb, pIter);
}
static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) {
SVgObj *pVgroup = pMsg->pVgroup;
SDbObj *pDb = pMsg->pDb;
@ -556,7 +563,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
pVgroup->numOfVnodes = pDb->cfg.replications;
pVgroup->createdTime = taosGetTimestampMs();
pVgroup->accessState = TSDB_VN_ALL_ACCCESS;
int32_t code = balanceAllocVnodes(pVgroup);
int32_t code = bnAllocVnodes(pVgroup);
if (code != TSDB_CODE_SUCCESS) {
mError("db:%s, no enough dnode to alloc %d vnodes to vgroup, reason:%s", pDb->name, pVgroup->numOfVnodes,
tstrerror(code));
@ -605,7 +612,7 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) {
}
void mnodeCleanupVgroups() {
sdbCloseTable(tsVgroupSdb);
sdbCloseTable(tsVgroupRid);
tsVgroupSdb = NULL;
}
@ -1095,8 +1102,6 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups);
}
@ -1118,8 +1123,6 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) {
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name);
}
#endif
@ -1147,8 +1150,6 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) {
mnodeDecVgroupRef(pVgroup);
}
sdbFreeIter(pIter);
mInfo("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
}
@ -1170,7 +1171,5 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
numOfVgroups++;
}
sdbFreeIter(pIter);
mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
}

16
src/mnode/src/mnodeWrite.c
View File

@ -17,7 +17,7 @@
#include "os.h"
#include "taosdef.h"
#include "tsched.h"
#include "tbalance.h"
#include "tbn.h"
#include "tgrant.h"
#include "tglobal.h"
#include "trpc.h"
@ -54,18 +54,8 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
rpcRsp->rsp = epSet;
rpcRsp->len = sizeof(SRpcEpSet);
mDebug("msg:%p, app:%p type:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
epSet->inUse = (i + 1) % epSet->numOfEps;
mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d, set inUse to %d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i],
htons(epSet->port[i]), epSet->inUse);
} else {
mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i],
htons(epSet->port[i]));
}
}
mDebug("msg:%p, app:%p type:%s in write queue, is redirected, numOfEps:%d inUse:%d", pMsg,
pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
return TSDB_CODE_RPC_REDIRECT;
}

19
src/os/inc/osFile.h
View File

@ -20,17 +20,6 @@
extern "C" {
#endif
#define tread(fd, buf, count) read(fd, buf, count)
#define twrite(fd, buf, count) write(fd, buf, count)
#define tlseek(fd, offset, whence) lseek(fd, offset, whence)
#define tclose(fd) \
{ \
if (FD_VALID(fd)) { \
close(fd); \
fd = FD_INITIALIZER; \
} \
}
int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
@ -39,7 +28,13 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP
#define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
#define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
#define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
#define taosClose(x) tclose(x)
#define taosClose(fd) \
{ \
if (FD_VALID(fd)) { \
close(fd); \
fd = FD_INITIALIZER; \
} \
}
// TAOS_OS_FUNC_FILE_SENDIFLE
int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);

2
src/os/src/detail/osFile.c
View File

@ -116,7 +116,7 @@ int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
}
int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
return (int64_t)tlseek(fd, (long)offset, whence);
return (int64_t)lseek(fd, (long)offset, whence);
}
#ifndef TAOS_OS_FUNC_FILE_SENDIFLE

150
src/plugins/monitor/src/monitorMain.c → src/plugins/monitor/src/monMain.c
View File

@ -27,12 +27,12 @@
#include "monitor.h"
#include "taoserror.h"
#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
#define monWarn(...) { if (monDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
#define monInfo(...) { if (monDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }}
#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
#define SQL_LENGTH 1030
#define LOG_LEN_STR 100
@ -48,12 +48,12 @@ typedef enum {
MON_CMD_CREATE_TB_ACCT_ROOT,
MON_CMD_CREATE_TB_SLOWQUERY,
MON_CMD_MAX
} EMonitorCommand;
} EMonCmd;
typedef enum {
MON_STATE_NOT_INIT,
MON_STATE_INITED
} EMonitorState;
} EMonState;
typedef struct {
pthread_t thread;
@ -64,17 +64,17 @@ typedef struct {
int8_t start; // enable/disable by mnode
int8_t quiting; // taosd is quiting
char sql[SQL_LENGTH + 1];
} SMonitorConn;
} SMonConn;
static SMonitorConn tsMonitor = {0};
static void monitorSaveSystemInfo();
static void *monitorThreadFunc(void *param);
static void monitorBuildMonitorSql(char *sql, int32_t cmd);
extern int32_t (*monitorStartSystemFp)();
extern void (*monitorStopSystemFp)();
extern void (*monitorExecuteSQLFp)(char *sql);
static SMonConn tsMonitor = {0};
static void monSaveSystemInfo();
static void *monThreadFunc(void *param);
static void monBuildMonitorSql(char *sql, int32_t cmd);
extern int32_t (*monStartSystemFp)();
extern void (*monStopSystemFp)();
extern void (*monExecuteSQLFp)(char *sql);
int32_t monitorInitSystem() {
int32_t monInitSystem() {
if (tsMonitor.ep[0] == 0) {
strcpy(tsMonitor.ep, tsLocalEp);
}
@ -90,29 +90,29 @@ int32_t monitorInitSystem() {
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) {
monError("failed to create thread to for monitor module, reason:%s", strerror(errno));
return -1;
}
pthread_attr_destroy(&thAttr);
mnDebug("monitor thread is launched");
monDebug("monitor thread is launched");
monitorStartSystemFp = monitorStartSystem;
monitorStopSystemFp = monitorStopSystem;
monStartSystemFp = monStartSystem;
monStopSystemFp = monStopSystem;
return 0;
}
int32_t monitorStartSystem() {
int32_t monStartSystem() {
taos_init();
tsMonitor.start = 1;
monitorExecuteSQLFp = monitorExecuteSQL;
mnInfo("monitor module start");
monExecuteSQLFp = monExecuteSQL;
monInfo("monitor module start");
return 0;
}
static void *monitorThreadFunc(void *param) {
mnDebug("starting to initialize monitor module ...");
static void *monThreadFunc(void *param) {
monDebug("starting to initialize monitor module ...");
while (1) {
static int32_t accessTimes = 0;
@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
if (tsMonitor.quiting) {
tsMonitor.state = MON_STATE_NOT_INIT;
mnInfo("monitor thread will quit, for taosd is quiting");
monInfo("monitor thread will quit, for taosd is quiting");
break;
} else {
taosGetDisk();
@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
}
if (dnodeGetDnodeId() <= 0) {
mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
monDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
continue;
}
@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
tsMonitor.state = MON_STATE_NOT_INIT;
tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
if (tsMonitor.conn == NULL) {
mnError("failed to connect to database, reason:%s", tstrerror(terrno));
monError("failed to connect to database, reason:%s", tstrerror(terrno));
continue;
} else {
mnDebug("connect to database success");
monDebug("connect to database success");
}
}
@ -151,16 +151,16 @@ static void *monitorThreadFunc(void *param) {
int code = 0;
for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) {
monitorBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
code = taos_errno(res);
taos_free_result(res);
if (code != 0) {
mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
break;
} else {
mnDebug("successfully to exec sql:%s", tsMonitor.sql);
monDebug("successfully to exec sql:%s", tsMonitor.sql);
}
}
@ -171,16 +171,16 @@ static void *monitorThreadFunc(void *param) {
if (tsMonitor.state == MON_STATE_INITED) {
if (accessTimes % tsMonitorInterval == 0) {
monitorSaveSystemInfo();
monSaveSystemInfo();
}
}
}
mnInfo("monitor thread is stopped");
monInfo("monitor thread is stopped");
return NULL;
}
static void monitorBuildMonitorSql(char *sql, int32_t cmd) {
static void monBuildMonitorSql(char *sql, int32_t cmd) {
memset(sql, 0, SQL_LENGTH);
if (cmd == MON_CMD_CREATE_DB) {
@ -236,47 +236,47 @@ static void monitorBuildMonitorSql(char *sql, int32_t cmd) {
sql[SQL_LENGTH] = 0;
}
void monitorStopSystem() {
void monStopSystem() {
tsMonitor.start = 0;
tsMonitor.state = MON_STATE_NOT_INIT;
monitorExecuteSQLFp = NULL;
mnInfo("monitor module stopped");
monExecuteSQLFp = NULL;
monInfo("monitor module stopped");
}
void monitorCleanUpSystem() {
void monCleanupSystem() {
tsMonitor.quiting = 1;
monitorStopSystem();
monStopSystem();
pthread_join(tsMonitor.thread, NULL);
if (tsMonitor.conn != NULL) {
taos_close(tsMonitor.conn);
tsMonitor.conn = NULL;
}
mnInfo("monitor module is cleaned up");
monInfo("monitor module is cleaned up");
}
// unit is MB
static int32_t monitorBuildMemorySql(char *sql) {
static int32_t monBuildMemorySql(char *sql) {
float sysMemoryUsedMB = 0;
bool suc = taosGetSysMemory(&sysMemoryUsedMB);
if (!suc) {
mnDebug("failed to get sys memory info");
monDebug("failed to get sys memory info");
}
float procMemoryUsedMB = 0;
suc = taosGetProcMemory(&procMemoryUsedMB);
if (!suc) {
mnDebug("failed to get proc memory info");
monDebug("failed to get proc memory info");
}
return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
}
// unit is %
static int32_t monitorBuildCpuSql(char *sql) {
static int32_t monBuildCpuSql(char *sql) {
float sysCpuUsage = 0, procCpuUsage = 0;
bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
if (!suc) {
mnDebug("failed to get cpu usage");
monDebug("failed to get cpu usage");
}
if (sysCpuUsage <= procCpuUsage) {
@ -287,72 +287,72 @@ static int32_t monitorBuildCpuSql(char *sql) {
}
// unit is GB
static int32_t monitorBuildDiskSql(char *sql) {
static int32_t monBuildDiskSql(char *sql) {
return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB);
}
// unit is Kb
static int32_t monitorBuildBandSql(char *sql) {
static int32_t monBuildBandSql(char *sql) {
float bandSpeedKb = 0;
bool suc = taosGetBandSpeed(&bandSpeedKb);
if (!suc) {
mnDebug("failed to get bandwidth speed");
monDebug("failed to get bandwidth speed");
}
return sprintf(sql, ", %f", bandSpeedKb);
}
static int32_t monitorBuildReqSql(char *sql) {
static int32_t monBuildReqSql(char *sql) {
SStatisInfo info = dnodeGetStatisInfo();
return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
}
static int32_t monitorBuildIoSql(char *sql) {
static int32_t monBuildIoSql(char *sql) {
float readKB = 0, writeKB = 0;
bool suc = taosGetProcIO(&readKB, &writeKB);
if (!suc) {
mnDebug("failed to get io info");
monDebug("failed to get io info");
}
return sprintf(sql, ", %f, %f", readKB, writeKB);
}
static void monitorSaveSystemInfo() {
static void monSaveSystemInfo() {
int64_t ts = taosGetTimestampUs();
char * sql = tsMonitor.sql;
int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts);
pos += monitorBuildCpuSql(sql + pos);
pos += monitorBuildMemorySql(sql + pos);
pos += monitorBuildDiskSql(sql + pos);
pos += monitorBuildBandSql(sql + pos);
pos += monitorBuildIoSql(sql + pos);
pos += monitorBuildReqSql(sql + pos);
pos += monBuildCpuSql(sql + pos);
pos += monBuildMemorySql(sql + pos);
pos += monBuildDiskSql(sql + pos);
pos += monBuildBandSql(sql + pos);
pos += monBuildIoSql(sql + pos);
pos += monBuildReqSql(sql + pos);
void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
int code = taos_errno(res);
taos_free_result(res);
if (code != 0) {
mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
} else {
mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
monDebug("successfully to save system info, sql:%s", tsMonitor.sql);
}
}
static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
int32_t c = taos_errno(result);
if (c != TSDB_CODE_SUCCESS) {
mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
monError("save %s failed, reason:%s", (char *)param, tstrerror(c));
} else {
int32_t rows = taos_affected_rows(result);
mnDebug("save %s succ, rows:%d", (char *)param, rows);
monDebug("save %s succ, rows:%d", (char *)param, rows);
}
taos_free_result(result);
}
void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
void monSaveAcctLog(SAcctMonitorObj *pMon) {
if (tsMonitor.state != MON_STATE_INITED) return;
char sql[1024] = {0};
@ -382,11 +382,11 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
pMon->totalConns, pMon->maxConns,
pMon->accessState);
mnDebug("save account info, sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
monDebug("save account info, sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info");
}
void monitorSaveLog(int32_t level, const char *const format, ...) {
void monSaveLog(int32_t level, const char *const format, ...) {
if (tsMonitor.state != MON_STATE_INITED) return;
va_list argpointer;
@ -403,13 +403,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
len += sprintf(sql + len, "', '%s')", tsLocalEp);
sql[len++] = 0;
mnDebug("save log, sql: %s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
monDebug("save log, sql: %s", sql);
taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log");
}
void monitorExecuteSQL(char *sql) {
void monExecuteSQL(char *sql) {
if (tsMonitor.state != MON_STATE_INITED) return;
mnDebug("execute sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
monDebug("execute sql:%s", sql);
taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql");
}

38
src/query/inc/qExecutor.h
View File

@ -63,9 +63,11 @@ typedef struct SSqlGroupbyExpr {
typedef struct SResultRow {
int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer
int32_t rowId:15;
bool closed:1; // this result status: closed or opened
uint16_t numOfRows; // number of rows of current time window
int32_t rowId:29; // row index in buffer page
bool startInterp; // the time window start timestamp has done the interpolation already.
bool endInterp; // the time window end timestamp has done the interpolation already.
bool closed; // this result status: closed or opened
uint32_t numOfRows; // number of rows of current time window
SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo
union {STimeWindow win; char* key;}; // start key of current time window
} SResultRow;
@ -81,16 +83,15 @@ typedef struct SResultRec {
int32_t threshold; // result size threshold in rows.
} SResultRec;
typedef struct SWindowResInfo {
SResultRow** pResult; // result list
int16_t type:8; // data type for hash key
int32_t size:24; // number of result set
int32_t threshold; // threshold to halt query and return the generated results.
int32_t capacity; // max capacity
int32_t curIndex; // current start active index
int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key
} SWindowResInfo;
typedef struct SResultRowInfo {
SResultRow** pResult; // result list
int16_t type:8; // data type for hash key
int32_t size:24; // number of result set
int32_t capacity; // max capacity
int32_t curIndex; // current start active index
int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key
} SResultRowInfo;
typedef struct SColumnFilterElem {
int16_t bytes; // column length
@ -113,7 +114,7 @@ typedef struct STableQueryInfo {
STimeWindow win;
STSCursor cur;
void* pTable; // for retrieve the page id list
SWindowResInfo windowResInfo;
SResultRowInfo windowResInfo;
} STableQueryInfo;
typedef struct SQueryCostInfo {
@ -177,7 +178,7 @@ typedef struct SQueryRuntimeEnv {
uint16_t* offset;
uint16_t scanFlag; // denotes reversed scan of data or not
SFillInfo* pFillInfo;
SWindowResInfo windowResInfo;
SResultRowInfo windowResInfo;
STSBuf* pTSBuf;
STSCursor cur;
SQueryCostInfo summary;
@ -187,6 +188,8 @@ typedef struct SQueryRuntimeEnv {
bool topBotQuery; // false
bool groupbyNormalCol; // denote if this is a groupby normal column query
bool hasTagResults; // if there are tag values in final result or not
bool timeWindowInterpo;// if the time window start/end required interpolation
bool queryWindowIdentical; // all query time windows are identical for all tables in one group
int32_t interBufSize; // intermediate buffer sizse
int32_t prevGroupId; // previous executed group id
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
@ -195,6 +198,8 @@ typedef struct SQueryRuntimeEnv {
SResultRowPool* pool; // window result object pool
int32_t* rowCellInfoOffset;// offset value for each row result cell info
char** prevRow;
char** nextRow;
} SQueryRuntimeEnv;
enum {
@ -212,7 +217,8 @@ typedef struct SQInfo {
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
SQueryRuntimeEnv runtimeEnv;
SArray* arrTableIdInfo;
// SArray* arrTableIdInfo;
SHashObj* arrTableIdInfo;
int32_t groupIndex;
/*

18
src/query/inc/qUtil.h
View File

@ -30,19 +30,19 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ
void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type);
SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);
int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type);
void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
void cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo);
void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo);
void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv);
int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo);
void closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SResultRowInfo* pWindowResInfo);
void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) {
assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
return pWindowResInfo->pResult[slot];
}
@ -50,7 +50,7 @@ static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)
#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1)
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot);
int32_t initResultRow(SResultRow *pResultRow);

29
src/query/inc/tsqlfunction.h
View File

@ -152,6 +152,11 @@ typedef struct SResultRowCellInfo {
uint32_t numOfRes; // num of output result in current buffer
} SResultRowCellInfo;
typedef struct SPoint1 {
int64_t key;
double val;
} SPoint1;
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
struct SQLFunctionCtx;
@ -194,6 +199,8 @@ typedef struct SQLFunctionCtx {
SResultRowCellInfo *resultInfo;
SExtTagsInfo tagInfo;
SPoint1 start;
SPoint1 end;
} SQLFunctionCtx;
typedef struct SQLAggFuncElem {
@ -243,21 +250,11 @@ enum {
};
typedef struct STwaInfo {
TSKEY lastKey;
int8_t hasResult; // flag to denote has value
int16_t type; // source data type
TSKEY SKey;
TSKEY EKey;
union {
double dOutput;
int64_t iOutput;
};
union {
double dLastValue;
int64_t iLastValue;
};
TSKEY lastKey;
int8_t hasResult; // flag to denote has value
double dOutput;
double lastValue;
STimeWindow win;
} STwaInfo;
/* global sql function array */
@ -276,8 +273,6 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha
(_r)->initialized = false; \
} while (0)
//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf);
static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag

107
src/query/src/qAst.c
View File

@ -370,6 +370,66 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
#endif
}
static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) {
switch(type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t* p = (int8_t*) dest;
int8_t* pSrc = (int8_t*) src;
for(int32_t i = 0; i < numOfRows; ++i) {
p[i] = pSrc[numOfRows - i - 1];
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t* p = (int16_t*) dest;
int16_t* pSrc = (int16_t*) src;
for(int32_t i = 0; i < numOfRows; ++i) {
p[i] = pSrc[numOfRows - i - 1];
}
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t* p = (int32_t*) dest;
int32_t* pSrc = (int32_t*) src;
for(int32_t i = 0; i < numOfRows; ++i) {
p[i] = pSrc[numOfRows - i - 1];
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t* p = (int64_t*) dest;
int64_t* pSrc = (int64_t*) src;
for(int32_t i = 0; i < numOfRows; ++i) {
p[i] = pSrc[numOfRows - i - 1];
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float* p = (float*) dest;
float* pSrc = (float*) src;
for(int32_t i = 0; i < numOfRows; ++i) {
p[i] = pSrc[numOfRows - i - 1];
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double* p = (double*) dest;
double* pSrc = (double*) src;
for(int32_t i = 0; i < numOfRows; ++i) {
p[i] = pSrc[numOfRows - i - 1];
}
break;
}
default: assert(0);
}
}
void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*getSourceDataBlock)(void *, const char*, int32_t)) {
if (pExprs == NULL) {
@ -387,6 +447,8 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
/* the right output has result from the right child syntax tree */
char *pRightOutput = malloc(sizeof(int64_t) * numOfRows);
char *pdata = malloc(sizeof(int64_t) * numOfRows);
if (pRight->nodeType == TSQL_NODE_EXPR) {
tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock);
}
@ -398,52 +460,75 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
* the type of returned value of one expression is always double float precious
*/
_bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr);
fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, order);
fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
} else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight
_bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr);
// set input buffer
char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, order);
if (order == TSDB_ORDER_DESC) {
reverseCopy(pdata, pInputData, pRight->pSchema->type, numOfRows);
fp(pLeftOutput, pdata, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
} else {
fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12
_bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr);
fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order);
fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
}
} else if (pLeft->nodeType == TSQL_NODE_COL) {
// column data specified on left-hand-side
char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId);
if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2
_bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr);
fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, order);
if (order == TSDB_ORDER_DESC) {
reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows);
fp(pdata, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
} else {
fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight
// column data specified on right-hand-side
char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
_bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr);
fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order);
// both columns are descending order, do not reverse the source data
fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order);
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12
_bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr);
fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order);
if (order == TSDB_ORDER_DESC) {
reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows);
fp(pdata, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
} else {
fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
}
}
} else {
// column data specified on left-hand-side
if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2
_bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr);
fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, order);
fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC);
} else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight
// column data specified on right-hand-side
char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
_bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr);
fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, order);
if (order == TSDB_ORDER_DESC) {
reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows);
fp(&pLeft->pVal->i64Key, pdata, numOfRows, 1, pOutput, TSDB_ORDER_ASC);
} else {
fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12
_bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr);
fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, order);
fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC);
}
}

873
src/query/src/qExecutor.c
View File

File diff suppressed because it is too large Load Diff

19
src/query/src/qParserImpl.c
View File

@ -405,14 +405,29 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
if (type->type == 0) {
pField->bytes = 0;
} else {
pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
int32_t bytes = -(int32_t)(type->type);
if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
// we have to postpone reporting the error because it cannot be done here
// as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow
bytes = TSDB_MAX_NCHAR_LEN + 1;
} else {
bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
}
pField->bytes = (int16_t)bytes;
}
} else if (i == TSDB_DATA_TYPE_BINARY) {
/* for binary, the TOKENTYPE is the length of binary */
if (type->type == 0) {
pField->bytes = 0;
} else {
pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE);
int32_t bytes = -(int32_t)(type->type);
if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
// refer comment for NCHAR above
bytes = TSDB_MAX_BINARY_LEN + 1;
} else {
bytes += VARSTR_HEADER_SIZE;
}
pField->bytes = (int16_t)bytes;
}
}
break;

8
src/query/src/qResultbuf.c
View File

@ -423,9 +423,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
unlink(pResultBuf->path);
tfree(pResultBuf->path);
SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
while(taosHashIterNext(iter)) {
SArray** p = (SArray**) taosHashIterGet(iter);
SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
while(p) {
size_t n = taosArrayGetSize(*p);
for(int32_t i = 0; i < n; ++i) {
SPageInfo* pi = taosArrayGetP(*p, i);
@ -434,10 +433,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
}
taosArrayDestroy(*p);
p = taosHashIterate(pResultBuf->groupSet, p);
}
taosHashDestroyIter(iter);
tdListFree(pResultBuf->lruList);
taosArrayDestroy(pResultBuf->emptyDummyIdList);
taosHashCleanup(pResultBuf->groupSet);

136
src/query/src/qUtil.c
View File

@ -43,51 +43,48 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
return size;
}
int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) {
pWindowResInfo->capacity = size;
pWindowResInfo->threshold = threshold;
pWindowResInfo->type = type;
pWindowResInfo->curIndex = -1;
pWindowResInfo->size = 0;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) {
pResultRowInfo->capacity = size;
pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES);
if (pWindowResInfo->pResult == NULL) {
pResultRowInfo->type = type;
pResultRowInfo->curIndex = -1;
pResultRowInfo->size = 0;
pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES);
if (pResultRowInfo->pResult == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
if (pWindowResInfo == NULL) {
void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) {
if (pResultRowInfo == NULL) {
return;
}
if (pWindowResInfo->capacity == 0) {
assert(pWindowResInfo->pResult == NULL);
if (pResultRowInfo->capacity == 0) {
assert(pResultRowInfo->pResult == NULL);
return;
}
if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) {
for(int32_t i = 0; i < pWindowResInfo->size; ++i) {
tfree(pWindowResInfo->pResult[i]->key);
if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) {
for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
tfree(pResultRowInfo->pResult[i]->key);
}
}
tfree(pWindowResInfo->pResult);
tfree(pResultRowInfo->pResult);
}
void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) {
if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) {
void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) {
if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) {
return;
}
// assert(pWindowResInfo->size == 1);
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
SResultRow *pWindowRes = pWindowResInfo->pResult[i];
clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
SResultRow *pWindowRes = pResultRowInfo->pResult[i];
clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
int32_t groupIndex = 0;
int64_t uid = 0;
@ -96,30 +93,30 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR
taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(groupIndex)));
}
pWindowResInfo->curIndex = -1;
pWindowResInfo->size = 0;
pResultRowInfo->curIndex = -1;
pResultRowInfo->size = 0;
pWindowResInfo->startTime = TSKEY_INITIAL_VAL;
pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL;
pResultRowInfo->startTime = TSKEY_INITIAL_VAL;
pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
}
void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) {
SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo;
if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) {
return;
}
int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo);
assert(num >= 0 && num <= numOfClosed);
int16_t type = pWindowResInfo->type;
int16_t type = pResultRowInfo->type;
int64_t uid = getResultInfoUId(pRuntimeEnv);
char *key = NULL;
int16_t bytes = -1;
for (int32_t i = 0; i < num; ++i) {
SResultRow *pResult = pWindowResInfo->pResult[i];
SResultRow *pResult = pResultRowInfo->pResult[i];
if (pResult->closed) { // remove the window slot from hash table
getResultRowKeyInfo(pResult, type, &key, &bytes);
SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
@ -129,23 +126,23 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
}
}
int32_t remain = pWindowResInfo->size - num;
int32_t remain = pResultRowInfo->size - num;
// clear all the closed windows from the window list
for (int32_t k = 0; k < remain; ++k) {
copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type);
copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type);
}
// move the unclosed window in the front of the window list
for (int32_t k = remain; k < pWindowResInfo->size; ++k) {
SResultRow *pWindowRes = pWindowResInfo->pResult[k];
clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type);
for (int32_t k = remain; k < pResultRowInfo->size; ++k) {
SResultRow *pWindowRes = pResultRowInfo->pResult[k];
clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type);
}
pWindowResInfo->size = remain;
pResultRowInfo->size = remain;
for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
SResultRow *pResult = pWindowResInfo->pResult[k];
for (int32_t k = 0; k < pResultRowInfo->size; ++k) {
SResultRow *pResult = pResultRowInfo->pResult[k];
getResultRowKeyInfo(pResult, type, &key, &bytes);
SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
@ -153,43 +150,43 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
assert(p != NULL);
int32_t v = (*p - num);
assert(v >= 0 && v <= pWindowResInfo->size);
assert(v >= 0 && v <= pResultRowInfo->size);
SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
}
pWindowResInfo->curIndex = -1;
pResultRowInfo->curIndex = -1;
}
void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) {
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) {
SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo;
if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) {
return;
}
int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo);
clearFirstNWindowRes(pRuntimeEnv, numOfClosed);
}
int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) {
int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) {
int32_t i = 0;
while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) {
while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
++i;
}
return i;
}
void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) {
assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
if (pWindowResInfo->pResult[i]->closed) {
for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
if (pResultRowInfo->pResult[i]->closed) {
continue;
}
pWindowResInfo->pResult[i]->closed = true;
pResultRowInfo->pResult[i]->closed = true;
}
}
@ -198,41 +195,41 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) {
* the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time.
* NOTE: remove redundant, only when the result set order equals to traverse order
*/
void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) {
assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size);
if (pWindowResInfo->size <= 1) {
void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) {
assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
if (pResultRowInfo->size <= 1) {
return;
}
// get the result order
int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1;
int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1;
if (order != resultOrder) {
return;
}
int32_t i = 0;
if (order == QUERY_ASC_FORWARD_STEP) {
TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey;
while (i < pWindowResInfo->size && (ekey < lastKey)) {
TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey;
while (i < pResultRowInfo->size && (ekey < lastKey)) {
++i;
}
} else if (order == QUERY_DESC_FORWARD_STEP) {
while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) {
while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) {
++i;
}
}
if (i < pWindowResInfo->size) {
pWindowResInfo->size = (i + 1);
if (i < pResultRowInfo->size) {
pResultRowInfo->size = (i + 1);
}
}
bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
return (getResultRow(pWindowResInfo, slot)->closed == true);
bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) {
return (getResultRow(pResultRowInfo, slot)->closed == true);
}
void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
getResultRow(pWindowResInfo, slot)->closed = true;
void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) {
getResultRow(pResultRowInfo, slot)->closed = true;
}
void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) {
@ -395,11 +392,10 @@ uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) {
}
SQuery* pQuery = pRuntimeEnv->pQuery;
if ((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) ||
pRuntimeEnv->groupbyNormalCol) {
if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) {
return 0;
}
STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current);
STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable);
return id->uid;
}

12
src/rpc/src/rpcMain.c
View File

@ -630,8 +630,16 @@ static void rpcReleaseConn(SRpcConn *pConn) {
} else {
// if there is an outgoing message, free it
if (pConn->outType && pConn->pReqMsg) {
if (pConn->pContext) pConn->pContext->pConn = NULL;
taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
SRpcReqContext *pContext = pConn->pContext;
if (pContext->pRsp) {
// for synchronous API, post semaphore to unblock app
pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR;
pContext->pRsp->pCont = NULL;
pContext->pRsp->contLen = 0;
tsem_post(pContext->pSem);
}
pContext->pConn = NULL;
taosRemoveRef(tsRpcRefId, pContext->rid);
}
}

15
src/rpc/src/rpcTcp.c
View File

@ -371,10 +371,13 @@ void taosCloseTcpConnection(void *chandle) {
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
SFdObj *pFdObj = chandle;
if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1;
SThreadObj *pThreadObj = pFdObj->pThreadObj;
return taosWriteMsg(pFdObj->fd, data, len);
int ret = taosWriteMsg(pFdObj->fd, data, len);
tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret);
return ret;
}
static void taosReportBrokenLink(SFdObj *pFdObj) {
@ -409,7 +412,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tDebug("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen);
return -1;
}
@ -420,7 +423,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
} else {
tTrace("TCP malloc mem:%p size:%d", buffer, size);
tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer);
}
msg = buffer + tsRpcOverhead;
@ -583,8 +586,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock(&pThreadObj->mutex);
tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d",
pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds);
tfree(pFdObj);
}

6
src/sync/inc/syncInt.h
View File

@ -139,15 +139,14 @@ typedef struct SsyncPeer {
char id[TSDB_EP_LEN + 32]; // peer vgId + end point
uint64_t version;
uint64_t sversion; // track the peer version in retrieve process
uint64_t lastFileVer; // track the file version while retrieve
uint64_t lastWalVer; // track the wal version while retrieve
int32_t syncFd;
int32_t peerFd; // forward FD
int32_t numOfRetrieves; // number of retrieves tried
int32_t fileChanged; // a flag to indicate file is changed during retrieving process
void * timer;
void * pConn;
int32_t notifyFd;
int32_t watchNum;
int32_t *watchFd;
int32_t refCount; // reference count
struct SSyncNode *pSyncNode;
} SSyncPeer;
@ -172,6 +171,7 @@ typedef struct SSyncNode {
FNotifyRole notifyRole;
FNotifyFlowCtrl notifyFlowCtrl;
FNotifyFileSynced notifyFileSynced;
FGetVersion getVersion;
pthread_mutex_t mutex;
} SSyncNode;

7
src/sync/src/syncMain.c
View File

@ -196,6 +196,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
pNode->confirmForward = pInfo->confirmForward;
pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
pNode->notifyFileSynced = pInfo->notifyFileSynced;
pNode->getVersion = pInfo->getVersion;
pNode->selfIndex = -1;
pNode->vgId = pInfo->vgId;
@ -497,7 +498,6 @@ int32_t syncDecPeerRef(SSyncPeer *pPeer) {
taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
sDebug("%s, resource is freed", pPeer->id);
tfree(pPeer->watchFd);
tfree(pPeer);
return 0;
}
@ -540,7 +540,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
pPeer->ip = ip;
pPeer->port = pInfo->nodePort;
pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId);
pPeer->peerFd = -1;
pPeer->syncFd = -1;
@ -1143,8 +1143,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
pPeer->syncFd = connFd;
syncCreateRestoreDataThread(pPeer);
} else {
sDebug("%s, TCP connection is already up(pfd:%d), close one, new pfd:%d sfd:%d", pPeer->id, pPeer->peerFd, connFd,
pPeer->syncFd);
sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd);
syncClosePeerConn(pPeer);
pPeer->peerFd = connFd;
pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);

12
src/sync/src/syncRestore.c
View File

@ -52,12 +52,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info
SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info
SFileAck fileAck;
SFileInfo minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */
SFileInfo sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */
SFileAck fileAck = {0};
int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
uint32_t pindex = 0; // index in last restore
uint32_t pindex = 0; // index in last restore
bool fileChanged = false;
*fversion = 0;
@ -134,7 +134,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
// data file is changed, code shall be set to 1
*fversion = minfo.fversion;
code = 1;
sDebug("%s, file changed while restore file", pPeer->id);
sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion);
}
if (code < 0) {
@ -160,7 +160,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
}
if (pHead->len == 0) {
sDebug("%s, wal is synced over", pPeer->id);
sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer);
code = 0;
break;
} // wal sync over

440
src/sync/src/syncRetrieve.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include <sys/inotify.h>
#include "os.h"
#include "taoserror.h"
#include "tlog.h"
#include "tutil.h"
#include "tglobal.h"
@ -25,110 +26,102 @@
#include "tsync.h"
#include "syncInt.h"
static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
sDebug("%s, start to monitor:%s", pPeer->id, name);
if (pPeer->notifyFd <= 0) {
pPeer->watchNum = 0;
pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
if (pPeer->notifyFd < 0) {
sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno));
return -1;
}
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
}
int32_t *wd = pPeer->watchFd + pPeer->watchNum;
if (*wd >= 0) {
if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
sError("%s, failed to remove wd:%d since %s", pPeer->id, *wd, strerror(errno));
return -1;
}
}
*wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_DELETE);
if (*wd == -1) {
sError("%s, failed to add %s since %s", pPeer->id, name, strerror(errno));
static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
if (code != 0) {
sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
return -1;
} else {
sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum);
}
pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles;
return 0;
pPeer->lastWalVer = wver;
return code;
}
static int32_t syncAreFilesModified(SSyncPeer *pPeer) {
if (pPeer->notifyFd <= 0) return 0;
static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
if (code != 0) {
sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
return true;
}
char buf[2048];
int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno));
if (wver != pPeer->lastWalVer) {
sDebug("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer);
return true;
}
return false;
}
static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
if (code != 0) {
sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
return -1;
}
int32_t code = 0;
if (len > 0) {
const struct inotify_event *event;
char *ptr;
for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
event = (const struct inotify_event *)ptr;
if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) {
sDebug("%s, processed file is changed", pPeer->id);
pPeer->fileChanged = 1;
code = 1;
break;
}
}
pPeer->lastFileVer = fver;
return code;
}
static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
uint64_t fver, wver;
int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
if (code != 0) {
sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
pPeer->fileChanged = 1;
return true;
}
return code;
if (fver != pPeer->lastFileVer) {
sDebug("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer);
pPeer->fileChanged = 1;
return true;
}
pPeer->fileChanged = 0;
return false;
}
static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
SSyncNode *pNode = pPeer->pSyncNode;
SFileInfo fileInfo;
SFileAck fileAck;
SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo));
SFileAck fileAck = {0};
int32_t code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
memset(&fileInfo, 0, sizeof(fileInfo));
memset(&fileAck, 0, sizeof(fileAck));
if (syncGetFileVersion(pNode, pPeer) < 0) return -1;
while (1) {
// retrieve file info
fileInfo.name[0] = 0;
fileInfo.size = 0;
fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
&fileInfo.size, &fileInfo.fversion);
// fileInfo.size = htonl(size);
sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
// send the file info
int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
if (ret < 0) {
code = -1;
sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break;
}
// if no file anymore, break
if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
sDebug("%s, no more files to sync", pPeer->id);
code = 0;
sDebug("%s, no more files to sync", pPeer->id);
break;
}
// wait for the ack from peer
ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck));
if (ret < 0) {
code = -1;
sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break;
}
@ -136,15 +129,6 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
// set the peer sync version
pPeer->sversion = fileInfo.fversion;
// get the full path to file
snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
// add the file into watch list
if (syncAddIntoWatchList(pPeer, name) < 0) {
sError("%s, failed to watch file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break;
}
// if sync is not required, continue
if (fileAck.sync == 0) {
fileInfo.index++;
@ -152,9 +136,13 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
continue;
}
// get the full path to file
snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
// send the file to peer
int32_t sfd = open(name, O_RDONLY);
if (sfd < 0) {
code = -1;
sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break;
}
@ -162,138 +150,112 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
close(sfd);
if (ret < 0) {
code = -1;
sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
break;
}
sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size);
sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
fileInfo.index++;
// check if processed files are modified
if (syncAreFilesModified(pPeer) != 0) {
sInfo("%s, file:%s are modified while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
if (syncAreFilesModified(pNode, pPeer)) {
code = -1;
break;
}
}
if (code < 0) {
sError("%s, failed to retrieve file", pPeer->id);
if (code != TSDB_CODE_SUCCESS) {
sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code);
}
return code;
}
/* if only a partial record is read out, set the IN_MODIFY flag in event,
so upper layer will reload the file to get a complete record */
static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) {
int32_t ret;
// if only a partial record is read out, upper layer will reload the file to get a complete record
static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) {
int32_t ret = read(sfd, pHead, sizeof(SWalHead));
if (ret < 0) {
sError("sfd:%d, failed to read wal head since %s, ret:%d", sfd, strerror(errno), ret);
return -1;
}
ret = read(sfd, pHead, sizeof(SWalHead));
if (ret < 0) return -1;
if (ret == 0) return 0;
if (ret == 0) {
sTrace("sfd:%d, read to the end of file, ret:%d", sfd, ret);
return 0;
}
if (ret != sizeof(SWalHead)) {
// file is not at end yet, it shall be reloaded
*pEvent = *pEvent | IN_MODIFY;
sDebug("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret);
return 0;
}
assert(pHead->len <= TSDB_MAX_WAL_SIZE);
ret = read(sfd, pHead->cont, pHead->len);
if (ret < 0) return -1;
if (ret < 0) {
sError("sfd:%d, failed to read wal content since %s, ret:%d", sfd, strerror(errno), ret);
return -1;
}
if (ret != pHead->len) {
// file is not at end yet, it shall be reloaded
*pEvent = *pEvent | IN_MODIFY;
sDebug("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret);
return 0;
}
return sizeof(SWalHead) + pHead->len;
}
static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
pPeer->watchNum = 0;
taosClose(pPeer->notifyFd);
pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
if (pPeer->notifyFd < 0) {
sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno));
return -1;
}
if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
if (pPeer->watchFd == NULL) {
sError("%s, failed to allocate watchFd", pPeer->id);
return -1;
}
memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
int32_t *wd = pPeer->watchFd;
*wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
if (*wd == -1) {
sError("%s, failed to watch last wal since %s", pPeer->id, strerror(errno));
return -1;
}
return 0;
}
static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
char buf[2048];
int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
if (len < 0 && errno != EAGAIN) {
sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno));
return -1;
}
if (len == 0) return 0;
struct inotify_event *event;
for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
event = (struct inotify_event *)ptr;
if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY;
if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE;
}
if (pEvent != 0) sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
return 0;
}
static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) {
SWalHead *pHead = malloc(SYNC_MAX_SIZE);
int32_t code = -1;
int32_t bytes = 0;
int32_t sfd;
sfd = open(name, O_RDONLY);
static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) {
int32_t sfd = open(name, O_RDONLY);
if (sfd < 0) {
free(pHead);
sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno));
return -1;
}
(void)lseek(sfd, offset, SEEK_SET);
sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion);
int32_t code = taosLSeek(sfd, offset, SEEK_SET);
if (code < 0) {
sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno));
close(sfd);
return -1;
}
sDebug("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion);
SWalHead *pHead = malloc(SYNC_MAX_SIZE);
int32_t bytes = 0;
while (1) {
int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent);
if (wsize < 0) break;
if (wsize == 0) {
code = 0;
code = syncReadOneWalRecord(sfd, pHead);
if (code < 0) {
sError("%s, failed to read one record from wal:%s", pPeer->id, name);
break;
}
if (code == 0) {
code = bytes;
sDebug("%s, read to the end of wal, bytes:%d", pPeer->id, bytes);
break;
}
sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version);
int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if (ret != wsize) break;
pPeer->sversion = pHead->version;
int32_t wsize = code;
int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
if (ret != wsize) {
code = -1;
sError("%s, failed to forward wal since %s, hver:%" PRIu64, pPeer->id, strerror(errno), pHead->version);
break;
}
pPeer->sversion = pHead->version;
bytes += wsize;
if (pHead->version >= fversion && fversion > 0) {
code = 0;
bytes = 0;
sDebug("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion);
break;
}
}
@ -301,92 +263,62 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi
free(pHead);
close(sfd);
if (code == 0) return bytes;
return -1;
return code;
}
static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
SSyncNode *pNode = pPeer->pSyncNode;
int32_t code = -1;
char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file
int32_t once = 0; // last WAL has once ever been processed
int64_t offset = 0;
uint64_t fversion = 0;
char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file
if (syncAreFilesModified(pPeer) != 0) return -1;
// get full path to wal file
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
while (1) {
int32_t once = 0; // last WAL has once ever been processed
int64_t offset = 0;
uint64_t fversion = 0;
uint32_t event = 0;
if (syncAreFilesModified(pNode, pPeer)) return -1;
if (syncGetWalVersion(pNode, pPeer) < 0) return -1;
// get full path to wal file
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
// monitor last wal
if (syncMonitorLastWal(pPeer, fname) < 0) break;
while (1) {
int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event);
if (bytes < 0) break;
// check file changes
if (syncCheckLastWalChanges(pPeer, &event) < 0) break;
// if file is not updated or updated once, set the fversion and sstatus
if (((event & IN_MODIFY) == 0) || once) {
if (fversion == 0) {
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt
sDebug("%s, fversion is 0 then set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
fversion = nodeVersion; // must read data to fversion
}
}
// if all data up to fversion is read out, it is over
if (pPeer->sversion >= fversion && fversion > 0) {
code = 0;
sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes);
break;
}
// if all data are read out, and no update
if ((bytes == 0) && ((event & IN_MODIFY) == 0)) {
// wal file is closed, break
if (event & IN_CLOSE_WRITE) {
code = 0;
sDebug("%s, current wal is closed", pPeer->id);
break;
}
// wal not closed, it means some data not flushed to disk, wait for a while
usleep(10000);
}
// if bytes>0, file is updated, or fversion is not reached but file still open, read again
once = 1;
offset += bytes;
sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes);
event = event & (~IN_MODIFY); // clear IN_MODIFY flag
int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset);
if (bytes < 0) {
sDebug("%s, failed to retrieve last wal", pPeer->id);
return bytes;
}
if (code < 0) break;
if (pPeer->sversion >= fversion && fversion > 0) break;
// check file changes
bool walModified = syncIsWalModified(pNode, pPeer);
index++;
wname[0] = 0;
code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
if (code < 0) break;
if (wname[0] == 0) {
code = 0;
break;
// if file is not updated or updated once, set the fversion and sstatus
if (!walModified || once) {
if (fversion == 0) {
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt
fversion = nodeVersion; // must read data to fversion
sDebug("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion);
}
}
// current last wal is closed, there is a new one
sDebug("%s, last wal is closed, try new one", pPeer->id);
// if all data up to fversion is read out, it is over
if (pPeer->sversion >= fversion && fversion > 0) {
sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d sver:%" PRIu64, pPeer->id, fversion, bytes,
pPeer->sversion);
return 0;
}
// if all data are read out, and no update
if (bytes == 0 && !walModified) {
// wal not closed, it means some data not flushed to disk, wait for a while
usleep(10000);
}
// if bytes > 0, file is updated, or fversion is not reached but file still open, read again
once = 1;
offset += bytes;
sDebug("%s, continue retrieve last wal, bytes:%d offset:%" PRId64, pPeer->id, bytes, offset);
}
taosClose(pPeer->notifyFd);
return code;
return -1;
}
static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
@ -394,7 +326,6 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
char fname[TSDB_FILENAME_LEN * 3];
char wname[TSDB_FILENAME_LEN * 2];
int32_t size;
struct stat fstat;
int32_t code = -1;
int64_t index = 0;
@ -402,9 +333,14 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
// retrieve wal info
wname[0] = 0;
code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
if (code < 0) break; // error
if (code < 0) {
sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code);
break;
}
if (wname[0] == 0) { // no wal file
sDebug("%s, no wal file", pPeer->id);
code = 0;
sDebug("%s, no wal file anymore", pPeer->id);
break;
}
@ -416,22 +352,35 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
// get the full path to wal file
snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
// send wal file,
// inotify is not required, old wal file won't be modified, even remove is ok
if (stat(fname, &fstat) < 0) break;
size = fstat.st_size;
// send wal file, old wal file won't be modified, even remove is ok
struct stat fstat;
if (stat(fname, &fstat) < 0) {
code = -1;
sDebug("%s, failed to stat wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code);
break;
}
size = fstat.st_size;
sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);
int32_t sfd = open(fname, O_RDONLY);
if (sfd < 0) break;
if (sfd < 0) {
code = -1;
sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code);
break;
}
code = taosSendFile(pPeer->syncFd, sfd, NULL, size);
close(sfd);
if (code < 0) break;
if (code < 0) {
sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code);
break;
}
index++;
if (syncAreFilesModified(pPeer) != 0) break;
if (syncAreFilesModified(pNode, pPeer)) {
code = -1;
break;
}
}
if (code == 0) {
@ -440,9 +389,9 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
SWalHead walHead;
memset(&walHead, 0, sizeof(walHead));
code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
} else {
sError("%s, failed to send wal since %s", pPeer->id, strerror(errno));
sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code);
}
return code;
@ -481,18 +430,19 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
pPeer->sversion = 0;
pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
sInfo("%s, start to retrieve file, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
if (syncRetrieveFile(pPeer) < 0) {
sError("%s, failed to retrieve file", pPeer->id);
sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
if (syncRetrieveFile(pPeer) != 0) {
sError("%s, failed to retrieve files", pPeer->id);
return -1;
}
// if no files are synced, there must be wal to sync, sversion must be larger than one
if (pPeer->sversion == 0) pPeer->sversion = 1;
sInfo("%s, start to retrieve wal", pPeer->id);
if (syncRetrieveWal(pPeer) < 0) {
sError("%s, failed to retrieve wal", pPeer->id);
sInfo("%s, start to retrieve wals", pPeer->id);
int32_t code = syncRetrieveWal(pPeer);
if (code != 0) {
sError("%s, failed to retrieve wals, code:0x%x", pPeer->id, code);
return -1;
}
@ -506,7 +456,6 @@ void *syncRetrieveData(void *param) {
if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves);
pPeer->fileChanged = 0;
pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
if (pPeer->syncFd < 0) {
sError("%s, failed to open socket to sync", pPeer->id);
@ -529,7 +478,6 @@ void *syncRetrieveData(void *param) {
}
pPeer->fileChanged = 0;
taosClose(pPeer->notifyFd);
taosClose(pPeer->syncFd);
syncDecPeerRef(pPeer);

13
src/tsdb/src/tsdbFile.c
View File

@ -248,8 +248,13 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
if (pGroup == NULL) { // if not exists, create one
pFGroup->fileId = fid;
for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0)
goto _err;
if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) {
for (int i = type; i >= 0; i--) {
remove(pFGroup->files[i].fname);
}
return NULL;
}
}
pthread_rwlock_wrlock(&pFileH->fhlock);
@ -261,10 +266,6 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
}
return pGroup;
_err:
for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
return NULL;
}
void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) {

10
src/tsdb/src/tsdbRWHelper.c
View File

@ -917,6 +917,8 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst);
}
ASSERT((blkIdx == pIdx->numOfBlocks -1) || (!pCompBlock->last));
tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
blkIdx);
@ -1042,6 +1044,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast;
pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last;
ASSERT((blkIdx == pIdx->numOfBlocks-1) || (!pCompBlock->last));
tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
blkIdx);
@ -1622,11 +1626,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
pCfg->update);
if (pDataCols->numOfRows == 0) break;
if (tblkIdx == pIdx->numOfBlocks - 1) {
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
} else {
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
}
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
if (round == 0) {
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;

54
src/util/inc/hash.h
View File

@ -31,16 +31,18 @@ extern "C" {
typedef void (*_hash_free_fn_t)(void *param);
typedef struct SHashNode {
// char *key;
struct SHashNode *next;
uint32_t hashVal; // the hash value of key
uint32_t keyLen; // length of the key
// char *data;
size_t dataLen; // length of data
int8_t count; // reference count
int8_t removed; // flag to indicate removed
char data[];
} SHashNode;
#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode))
#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen)
#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode));
typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1,
@ -65,15 +67,6 @@ typedef struct SHashObj {
SArray *pMemBlock; // memory block allocated for SHashEntry
} SHashObj;
typedef struct SHashMutableIterator {
SHashObj *pHashObj;
int32_t entryIndex;
SHashNode *pCur;
SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current
size_t numOfChecked; // already check number of elements in hash table
size_t numOfEntries; // number of entries while the iterator is created
} SHashMutableIterator;
/**
* init the hash table
*
@ -142,33 +135,9 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
*/
void taosHashCleanup(SHashObj *pHashObj);
/**
*
* @param pHashObj
* @return
*/
SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj);
/**
*
* @param iter
* @return
*/
bool taosHashIterNext(SHashMutableIterator *iter);
/**
*
* @param iter
* @return
*/
void *taosHashIterGet(SHashMutableIterator *iter);
/**
*
* @param iter
* @return
*/
void* taosHashDestroyIter(SHashMutableIterator* iter);
/*
void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *);
*/
/**
*
@ -179,6 +148,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj);
size_t taosHashGetMemSize(const SHashObj *pHashObj);
void *taosHashIterate(SHashObj *pHashObj, void *p);
void taosHashCancelIterate(SHashObj *pHashObj, void *p);
#ifdef __cplusplus
}
#endif

21
src/util/inc/tfile.h
View File

@ -20,23 +20,26 @@
extern "C" {
#endif
#include <unistd.h>
// init taos file module
int32_t tfinit();
int32_t tfInit();
// clean up taos file module
void tfcleanup();
void tfCleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfopen(const char *pathname, int32_t flags);
int64_t tfclose(int64_t tfd);
int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
int64_t tfread(int64_t tfd, void *buf, int64_t count);
int64_t tfOpen(const char *pathname, int32_t flags);
int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode);
int64_t tfClose(int64_t tfd);
int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
int64_t tfRead(int64_t tfd, void *buf, int64_t count);
int32_t tfFsync(int64_t tfd);
bool tfValid(int64_t tfd);
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
int32_t tfFtruncate(int64_t tfd, int64_t length);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TREF_H
#endif // TDENGINE_TFILE_H

352
src/util/src/hash.c
View File

@ -76,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
SHashNode *pNode = pe->next;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
assert(pNode->hashVal == hashVal);
break;
}
@ -114,15 +114,25 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
* @param dsize size of actual data
* @return hash node
*/
static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
assert(pNode->keyLen == pNewNode->keyLen);
pNode->count--;
if (prev != NULL) {
prev->next = pNewNode;
} else {
pe->next = pNewNode;
}
pNewNode->next = pNode->next;
if (pNode->count <= 0) {
pNewNode->next = pNode->next;
DO_FREE_HASH_NODE(pNode);
} else {
pNewNode->next = pNode;
pe->num++;
atomic_add_fetch_64(&pHashObj->size, 1);
}
return pNewNode;
}
@ -139,11 +149,11 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode);
* @param pIter
* @return
*/
static SHashNode *getNextHashNode(SHashMutableIterator *pIter);
SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
if (capacity == 0 || fn == NULL) {
return NULL;
assert(fn != NULL);
if (capacity == 0) {
capacity = 4;
}
SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj));
@ -213,7 +223,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
SHashNode* prev = NULL;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
assert(pNode->hashVal == hashVal);
break;
}
@ -244,8 +254,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
} else {
// not support the update operation, return error
if (pHashObj->enableUpdate) {
doUpdateHashNode(pe, prev, pNode, pNewNode);
DO_FREE_HASH_NODE(pNode);
doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode);
} else {
DO_FREE_HASH_NODE(pNewNode);
}
@ -335,22 +344,10 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHashEntry *pe = pHashObj->hashList[slot];
// no data, return directly
if (pe->num == 0) {
__rd_unlock(&pHashObj->lock, pHashObj->type);
return -1;
}
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWLockLatch(&pe->latch);
}
if (pe->num == 0) {
assert(pe->next == NULL);
} else {
assert(pe->next != NULL);
}
// double check after locked
if (pe->num == 0) {
assert(pe->next == NULL);
@ -360,37 +357,37 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
return -1;
}
int code = -1;
SHashNode *pNode = pe->next;
SHashNode *pRes = NULL;
SHashNode *prevNode = NULL;
// remove it
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
pe->num -= 1;
pRes = pNode;
pe->next = pNode->next;
} else {
while (pNode->next != NULL) {
if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) {
assert((pNode->next)->hashVal == hashVal);
break;
while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0)
break;
prevNode = pNode;
pNode = pNode->next;
}
if (pNode) {
code = 0; // it is found
pNode->count--;
pNode->removed = 1;
if (pNode->count <= 0) {
if (prevNode) {
prevNode->next = pNode->next;
} else {
pe->next = pNode->next;
}
if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
pNode = pNode->next;
pe->num--;
atomic_sub_fetch_64(&pHashObj->size, 1);
FREE_HASH_NODE(pHashObj, pNode);
}
if (pNode->next != NULL) {
pe->num -= 1;
pRes = pNode->next;
pNode->next = pNode->next->next;
}
}
if (pe->num == 0) {
assert(pe->next == NULL);
} else {
assert(pe->next != NULL);
}
}
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWUnLockLatch(&pe->latch);
@ -398,17 +395,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
__rd_unlock(&pHashObj->lock, pHashObj->type);
if (data != NULL && pRes != NULL) {
memcpy(data, GET_HASH_NODE_DATA(pRes), dsize);
}
if (pRes != NULL) {
atomic_sub_fetch_64(&pHashObj->size, 1);
FREE_HASH_NODE(pHashObj, pRes);
return 0;
} else {
return -1;
}
return code;
}
int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) {
@ -531,98 +518,6 @@ void taosHashCleanup(SHashObj *pHashObj) {
free(pHashObj);
}
SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator));
if (pIter == NULL) {
return NULL;
}
pIter->pHashObj = pHashObj;
// keep it in local variable, in case the resize operation expand the size
pIter->numOfEntries = pHashObj->capacity;
return pIter;
}
bool taosHashIterNext(SHashMutableIterator *pIter) {
if (pIter == NULL) {
return false;
}
size_t size = taosHashGetSize(pIter->pHashObj);
if (size == 0) {
return false;
}
// check the first one
if (pIter->numOfChecked == 0) {
assert(pIter->pCur == NULL && pIter->pNext == NULL);
while (1) {
SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
if (pEntry->num == 0) {
assert(pEntry->next == NULL);
pIter->entryIndex++;
continue;
}
if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
taosRLockLatch(&pEntry->latch);
}
pIter->pCur = pEntry->next;
if (pIter->pCur->next) {
pIter->pNext = pIter->pCur->next;
if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
taosRUnLockLatch(&pEntry->latch);
}
} else {
if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
taosRUnLockLatch(&pEntry->latch);
}
pIter->pNext = getNextHashNode(pIter);
}
break;
}
pIter->numOfChecked++;
return true;
} else {
assert(pIter->pCur != NULL);
if (pIter->pNext) {
pIter->pCur = pIter->pNext;
} else { // no more data in the hash list
return false;
}
pIter->numOfChecked++;
if (pIter->pCur->next) {
pIter->pNext = pIter->pCur->next;
} else {
pIter->pNext = getNextHashNode(pIter);
}
return true;
}
}
void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); }
void *taosHashDestroyIter(SHashMutableIterator *iter) {
if (iter == NULL) {
return NULL;
}
free(iter);
return NULL;
}
// for profile only
int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
if (pHashObj == NULL || pHashObj->size == 0) {
@ -759,6 +654,8 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
pNewNode->keyLen = (uint32_t)keyLen;
pNewNode->hashVal = hashVal;
pNewNode->dataLen = dsize;
pNewNode->count = 1;
memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
@ -775,35 +672,6 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) {
pEntry->num += 1;
}
SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
assert(pIter != NULL);
pIter->entryIndex++;
SHashNode *p = NULL;
while (pIter->entryIndex < pIter->numOfEntries) {
SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
if (pEntry->num == 0) {
pIter->entryIndex++;
continue;
}
if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
taosRLockLatch(&pEntry->latch);
}
p = pEntry->next;
if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
taosRUnLockLatch(&pEntry->latch);
}
return p;
}
return NULL;
}
size_t taosHashGetMemSize(const SHashObj *pHashObj) {
if (pHashObj == NULL) {
return 0;
@ -811,3 +679,129 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
}
// release the pNode, return next pNode, and lock the current entry
static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p);
SHashNode *prevNode = NULL;
*slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity);
SHashEntry *pe = pHashObj->hashList[*slot];
// lock entry
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWLockLatch(&pe->latch);
}
SHashNode *pNode = pe->next;
while (pNode) {
if (pNode == pOld)
break;
prevNode = pNode;
pNode = pNode->next;
}
if (pNode) {
pNode = pNode->next;
while (pNode) {
if (pNode->removed == 0) break;
pNode = pNode->next;
}
pOld->count--;
if (pOld->count <=0) {
if (prevNode) {
prevNode->next = pOld->next;
} else {
pe->next = pOld->next;
}
pe->num--;
atomic_sub_fetch_64(&pHashObj->size, 1);
FREE_HASH_NODE(pHashObj, pOld);
}
} else {
uError("pNode:%p data:%p is not there!!!", pNode, p);
}
return pNode;
}
void *taosHashIterate(SHashObj *pHashObj, void *p) {
if (pHashObj == NULL) return NULL;
int slot = 0;
char *data = NULL;
// only add the read lock to disable the resize process
__rd_lock(&pHashObj->lock, pHashObj->type);
SHashNode *pNode = NULL;
if (p) {
pNode = taosHashReleaseNode(pHashObj, p, &slot);
if (pNode == NULL) {
SHashEntry *pe = pHashObj->hashList[slot];
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWUnLockLatch(&pe->latch);
}
slot = slot + 1;
}
}
if (pNode == NULL) {
for (; slot < pHashObj->capacity; ++slot) {
SHashEntry *pe = pHashObj->hashList[slot];
// lock entry
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWLockLatch(&pe->latch);
}
pNode = pe->next;
while (pNode) {
if (pNode->removed == 0) break;
pNode = pNode->next;
}
if (pNode) break;
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWUnLockLatch(&pe->latch);
}
}
}
if (pNode) {
SHashEntry *pe = pHashObj->hashList[slot];
pNode->count++;
data = GET_HASH_NODE_DATA(pNode);
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWUnLockLatch(&pe->latch);
}
}
__rd_unlock(&pHashObj->lock, pHashObj->type);
return data;
}
void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
if (pHashObj == NULL || p == NULL) return;
// only add the read lock to disable the resize process
__rd_lock(&pHashObj->lock, pHashObj->type);
int slot;
taosHashReleaseNode(pHashObj, p, &slot);
SHashEntry *pe = pHashObj->hashList[slot];
if (pHashObj->type == HASH_ENTRY_LOCK) {
taosWUnLockLatch(&pe->latch);
}
__rd_unlock(&pHashObj->lock, pHashObj->type);
}

80
src/util/src/tfile.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tulog.h"
@ -21,40 +22,52 @@
static int32_t tsFileRsetId = -1;
static void taosCloseFile(void *p) {
static void tfCloseFile(void *p) {
close((int32_t)(uintptr_t)p);
}
int32_t tfinit() {
tsFileRsetId = taosOpenRef(2000, taosCloseFile);
return tsFileRsetId;
int32_t tfInit() {
tsFileRsetId = taosOpenRef(2000, tfCloseFile);
if (tsFileRsetId > 0) {
return 0;
} else {
return -1;
}
}
void tfcleanup() {
void tfCleanup() {
if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
tsFileRsetId = -1;
}
int64_t tfopen(const char *pathname, int32_t flags) {
int32_t fd = open(pathname, flags);
static int64_t tfOpenImp(int32_t fd) {
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
void *p = (void *)(int64_t)fd;
void * p = (void *)(int64_t)fd;
int64_t rid = taosAddRef(tsFileRsetId, p);
if (rid < 0) close(fd);
return rid;
}
int64_t tfclose(int64_t tfd) {
int64_t tfOpen(const char *pathname, int32_t flags) {
int32_t fd = open(pathname, flags);
return tfOpenImp(fd);
}
int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) {
int32_t fd = open(pathname, flags, mode);
return tfOpenImp(fd);
}
int64_t tfClose(int64_t tfd) {
return taosRemoveRef(tsFileRsetId, tfd);
}
int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
@ -67,7 +80,7 @@ int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
return ret;
}
int64_t tfread(int64_t tfd, void *buf, int64_t count) {
int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
@ -79,3 +92,44 @@ int64_t tfread(int64_t tfd, void *buf, int64_t count) {
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
int32_t tfFsync(int64_t tfd) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int32_t code = fsync(fd);
taosReleaseRef(tsFileRsetId, tfd);
return code;
}
bool tfValid(int64_t tfd) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return false;
taosReleaseRef(tsFileRsetId, tfd);
return true;
}
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosLSeek(fd, offset, whence);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
int32_t tfFtruncate(int64_t tfd, int64_t length) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int32_t code = taosFtruncate(fd, length);
taosReleaseRef(tsFileRsetId, tfd);
return code;
}

Some files were not shown because too many files have changed in this diff Show More