xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/update

This commit is contained in:
Hongze Cheng 2020-10-29 16:01:23 +08:00
parent 57e93db149 54b34680d6
commit af08c4a5a1
21 changed files with 712 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
Jenkinsfile vendored
View File

@ -86,6 +86,13 @@ pipeline {
./crash_gen.sh -a -p -t 4 -s 2000
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4
./handle_crash_gen_val_log.sh
'''
}
sh '''
date
cd ${WKC}/tests
@ -131,14 +138,33 @@ pipeline {
sh'''
cd ${WORKSPACE}
git checkout develop
cd tests/gotest
bash batchtest.sh
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
}
}
@ -146,5 +172,82 @@ pipeline {
}
}
post {
success {
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
}

7
src/client/src/tscAsync.c
View File

@ -427,8 +427,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
} else {
assert(code == TSDB_CODE_SUCCESS);
}
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL);
// param already freed by other routine and pSql in tscCache when ctrl + c
if (atomic_load_ptr(&pSql->param) == NULL) {
return;
}
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
SSqlObj * pParObj = trs->pParentSql;

4
src/inc/taosmsg.h
View File

@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" )
#ifndef TAOS_MESSAGE_C
TSDB_MSG_TYPE_MAX // 105
#endif

25
src/inc/twal.h
View File

@ -24,20 +24,20 @@ extern "C" {
#define TAOS_WAL_FSYNC 2
typedef struct {
int8_t msgType;
int8_t reserved[3];
int32_t len;
uint64_t version;
uint32_t signature;
uint32_t cksum;
char cont[];
int8_t msgType;
int8_t reserved[3];
int32_t len;
uint64_t version;
uint32_t signature;
uint32_t cksum;
char cont[];
} SWalHead;
typedef struct {
int8_t walLevel; // wal level
int32_t fsyncPeriod; // millisecond
int8_t wals; // number of WAL files;
int8_t keep; // keep the wal file when closed
int8_t walLevel; // wal level
int32_t fsyncPeriod; // millisecond
int8_t wals; // number of WAL files;
int8_t keep; // keep the wal file when closed
} SWalCfg;
typedef void* twalh; // WAL HANDLE
@ -53,9 +53,6 @@ int walRestore(twalh, void *pVnode, FWalWrite writeFp);
int walGetWalFile(twalh, char *name, uint32_t *index);
int64_t walGetVersion(twalh);
extern int wDebugFlag;
#ifdef __cplusplus
}
#endif

5
src/kit/shell/src/shellMain.c
View File

@ -80,7 +80,10 @@ int main(int argc, char* argv[]) {
shellParseArgument(argc, argv, &args);
if (args.netTestRole && args.netTestRole[0] != 0) {
taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole);
taos_init();
CmdArguments cmdArgs;
memcpy(&cmdArgs, &args, sizeof(SShellArguments));
taosNetTest(&cmdArgs);
exit(0);
}

4
src/mnode/src/mnodeVgroup.c
View File

@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
for (int32_t i = 0; i < pShow->maxReplica; ++i) {
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1);
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1);
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
}

7
src/rpc/src/rpcMain.c
View File

@ -1076,6 +1076,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
}
} else { // msg is passed to app only parsing is ok
if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS);
rpcFreeMsg(pRecv->msg);
return pConn;
}
rpcProcessIncomingMsg(pConn, pHead, pContext);
}
}

4
src/sync/src/syncRestore.c
View File

@ -56,6 +56,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
int code = -1;
char name[TSDB_FILENAME_LEN * 2] = {0};
uint32_t pindex = 0; // index in last restore
bool fileChanged = false;
*fversion = 0;
sinfo.index = 0;
@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
close(dfd);
if (ret < 0) break;
fileChanged = true;
sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
}
if (code == 0 && (minfo.fversion != sinfo.fversion)) {
if (code == 0 && fileChanged) {
// data file is changed, code shall be set to 1
*fversion = minfo.fversion;
code = 1;

26
src/tsdb/src/tsdbFile.c
View File

@ -79,7 +79,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
DIR * dir = NULL;
int fid = 0;
int vid = 0;
regex_t regex1, regex2;
regex_t regex1 = {0}, regex2 = {0};
int code = 0;
char fname[TSDB_FILENAME_LEN] = "\0";
@ -95,9 +95,27 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
dir = opendir(tDataDir);
if (dir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
if (errno == ENOENT) {
tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir);
terrno = TAOS_SYSTEM_ERROR(errno);
if (taosMkDir(tDataDir, 0755) < 0) {
tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
dir = opendir(tDataDir);
if (dir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
} else {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
}
code = regcomp(&regex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED);

1
src/util/CMakeLists.txt
View File

@ -1,6 +1,7 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(tutil ${SRC})
TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)

22
src/util/inc/tnettest.h
View File

@ -20,7 +20,27 @@
extern "C" {
#endif
void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole);
typedef struct CmdArguments {
char* host;
char* password;
char* user;
char* auth;
char* database;
char* timezone;
bool is_raw_time;
bool is_use_passwd;
char file[TSDB_FILENAME_LEN];
char dir[TSDB_FILENAME_LEN];
int threadNum;
char* commands;
int abort;
int port;
int endPort;
int pktLen;
char* netTestRole;
} CmdArguments;
void taosNetTest(CmdArguments* args);
#ifdef __cplusplus
}

230
src/util/src/tnettest.c
View File

@ -15,11 +15,16 @@
#include "os.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "taoserror.h"
#include "tulog.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tsocket.h"
#include "trpc.h"
#include "rpcHead.h"
#include "tutil.h"
#include "tnettest.h"
#define MAX_PKG_LEN (64*1000)
#define BUFFER_SIZE (MAX_PKG_LEN + 1024)
@ -30,9 +35,15 @@ typedef struct {
uint16_t pktLen;
} info_s;
static char serverFqdn[TSDB_FQDN_LEN];
extern int tsRpcMaxUdpSize;
static char g_user[TSDB_USER_LEN+1] = {0};
static char g_pass[TSDB_PASSWORD_LEN+1] = {0};
static char g_serverFqdn[TSDB_FQDN_LEN] = {0};
static uint16_t g_startPort = 0;
static uint16_t g_endPort = 6042;
static uint32_t g_pktLen = 0;
static void *bindUdpPort(void *sarg) {
info_s *pinfo = (info_s *)sarg;
@ -321,20 +332,146 @@ static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uin
return ;
}
static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) {
uint32_t serverIp = taosGetIpFromFqdn(serverFqdn);
void* tnetInitRpc(char* secretEncrypt, char spi) {
SRpcInit rpcInit;
void* pRpcConn = NULL;
taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt);
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
rpcInit.label = "NET-TEST";
rpcInit.numOfThreads = 1; // every DB connection has only one thread
rpcInit.cfp = NULL;
rpcInit.sessions = 16;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = g_user;
rpcInit.idleTime = 2000;
rpcInit.ckey = "key";
rpcInit.spi = spi;
rpcInit.secret = secretEncrypt;
pRpcConn = rpcOpen(&rpcInit);
return pRpcConn;
}
static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) {
SRpcEpSet epSet;
SRpcMsg reqMsg;
SRpcMsg rspMsg;
void* pRpcConn;
char secretEncrypt[32] = {0};
pRpcConn = tnetInitRpc(secretEncrypt, spi);
if (NULL == pRpcConn) {
return -1;
}
memset(&epSet, 0, sizeof(SRpcEpSet));
epSet.inUse = 0;
epSet.numOfEps = 1;
epSet.port[0] = port;
strcpy(epSet.fqdn[0], serverFqdn);
reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST;
reqMsg.pCont = rpcMallocCont(pktLen);
reqMsg.contLen = pktLen;
reqMsg.code = 0;
reqMsg.handle = NULL; // rpc handle returned to app
reqMsg.ahandle = NULL; // app handle set by client
rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg);
// handle response
if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) {
//printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code));
return -1;
}
rpcFreeCont(rspMsg.pCont);
rpcClose(pRpcConn);
return 0;
}
static void rpcCheckPort(uint32_t hostIp) {
int ret;
char spi;
for (uint16_t port = g_startPort; port <= g_endPort; port++) {
//printf("test: %s:%d\n", info.host, port);
printf("\n");
//================ check tcp port ================
int32_t pktLen;
if (g_pktLen <= tsRpcMaxUdpSize) {
pktLen = tsRpcMaxUdpSize + 1000;
} else {
pktLen = g_pktLen;
}
spi = 1;
ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
if (ret != 0) {
spi = 0;
ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
if (ret != 0) {
printf("TCP port:%d test fail.\t\t", port);
} else {
//printf("tcp port:%d test ok.\t\t", port);
printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
}
} else {
//printf("tcp port:%d test ok.\t\t", port);
printf("TCP port:\033[32m%d test OK\033[0m\t\t", port);
}
//================ check udp port ================
if (g_pktLen >= tsRpcMaxUdpSize) {
pktLen = tsRpcMaxUdpSize - 1000;
} else {
pktLen = g_pktLen;
}
spi = 0;
ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
if (ret != 0) {
spi = 1;
ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi);
if (ret != 0) {
printf("udp port:%d test fail.\t\n", port);
} else {
//printf("udp port:%d test ok.\t\n", port);
printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
}
} else {
//printf("udp port:%d test ok.\t\n", port);
printf("UDP port:\033[32m%d test OK\033[0m\t\n", port);
}
}
printf("\n");
return ;
}
static void taosNetTestClient(int flag) {
uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn);
if (serverIp == 0xFFFFFFFF) {
printf("Failed to resolve FQDN:%s", serverFqdn);
printf("Failed to resolve FQDN:%s", g_serverFqdn);
exit(-1);
}
checkPort(serverIp, startPort, endPort, pktLen);
if (0 == flag) {
checkPort(serverIp, g_startPort, g_endPort, g_pktLen);
} else {
rpcCheckPort(serverIp);
}
return;
}
static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) {
int port = startPort;
@ -375,49 +512,66 @@ static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen)
}
void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) {
if (pktLen > MAX_PKG_LEN) {
printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN);
exit(-1);
}
if (port && endPort) {
if (port > endPort) {
printf("endPort[%d] must not lesss port[%d]\n", endPort, port);
exit(-1);
}
}
if (host && host[0] != 0) {
if (strlen(host) >= TSDB_EP_LEN) {
printf("host invalid: %s\n", host);
exit(-1);
}
taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort);
void taosNetTest(CmdArguments *args) {
if (0 == args->pktLen) {
g_pktLen = 1000;
} else {
tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);
g_pktLen = args->pktLen;
}
if (args->port && args->endPort) {
if (args->port > args->endPort) {
printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port);
exit(-1);
}
}
if (args->host && args->host[0] != 0) {
if (strlen(args->host) >= TSDB_EP_LEN) {
printf("host invalid: %s\n", args->host);
exit(-1);
}
taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort);
} else {
tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN);
g_startPort = tsServerPort;
}
if (port) {
g_startPort = port;
if (args->port) {
g_startPort = args->port;
}
if (endPort) {
g_endPort = endPort;
if (args->endPort) {
g_endPort = args->endPort;
}
if (port > endPort) {
if (g_startPort > g_endPort) {
printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort);
exit(-1);
}
if (0 == strcmp("client", netTestRole)) {
printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen);
taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen);
} else if (0 == strcmp("server", netTestRole)) {
taosNetTestServer(g_startPort, g_endPort, pktLen);
if (args->is_use_passwd) {
if (args->password == NULL) args->password = getpass("Enter password: ");
} else {
args->password = TSDB_DEFAULT_PASS;
}
tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN);
if (args->user == NULL) {
args->user = TSDB_DEFAULT_USER;
}
tstrncpy(g_user, args->user, TSDB_USER_LEN);
if (0 == strcmp("client", args->netTestRole)) {
printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
taosNetTestClient(0);
} else if (0 == strcmp("clients", args->netTestRole)) {
printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen);
taosNetTestClient(1);
} else if (0 == strcmp("server", args->netTestRole)) {
taosNetTestServer(g_startPort, g_endPort, g_pktLen);
}
}

10
src/vnode/src/vnodeMain.c
View File

@ -292,11 +292,15 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
if (pVnode->tsdb == NULL) {
vnodeCleanUp(pVnode);
return terrno;
} else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) {
} else if (terrno != TSDB_CODE_SUCCESS) {
vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno));
vnodeCleanUp(pVnode);
return terrno;
if (pVnode->syncCfg.replica <= 1) {
vnodeCleanUp(pVnode);
return terrno;
} else {
pVnode->version = 0;
}
}
sprintf(temp, "%s/wal", rootDir);

41
src/wal/inc/walInt.h Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_WAL_INT_H
#define TDENGINE_WAL_INT_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tlog.h"
extern int32_t wDebugFlag;
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
#define walPrefix "wal"
#define walSignature (uint32_t)(0xFAFBFDFE)
#ifdef __cplusplus
}
#endif
#endif

27
src/wal/inc/walMgmt.h Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_WAL_MGMT_H
#define TDENGINE_WAL_MGMT_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif

21
src/wal/src/walMgmt.c Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "twal.h"
#include "walInt.h"
#include "walMgmt.h"

16
src/wal/src/walMain.c → src/wal/src/walWrite.c
View File

@ -14,11 +14,10 @@
*/
#define _DEFAULT_SOURCE
#define TAOS_RANDOM_FILE_FAIL_TEST
#include "os.h"
#include "tlog.h"
#include "twal.h"
#include "walInt.h"
#include "walMgmt.h"
#include "tchecksum.h"
#include "tutil.h"
#include "ttimer.h"
@ -26,14 +25,6 @@
#include "twal.h"
#include "tqueue.h"
#define walPrefix "wal"
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
typedef struct {
uint64_t version;
@ -54,7 +45,6 @@ typedef struct {
static void *walTmrCtrl = NULL;
static int tsWalNum = 0;
static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT;
static uint32_t walSignature = 0xFAFBFDFE;
static int walHandleExistingFiles(const char *path);
static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp);
static int walRemoveWalFiles(const char *path);

32
tests/pytest/handle_crash_gen_val_log.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/bash
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log
for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'`
do
if [ -n "$memError" ]; then
if [ "$memError" -gt 12 ]; then
echo -e "${RED} ## Memory errors number valgrind reports is $memError.\
More than our threshold! ## ${NC}"
fi
fi
done
grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log
for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'`
do
if [ -n "$defiMemError" ]; then
if [ "$defiMemError" -gt 3 ]; then
echo -e "${RED} ## Memory errors number valgrind reports \
Definitely lost is $defiMemError. More than our threshold! ## ${NC}"
exit 8
fi
fi
done

66
tests/script/general/db/nosuchfile.sim Normal file
View File

@ -0,0 +1,66 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c wallevel -v 2
print ========== step1
system sh/exec.sh -n dnode1 -s start
sql connect
sleep 3000
print ========== step3
sql create database d1
sql create table d1.t1 (t timestamp, i int)
sql insert into d1.t1 values(now+1s, 35)
sql insert into d1.t1 values(now+2s, 34)
sql insert into d1.t1 values(now+3s, 33)
sql insert into d1.t1 values(now+4s, 32)
sql insert into d1.t1 values(now+5s, 31)
print ========== step4
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
sleep 3000
print ========== step5
sql select * from d1.t1 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 31 then
return -1
endi
if $data11 != 32 then
return -1
endi
if $data21 != 33 then
return -1
endi
if $data31 != 34 then
return -1
endi
if $data41 != 35 then
return -1
endi
print ========== step6
system_content rm -rf ../../../sim/dnode1/data/vnode/vnode2/tsdb/data
print ========== step7
sql select * from d1.t1 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != null then
return -1
endi
if $data11 != null then
return -1
endi
if $data21 != null then
return -1
endi
if $data31 != null then
return -1
endi
if $data41 != null then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/script/jenkins/basic.txt
View File

@ -64,6 +64,7 @@ cd ../../../debug; make
./test.sh -f general/db/repeat.sim
./test.sh -f general/db/tables.sim
./test.sh -f general/db/vnodes.sim
./test.sh -f general/db/nosuchfile.sim
./test.sh -f general/field/2.sim
./test.sh -f general/field/3.sim
@ -277,6 +278,7 @@ cd ../../../debug; make
./test.sh -f unique/dnode/balance2.sim
./test.sh -f unique/dnode/balance3.sim
./test.sh -f unique/dnode/balancex.sim
./test.sh -f unique/dnode/data1.sim
./test.sh -f unique/dnode/offline1.sim
./test.sh -f unique/dnode/offline2.sim
./test.sh -f unique/dnode/reason.sim

137
tests/script/unique/dnode/data1.sim Normal file
View File

@ -0,0 +1,137 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c wallevel -v 2
system sh/cfg.sh -n dnode2 -c wallevel -v 2
system sh/cfg.sh -n dnode3 -c wallevel -v 2
system sh/cfg.sh -n dnode4 -c wallevel -v 2
print ========== step1
system sh/exec.sh -n dnode1 -s start
sql connect
sleep 3000
print ========== step2
sql create dnode $hostname2
system sh/exec.sh -n dnode2 -s start
sql create dnode $hostname3
system sh/exec.sh -n dnode3 -s start
sql create dnode $hostname4
system sh/exec.sh -n dnode4 -s start
$x = 0
show2:
$x = $x + 1
sleep 3000
if $x == 10 then
return -1
endi
sql show dnodes
print dnode1 openVnodes $data2_1
print dnode2 openVnodes $data2_2
print dnode3 openVnodes $data2_3
print dnode4 openVnodes $data2_4
if $data2_1 != 0 then
goto show2
endi
if $data2_2 != 0 then
goto show2
endi
if $data2_3 != 0 then
goto show2
endi
if $data2_4 != 0 then
goto show2
endi
print ========== step3
sql create database d1 replica 3
sql create table d1.t1 (t timestamp, i int)
sql insert into d1.t1 values(now+1s, 35)
sql insert into d1.t1 values(now+2s, 34)
sql insert into d1.t1 values(now+3s, 33)
sql insert into d1.t1 values(now+4s, 32)
sql insert into d1.t1 values(now+5s, 31)
$x = 0
show3:
$x = $x + 1
sleep 3000
if $x == 10 then
return -1
endi
sql show dnodes
print dnode1 openVnodes $data2_1
print dnode2 openVnodes $data2_2
print dnode3 openVnodes $data2_3
print dnode4 openVnodes $data2_4
if $data2_1 != 0 then
goto show3
endi
if $data2_2 != 1 then
goto show3
endi
if $data2_3 != 1 then
goto show3
endi
if $data2_4 != 1 then
goto show3
endi
print ========== step4
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
print ========== step5
system_content rm -rf ../../../sim/dnode4/data/vnode/vnode2/tsdb/data
print ========== step6
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
sleep 10000
print ========== step7
sql select * from d1.t1 order by t desc
print $data01 $data11 $data21 $data31 $data41
if $data01 != 31 then
return -1
endi
if $data11 != 32 then
return -1
endi
if $data21 != 33 then
return -1
endi
if $data31 != 34 then
return -1
endi
if $data41 != 35 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
system sh/exec.sh -n dnode4 -s stop -x SIGINT
system sh/exec.sh -n dnode5 -s stop -x SIGINT
system sh/exec.sh -n dnode6 -s stop -x SIGINT
system sh/exec.sh -n dnode7 -s stop -x SIGINT
system sh/exec.sh -n dnode8 -s stop -x SIGINT