xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-2330]<test>merge concurrent inquiry into ci

This commit is contained in:
liuyq-617 2020-12-11 10:34:26 +08:00
parent 1a6a295ecf
commit b76b468ae2
3 changed files with 262 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

307
Jenkinsfile vendored
View File

@ -1,3 +1,47 @@
properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine'
}
// execute this before anything else, including requesting any time on an agent
if (currentBuild.rawBuild.getCauses().toString().contains('BranchIndexingCause')) {
print "INFO: Build skipped due to trigger being Branch Indexing"
currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful
return
}
def pre_test(){
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
sudo rmtaos
'''
}
sh '''
cd ${WKC}
rm -rf *
cd ${WK}
git reset --hard
git checkout develop
git pull
cd ${WKC}
rm -rf *
mv ${WORKSPACE}/* .
cd ${WK}
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests
'''
return 1
}
pipeline { pipeline {
agent none agent none
environment{ environment{
@ -8,85 +52,31 @@ pipeline {
stages { stages {
stage('Parallel test stage') { stage('Parallel test stage') {
parallel { parallel {
stage('pytest') { stage('python p1') {
agent{label '184'} agent{label 'p1'}
steps { steps {
pre_test()
sh ''' sh '''
date
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests cd ${WKC}/tests
#./test-all.sh smoke ./test-all.sh p1
./test-all.sh pytest
date''' date'''
} }
} }
stage('test_b1') { stage('test_b1') {
agent{label 'master'} agent{label 'b1'}
steps { steps {
pre_test()
sh ''' sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh b1 ./test-all.sh b1
date''' date'''
} }
} }
stage('test_crash_gen') { stage('test_crash_gen') {
agent{label "185"} agent{label "b2"}
steps { steps {
sh ''' pre_test()
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
@ -109,193 +99,42 @@ pipeline {
} }
stage('test_valgrind') { stage('test_valgrind') {
agent{label "186"} agent{label "b3"}
steps { steps {
pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
'''
}
sh ''' sh '''
cd ${WKC}
git reset --hard
git checkout develop
git pull
git submodule update
cd ${WK}
git reset --hard
git checkout develop
git pull
export TZ=Asia/Harbin
date
rm -rf ${WK}/debug
mkdir debug
cd debug
cmake .. > /dev/null
make > /dev/null
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
date date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b3 ./test-all.sh b3
date''' date'''
} }
} }
stage('connector'){ stage('python p2'){
agent{label "release"} agent{label "p2"}
steps{ steps{
sh''' pre_test()
cd ${WORKSPACE} sh '''
git checkout develop date
cd ${WKC}/tests
./test-all.sh p2
date
''' '''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
} }
} }
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
} }
} }
} }
post {
success { }
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
}

221
tests/Jenkinsfile vendored
View File

@ -1,18 +1,3 @@
properties([pipelineTriggers([githubPush()])])
node {
git url: 'https://github.com/taosdata/TDengine'
}
// execute this before anything else, including requesting any time on an agent
if (currentBuild.rawBuild.getCauses().toString().contains('BranchIndexingCause')) {
print "INFO: Build skipped due to trigger being Branch Indexing"
currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful
return
}
def pre_test(){ def pre_test(){
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
@ -21,15 +6,14 @@ def pre_test(){
} }
sh ''' sh '''
cd ${WKC} cd ${WKC}
rm -rf * git reset --hard
git checkout ${BRANCH}
git pull
git submodule update
cd ${WK} cd ${WK}
git reset --hard git reset --hard
git checkout develop git checkout ${BRANCH}
git pull git pull
cd ${WKC}
rm -rf *
mv ${WORKSPACE}/* .
cd ${WK}
export TZ=Asia/Harbin export TZ=Asia/Harbin
date date
rm -rf ${WK}/debug rm -rf ${WK}/debug
@ -38,13 +22,13 @@ def pre_test(){
cmake .. > /dev/null cmake .. > /dev/null
make > /dev/null make > /dev/null
make install > /dev/null make install > /dev/null
cd ${WKC}/tests
''' '''
return 1 return 1
} }
pipeline { pipeline {
agent none agent none
environment{ environment{
BRANCH = 'develop'
WK = '/var/lib/jenkins/workspace/TDinternal' WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDinternal/community' WKC= '/var/lib/jenkins/workspace/TDinternal/community'
} }
@ -52,13 +36,13 @@ pipeline {
stages { stages {
stage('Parallel test stage') { stage('Parallel test stage') {
parallel { parallel {
stage('python p1') { stage('pytest') {
agent{label 'p1'} agent{label '184'}
steps { steps {
pre_test() pre_test()
sh ''' sh '''
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh p1 ./test-all.sh pytest
date''' date'''
} }
} }
@ -66,6 +50,12 @@ pipeline {
agent{label 'master'} agent{label 'master'}
steps { steps {
pre_test() pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
python3 concurrent_inquiry.py -c 1
'''
}
sh ''' sh '''
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b1 ./test-all.sh b1
@ -74,9 +64,12 @@ pipeline {
} }
stage('test_crash_gen') { stage('test_crash_gen') {
agent{label "b2"} agent{label "185"}
steps { steps {
pre_test() pre_test()
sh '''
cd ${WKC}/tests/pytest
'''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
@ -90,7 +83,6 @@ pipeline {
''' '''
} }
sh ''' sh '''
date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b2 ./test-all.sh b2
date date
@ -99,42 +91,177 @@ pipeline {
} }
stage('test_valgrind') { stage('test_valgrind') {
agent{label "b3"} agent{label "186"}
steps { steps {
pre_test() pre_test()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
'''
}
sh ''' sh '''
cd ${WKC}/tests/pytest
./valgrind-test.sh 2>&1 > mem-error-out.log
./handle_val_log.sh
date date
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b3 ./test-all.sh b3
date''' date'''
} }
} }
stage('python p2'){ stage('connector'){
agent{label "p2"} agent{label "release"}
steps{ steps{
pre_test() sh'''
sh ''' cd ${WORKSPACE}
date git checkout develop
cd ${WKC}/tests
./test-all.sh p2
date
''' '''
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/gotest
bash batchtest.sh
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
python3 PythonChecker.py
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
mvn clean package assembly:single >/dev/null
java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
'''
}
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC#
dotnet run
'''
}
} }
} }
stage('arm64_build'){
agent{label 'arm64'}
steps{
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
stage('arm32_build'){
agent{label 'arm32'}
steps{
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh '''
cd ${WK}
git fetch
git checkout develop
git pull
cd ${WKC}
git fetch
git checkout develop
git pull
git submodule update
cd ${WKC}/packaging
./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0
'''
}
}
}
} }
} }
} }
post {
} success {
emailext (
subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>变更概要：${CHANGES}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "yqliu@taosdata.com,pxiao@taosdata.com",
from: "support@taosdata.com"
)
}
}
}

28
tests/pytest/concurrent_inquiry.py
View File

@ -43,7 +43,7 @@ class ConcurrentInquiry:
self.subtb_stru_list=[] self.subtb_stru_list=[]
self.stb_tag_list=[] self.stb_tag_list=[]
self.subtb_tag_list=[] self.subtb_tag_list=[]
self.probabilities = [0.95,0.05] self.probabilities = [0.05,0.95]
self.ifjoin = [0,1] self.ifjoin = [0,1]
def SetThreadsNum(self,num): def SetThreadsNum(self,num):
self.numOfTherads=num self.numOfTherads=num
@ -117,15 +117,15 @@ class ConcurrentInquiry:
return 'where '+random.choice([' and ',' or ']).join(l) return 'where '+random.choice([' and ',' or ']).join(l)
def con_interval(self,tlist,col_list,tag_list): def con_interval(self,tlist,col_list,tag_list):
interval = 'interval' + str(random.randint(0,100)) + random.choice(['a','s','d','w','n','y']) interval = 'interval(' + str(random.randint(0,100)) + random.choice(['a','s','d','w','n','y']) + ')'
return interval return interval
def con_limit(self,tlist,col_list,tag_list): def con_limit(self,tlist,col_list,tag_list):
rand1 = str(random.randint(0,1000)) rand1 = str(random.randint(0,1000))
rand2 = str(random.randint(0,1000)) rand2 = str(random.randint(0,1000))
return random.choice(['limit ' + rand1,'limit ' + rand1 + 'offset '+rand2, return random.choice(['limit ' + rand1,'limit ' + rand1 + ' offset '+rand2,
'slimit ' + rand1,'slimit ' + rand1 + 'offset ' + rand2,'limit '+rand1 + 'slimit '+ rand2, ' slimit ' + rand1,' slimit ' + rand1 + ' offset ' + rand2,'limit '+rand1 + ' slimit '+ rand2,
'limit '+ rand1 + 'offset' + rand2 + 'slimit '+ rand1 + 'soffset ' + rand2 ]) 'limit '+ rand1 + ' offset' + rand2 + ' slimit '+ rand1 + ' soffset ' + rand2 ])
def con_fill(self,tlist,col_list,tag_list): def con_fill(self,tlist,col_list,tag_list):
return random.choice(['fill(null)','fill(prev)','fill(none)','fill(LINEAR)']) return random.choice(['fill(null)','fill(prev)','fill(none)','fill(LINEAR)'])
@ -194,9 +194,10 @@ class ConcurrentInquiry:
tag_list = [] tag_list = []
col_intersection = [] col_intersection = []
tag_intersection = [] tag_intersection = []
subtable = None
if bool(random.getrandbits(1)): if bool(random.getrandbits(1)):
subtable = True
tbname = random.sample(self.subtb_list,2) tbname = random.sample(self.subtb_list,2)
for i in tbname: for i in tbname:
col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) col_list.append(self.subtb_stru_list[self.subtb_list.index(i)])
@ -227,14 +228,15 @@ class ConcurrentInquiry:
sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1]))) sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1])))
sql += ','.join(sel_col_tag) sql += ','.join(sel_col_tag)
sql = sql + 'from '+ ','.join(tbname) + ' ' #select col & func sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func
con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill] join_section = None
sel_con=random.sample(con_func,random.randint(0,len(con_func))) if subtable:
sel_con_list=[] join_section = ''.join(random.choices(col_intersection))
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + join_section + '=t2.' + join_section
else:
join_section = ''.join(random.choices(col_intersection+tag_intersection))
sql += 'where t1._c0 = t2._c0 and ' + 't1.' + join_section + '=t2.' + join_section
# for i in sel_con:
# sel_con_list.append(i(tlist,col_list,tag_list)) #获取对应的条件函数
sql+=' '.join(sel_con_list) # condition
print(sql) print(sql)
return sql return sql