Merge branch 'develop' into hotfix/mergemaster

This commit is contained in:
Shengliang Guan 2020-08-19 22:31:55 +08:00 committed by GitHub
commit 05159cc9c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 290 additions and 62 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

View File

@ -15,6 +15,9 @@ TDengine的模块之一是时序数据库。但除此之外为减少研发的
采用TDengine可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是因充分利用了物联网时序数据的特点它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
<center> <img src="../assets/EcoSystem.png"> </center>
<center>图 1. TDengine技术生态图</center>
## TDengine 总体适用场景
作为一个IOT大数据平台TDengine的典型适用场景是在IOT范畴而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统比如CRMERP等不在本文讨论范围内。

View File

@ -12,7 +12,7 @@ TDengine 采用 SQL 作为查询语言。应用程序可以通过 C/C++, Java, G
- 标签和数值的多种过滤条件:\>, \<, =, \<>, like 等
- 聚合结果的分组Group by、排序Order by、约束输出Limit/Offset
- 数值列及聚合结果的四则运算
- 时间戳对齐的连接查询Join Query操作
- 时间戳对齐的连接查询Join Query: 隐式连接)操作
- 多种聚合/计算函数: count, max, min, avg, sum, twa, stddev, leastsquares, top, bottom, first, last, percentile, apercentile, last_row, spread, diff等
例如在TAOS Shell中从表d1001中查询出vlotage > 215的记录按时间降序排列仅仅输出2条。

1
snap/gui/t-dengine.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 10 KiB

17
snap/hooks/install Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
if [ ! -d /var/lib/taos ]; then
mkdir -p /var/lib/taos
fi
if [ ! -d /var/log/taos ]; then
mkdir -p -m777 /var/log/taos
fi
if [ ! -d /etc/taos ]; then
mkdir -p /etc/taos
fi
if [ ! -f /etc/taos/taos.cfg ]; then
cp $SNAP/etc/taos/taos.cfg /etc/taos/taos.cfg
fi

25
snap/local/launcher.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/sh
# Wrapper to check for custom config in $SNAP_USER_COMMON or $SNAP_COMMON and
# use it otherwise fall back to the included basic config which will at least
# allow mosquitto to run and do something.
# This script will also copy the full example config in to SNAP_USER_COMMON or
# SNAP_COMMON so that people can refer to it.
#
# The decision about whether to use SNAP_USER_COMMON or SNAP_COMMON is taken
# based on the user that runs the command. If the user is root, it is assumed
# that mosquitto is being run as a system daemon, and SNAP_COMMON will be used.
# If a non-root user runs the command, then SNAP_USER_COMMON will be used.
case "$SNAP_USER_COMMON" in
*/root/snap/tdengine/common*) COMMON=$SNAP_COMMON ;;
*) COMMON=$SNAP_USER_COMMON ;;
esac
if [ -d /etc/taos ]; then
CONFIG_FILE="/etc/taos"
else
CONFIG_FILE="$SNAP/etc/taos"
fi
# Launch the snap
$SNAP/usr/bin/taosd -c $CONFIG_FILE $@

25
snap/local/taoswrapper.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/sh
# Wrapper to check for custom config in $SNAP_USER_COMMON or $SNAP_COMMON and
# use it otherwise fall back to the included basic config which will at least
# allow mosquitto to run and do something.
# This script will also copy the full example config in to SNAP_USER_COMMON or
# SNAP_COMMON so that people can refer to it.
#
# The decision about whether to use SNAP_USER_COMMON or SNAP_COMMON is taken
# based on the user that runs the command. If the user is root, it is assumed
# that mosquitto is being run as a system daemon, and SNAP_COMMON will be used.
# If a non-root user runs the command, then SNAP_USER_COMMON will be used.
case "$SNAP_USER_COMMON" in
*/root/snap/tdengine/common*) COMMON=$SNAP_COMMON ;;
*) COMMON=$SNAP_USER_COMMON ;;
esac
if [ -d /etc/taos ]; then
CONFIG_FILE="/etc/taos"
else
CONFIG_FILE="$SNAP/etc/taos"
fi
# Launch the snap
$SNAP/usr/bin/taos -c $CONFIG_FILE $@

113
snap/snapcraft.yaml Normal file
View File

@ -0,0 +1,113 @@
name: tdengine
base: core18 # the base snap is the execution environment for this snap
version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2'
icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT.
description: |
TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations.
grade: stable
confinement: classic
apps:
tdengine:
command: launcher.sh
daemon: simple
restart-condition: always
plugs:
- network
- network-bind
- system-observe
- systemfiles
taos:
command: taoswrapper.sh
plugs:
- network
- systemfiles
taosdemo:
command: usr/bin/taosdemo
plugs:
- network
plugs:
systemfiles:
interface: system-files
read:
- /etc/taos
- /var/lib/taos
- /tmp
write:
- /var/log/taos
- /var/lib/taos
- /tmp
parts:
script:
plugin: dump
source: snap/local/
prime:
- launcher.sh
- taoswrapper.sh
tdengine:
source: .
source-type: local
plugin: cmake
build-packages:
- gcc
- g++
- make
- cmake
override-build: |
snapcraftctl build
if [ ! -d $SNAPCRAFT_STAGE/usr ]; then
mkdir $SNAPCRAFT_STAGE/usr
fi
if [ ! -d $SNAPCRAFT_STAGE/etc/taos ]; then
mkdir -p $SNAPCRAFT_STAGE/etc/taos
fi
cp $SNAPCRAFT_PART_BUILD/build/* -rf $SNAPCRAFT_STAGE/usr/
cp $SNAPCRAFT_PART_SRC/packaging/cfg/taos.cfg -rf $SNAPCRAFT_STAGE/etc/taos/
if [ ! -d $SNAPCRAFT_STAGE/var/lib/taos ]; then
mkdir -p $SNAPCRAFT_STAGE/var/lib/taos
fi
if [ ! -d $SNAPCRAFT_STAGE/var/log/taos ]; then
mkdir -p $SNAPCRAFT_STAGE/var/log/taos
fi
prime:
- etc/taos/taos.cfg
- usr/bin/taosd
- usr/bin/taos
- usr/bin/taosdemo
- usr/lib/libtaos.so.2.0.0.6
- usr/lib/libtaos.so.1
- usr/lib/libtaos.so
override-prime: |
snapcraftctl prime
if [ ! -d $SNAPCRAFT_STAGE/var/lib/taos ]; then
cp -rf $SNAPCRAFT_STAGE/var/lib/taos $SNAPCRAFT_PRIME
fi
if [ ! -d $SNAPCRAFT_STAGE/var/log/taos ]; then
cp -rf $SNAPCRAFT_STAGE/var/log/taos $SNAPCRAFT_PRIME
fi
layout:
/var/lib/taos:
bind: $SNAP_DATA/var/lib/taos
/var/log/taos:
bind: $SNAP_DATA/var/log/taos
/etc/taos/taos.cfg:
bind-file: $SNAP_DATA/etc/taos/taos.cfg
hooks:
install:
plugs: [systemfiles]

View File

@ -904,6 +904,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes);
sql += index;
if (TK_ILLEGAL == sToken.type) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return TSDB_CODE_TSC_INVALID_SQL;
}
if (sToken.n == 0 || sToken.type == TK_RP) {
break;
}

View File

@ -1158,8 +1158,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList);
if (ret != TSDB_CODE_SUCCESS) {
tExprTreeDestroy(&pNode, NULL);
taosTFree(arithmeticExprStr);
taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -1168,6 +1169,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
SColIndex* pIndex = taosArrayGet(colList, k);
if (pIndex->flag == 1) {
taosTFree(arithmeticExprStr);
taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
@ -1649,10 +1652,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) ||
index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
// 2. check if sql function can be applied on this column data type
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
@ -1861,7 +1866,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);

View File

@ -100,7 +100,7 @@ static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet
pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0;
pEpSet->numOfEps = pVgroupInfo->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
strncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN);
tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i]));
pEpSet->port[i] = pVgroupInfo->epAddr[i].port;
}
taosCorEndRead(&pVgroupInfo->version);
@ -117,7 +117,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
pVgroupInfo->inUse = pEpSet->inUse;
pVgroupInfo->numOfEps = pEpSet->numOfEps;
for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) {
strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN);
tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], sizeof(pEpSet->fqdn[i]));
pVgroupInfo->epAddr[i].port = pEpSet->port[i];
}
tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse);
@ -158,6 +158,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pRsp->killConnection) {
tscKillConnection(pObj);
return;
} else {
if (pRsp->queryId) tscKillQuery(pObj, htonl(pRsp->queryId));
if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId));

View File

@ -684,6 +684,8 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
freeJoinSubqueryObj(pParentSql);
pParentSql->res.code = code;
tscQueueAsyncRes(pParentSql);
taosArrayDestroy(s1);
taosArrayDestroy(s2);
return;
}
@ -1296,7 +1298,9 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i);
pState->numOfRemain = i;
pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
if (0 == i) {
taosTFree(pState);
}
return pSql->res.code;
}
@ -1304,7 +1308,9 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) {
if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query
tscDestroyJoinSupporter(pSupporter);
pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
if (0 == i) {
taosTFree(pState);
}
break;
}
}
@ -2091,17 +2097,17 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) {
// return;
// }
tscFetchDatablockFromSubquery(pSql);
if (pRes->code != TSDB_CODE_SUCCESS) {
return;
}
// tscFetchDatablockFromSubquery(pSql);
// if (pRes->code != TSDB_CODE_SUCCESS) {
// return;
// }
}
if (pSql->res.code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
} else {
tscQueueAsyncRes(pSql);
}
// if (pSql->res.code == TSDB_CODE_SUCCESS) {
// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
// } else {
// tscQueueAsyncRes(pSql);
// }
}
static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {

View File

@ -5242,7 +5242,6 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE
j += 1;
}
}
assert(0);
}
@ -5933,6 +5932,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
if (p1 == NULL) {
goto _cleanup;
}
taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1);
for(int32_t j = 0; j < s; ++j) {
STableKeyInfo* info = taosArrayGet(pa, j);
@ -5956,8 +5956,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES);
index += 1;
}
taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1);
}
pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));

View File

@ -1,7 +1,7 @@
#!/bin/bash
DATA_DIR=/mnt/root/testdata
NUM_LOOP=1
NUM_LOOP=5
NUM_OF_FILES=100
rowsPerRequest=(1 100 500 1000 2000)
@ -13,6 +13,42 @@ function printTo {
}
function runTest {
declare -A avgRPR
for r in ${!rowsPerRequest[@]}; do
for c in `seq 1 $clients`; do
avgRPR[$r,$c]=0
done
done
for r in ${!rowsPerRequest[@]}; do
for c in `seq 1 $clients`; do
totalRPR=0
OUTPUT_FILE=tdengineTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out
for i in `seq 1 $NUM_LOOP`; do
restartTaosd
$TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles $NUM_OF_FILES \
-w -clients $c \
-rowsPerRequest ${rowsPerRequest[$r]}"
$TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles $NUM_OF_FILES \
-w -clients $c \
-rowsPerRequest ${rowsPerRequest[$r]} \
| tee $OUTPUT_FILE
RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'`
totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
printTo "rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR"
done
avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
printTo "r:${rowsPerRequest[$r]} c:$c avgRPR:${avgRPR[$r,$c]}"
done
done
printf "R/R, "
for c in `seq 1 $clients`; do
if [ "$c" == "1" ]; then
@ -23,29 +59,10 @@ function runTest {
done
printf "\n"
for r in ${rowsPerRequest[@]}; do
printf "$r, "
for r in ${!rowsPerRequest[@]}; do
printf "${rowsPerRequest[$r]}, "
for c in `seq 1 $clients`; do
totalRPR=0
for i in `seq 1 $NUM_LOOP`; do
restartTaosd
$TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1
printTo "loop i:$i, $TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles $NUM_OF_FILES \
-w -clients $c \
-rowsPerRequest $r"
RPR=`$TDTEST_DIR/tdengineTest \
-dataDir $DATA_DIR \
-numOfFiles $NUM_OF_FILES \
-w -clients $c \
-rowsPerRequest $r \
| grep speed | awk '{print $(NF-1)}'`
totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
printTo "rows:$r, clients:$c, i:$i RPR:$RPR"
done
avgRPR=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
printf "$avgRPR, "
printf "${avgRPR[$r,$c]}, "
done
printf "\n"
done
@ -80,10 +97,6 @@ while : ; do
verbose=true
shift ;;
-n)
NUM_LOOP=$2
shift 2;;
master)
master=true
develop=false
@ -98,18 +111,22 @@ while : ; do
clients=$2
shift 2;;
-n)
NUM_LOOP=$2
shift 2;;
*)
break ;;
esac
done
if $master ; then
printTo "Test master branch.."
echo "Test master branch.."
cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine.master
else
printTo "Test develop branch.."
cp /mnt/root/cfg/perftest/taos.cfg /etc/taos/taos.cfg
echo "Test develop branch.."
cp /mnt/root/cfg/develop/taos.cfg /etc/taos/taos.cfg
WORK_DIR=/mnt/root/TDengine
fi
@ -118,4 +135,4 @@ TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine
runTest
printTo "Test done!"
echo "Test done!"

View File

@ -25,11 +25,23 @@ class TDTestCase:
def run(self):
tdSql.prepare()
ret = tdSql.query('select server_version()')
tdSql.checkData(0, 0, "2.0.0.6")
sql = "select server_version()"
ret = tdSql.query(sql)
version = tdSql.getData(0, 0)[0:3]
expectedVersion = "2.0"
if(version == expectedVersion):
tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (sql, 0, 0, version, expectedVersion))
else:
tdLog.exit("sql:%s, row:%d col:%d data:%s != expect:%s" % (sql, 0, 0, version, expectedVersion))
ret = tdSql.query('select client_version()')
tdSql.checkData(0, 0, "2.0.0.6")
sql = "select client_version()"
ret = tdSql.query(sql)
version = tdSql.getData(0, 0)[0:3]
expectedVersion = "2.0"
if(version == expectedVersion):
tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (sql, 0, 0, version, expectedVersion))
else:
tdLog.exit("sql:%s, row:%d col:%d data:%s != expect:%s" % (sql, 0, 0, version, expectedVersion))
def stop(self):

View File

@ -49,11 +49,7 @@ class TDTestCase:
tdSql.query("select col1 + col2 from test1")
tdSql.checkRows(10)
tdSql.checkData(0, 0, 2.0)
tdSql.query("select col1 + col2 * col3 from test1")
tdSql.checkRows(10)
tdSql.checkData(1, 0, 6.0)
tdSql.checkData(0, 0, 2.0)
tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1")
tdSql.checkRows(10)

View File

@ -36,7 +36,8 @@ class TDTestCase:
"insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:02.000', 3, 2.1), ('2020-04-18 15:00:03.000', 4, 2.2)")
# inner join --- bug
tdSql.error("select * from tb1 a, tb2 b where a.ts = b.ts")
tdSql.query("select * from tb1 a, tb2 b where a.ts = b.ts")
tdSql.checkRows(0)
# join 3 tables -- bug exists
tdSql.error("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id")