xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into xiaoping/add_test_case2

This commit is contained in:
Ping Xiao 2020-11-16 17:33:25 +08:00
parent 7ca81e4271 4cad46a8ea
commit 423027acee
79 changed files with 5012 additions and 2839 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/version.inc
View File

@ -4,7 +4,7 @@ PROJECT(TDengine)
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "2.0.6.0")
SET(TD_VER_NUMBER "2.0.7.0")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2
packaging/deb/makedeb.sh
View File

@ -48,7 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin
#cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver

2
packaging/rpm/tdengine.spec
View File

@ -58,7 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri
cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin
#cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include

21
packaging/tools/get_client.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
#
log_dir=$1
result_file=$2
if [ ! -n "$1" ];then
echo "Pleas input the director of taosdlog."
echo "usage: ./get_client.sh <taosdlog directory> <result file>"
exit 1
else
log_dir=$1
fi
if [ ! -n "$2" ];then
result_file=clientInfo.txt
else
result_file=$2
fi
grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file}

3
packaging/tools/makeclient.sh
View File

@ -45,8 +45,7 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
#bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
else

4
packaging/tools/makeclient_power.sh
View File

@ -76,8 +76,10 @@ if [ "$osType" != "Darwin" ]; then
else
cp ${build_dir}/bin/taos ${install_dir}/bin/power
cp ${script_dir}/remove_power.sh ${install_dir}/bin
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${script_dir}/set_core.sh ${install_dir}/bin
cp ${script_dir}/get_client.sh ${install_dir}/bin
fi
else
cp ${bin_files} ${install_dir}/bin

3
packaging/tools/makepkg.sh
View File

@ -36,8 +36,7 @@ if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
else
#bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"

2
packaging/tools/makepkg_power.sh
View File

@ -77,8 +77,10 @@ else
cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd
cp ${script_dir}/remove_power.sh ${install_dir}/bin
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${build_dir}/bin/tarbitrator ${install_dir}/bin
cp ${script_dir}/set_core.sh ${install_dir}/bin
cp ${script_dir}/get_client.sh ${install_dir}/bin
fi
chmod a+x ${install_dir}/bin/* || :

2
snap/snapcraft.yaml
View File

@ -1,6 +1,6 @@
name: tdengine
base: core18
version: '2.0.6.0'
version: '2.0.7.0'
icon: snap/gui/t-dengine.svg
summary: an open-source big data platform designed and optimized for IoT.
description: |

162
src/client/src/tscFunctionImpl.c
View File

@ -28,6 +28,7 @@
#include "tscompression.h"
#include "tsqlfunction.h"
#include "tutil.h"
#include "ttype.h"
#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes))
#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes)
@ -2479,28 +2480,8 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
continue;
}
// TODO extract functions
double v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = GET_INT8_VAL(data);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = GET_INT16_VAL(data);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (double)(GET_INT64_VAL(data));
break;
case TSDB_DATA_TYPE_FLOAT:
v = GET_FLOAT_VAL(data);
break;
case TSDB_DATA_TYPE_DOUBLE:
v = GET_DOUBLE_VAL(data);
break;
default:
v = GET_INT32_VAL(data);
break;
}
GET_TYPED_DATA(v, double, pCtx->inputType, data);
if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
SET_DOUBLE_VAL(&pInfo->minval, v);
@ -2541,30 +2522,10 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo);
if (pInfo->stage == 0) {
// TODO extract functions
double v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (double)(GET_INT64_VAL(pData));
break;
case TSDB_DATA_TYPE_FLOAT:
v = GET_FLOAT_VAL(pData);
break;
case TSDB_DATA_TYPE_DOUBLE:
v = GET_DOUBLE_VAL(pData);
break;
default:
v = GET_INT32_VAL(pData);
break;
}
GET_TYPED_DATA(v, double, pCtx->inputType, pData);
if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
SET_DOUBLE_VAL(&pInfo->minval, v);
@ -2653,29 +2614,9 @@ static void apercentile_function(SQLFunctionCtx *pCtx) {
}
notNullElems += 1;
double v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = GET_INT8_VAL(data);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = GET_INT16_VAL(data);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (double)(GET_INT64_VAL(data));
break;
case TSDB_DATA_TYPE_FLOAT:
v = GET_FLOAT_VAL(data);
break;
case TSDB_DATA_TYPE_DOUBLE:
v = GET_DOUBLE_VAL(data);
break;
default:
v = GET_INT32_VAL(data);
break;
}
GET_TYPED_DATA(v, double, pCtx->inputType, data);
tHistogramAdd(&pInfo->pHisto, v);
}
@ -2700,26 +2641,7 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
double v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (double)(GET_INT64_VAL(pData));
break;
case TSDB_DATA_TYPE_FLOAT:
v = GET_FLOAT_VAL(pData);
break;
case TSDB_DATA_TYPE_DOUBLE:
v = GET_DOUBLE_VAL(pData);
break;
default:
v = GET_INT32_VAL(pData);
break;
}
GET_TYPED_DATA(v, double, pCtx->inputType, pData);
tHistogramAdd(&pInfo->pHisto, v);
@ -4142,22 +4064,7 @@ static void rate_function(SQLFunctionCtx *pCtx) {
notNullElems++;
int64_t v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = (int64_t)GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = (int64_t)GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_INT:
v = (int64_t)GET_INT32_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (int64_t)GET_INT64_VAL(pData);
break;
default:
assert(0);
}
GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
pRateInfo->firstValue = v;
@ -4207,22 +4114,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY *primaryKey = pCtx->ptsList;
int64_t v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = (int64_t)GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = (int64_t)GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_INT:
v = (int64_t)GET_INT32_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (int64_t)GET_INT64_VAL(pData);
break;
default:
assert(0);
}
GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
pRateInfo->firstValue = v;
@ -4349,22 +4241,7 @@ static void irate_function(SQLFunctionCtx *pCtx) {
notNullElems++;
int64_t v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = (int64_t)GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = (int64_t)GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_INT:
v = (int64_t)GET_INT32_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (int64_t)GET_INT64_VAL(pData);
break;
default:
assert(0);
}
GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
// TODO: calc once if only call this function once ????
if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->lastValue)) {
@ -4409,23 +4286,8 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
TSKEY *primaryKey = pCtx->ptsList;
int64_t v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = (int64_t)GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = (int64_t)GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_INT:
v = (int64_t)GET_INT32_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (int64_t)GET_INT64_VAL(pData);
break;
default:
assert(0);
}
GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
pRateInfo->firstKey = pRateInfo->lastKey;
pRateInfo->firstValue = pRateInfo->lastValue;

10
src/client/src/tscLocalMerge.c
View File

@ -373,7 +373,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
4096, (int32_t)numOfCols, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
tinfo.precision, pQueryInfo->fillType, pFillCol);
tinfo.precision, pQueryInfo->fillType, pFillCol, pSql);
}
}
@ -916,7 +916,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
}
while (1) {
int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
if (pQueryInfo->limit.offset < newRows) {
newRows -= pQueryInfo->limit.offset;
@ -945,7 +945,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
}
// all output in current group are completed
int32_t totalRemainRows = (int32_t)getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
int32_t totalRemainRows = (int32_t)getNumOfResWithFill(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
if (totalRemainRows <= 0) {
break;
}
@ -1294,7 +1294,7 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
// the first column must be the timestamp column
int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
int32_t rows = (int32_t) getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do fill gap
doFillResult(pSql, pLocalReducer, false);
}
@ -1323,7 +1323,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) {
int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey;
int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
int32_t rows = (int32_t)getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
if (rows > 0) {
doFillResult(pSql, pLocalReducer, true);
}

365
src/client/src/tscSQLParser.c
View File

@ -57,11 +57,11 @@ static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, in
static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo);
static char* getAccountId(SSqlObj* pSql);
static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name);
static bool has(SArray* pFieldList, int32_t startIdx, const char* name);
static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken);
static bool hasSpecifyDB(SStrToken* pTableName);
static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd);
static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd);
static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd);
static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd);
static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);
@ -70,7 +70,10 @@ static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t na
static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult);
static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
int8_t type, char* fieldName, SSqlExpr* pSqlExpr);
static int32_t changeFunctionID(int32_t optr, int16_t* functionId);
static int32_t convertFunctionId(int32_t optr, int16_t* functionId);
static uint8_t convertOptr(SStrToken *pToken);
static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery);
static bool validateIpAddress(const char* ip, size_t size);
@ -78,7 +81,7 @@ static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQuer
static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery);
static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);
static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd);
static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd);
static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
@ -124,6 +127,45 @@ static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo);
static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index);
static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid);
static uint8_t convertOptr(SStrToken *pToken) {
switch (pToken->type) {
case TK_LT:
return TSDB_RELATION_LESS;
case TK_LE:
return TSDB_RELATION_LESS_EQUAL;
case TK_GT:
return TSDB_RELATION_GREATER;
case TK_GE:
return TSDB_RELATION_GREATER_EQUAL;
case TK_NE:
return TSDB_RELATION_NOT_EQUAL;
case TK_AND:
return TSDB_RELATION_AND;
case TK_OR:
return TSDB_RELATION_OR;
case TK_EQ:
return TSDB_RELATION_EQUAL;
case TK_PLUS:
return TSDB_BINARY_OP_ADD;
case TK_MINUS:
return TSDB_BINARY_OP_SUBTRACT;
case TK_STAR:
return TSDB_BINARY_OP_MULTIPLY;
case TK_SLASH:
case TK_DIVIDE:
return TSDB_BINARY_OP_DIVIDE;
case TK_REM:
return TSDB_BINARY_OP_REMAINDER;
case TK_LIKE:
return TSDB_RELATION_LIKE;
case TK_ISNULL:
return TSDB_RELATION_ISNULL;
case TK_NOTNULL:
return TSDB_RELATION_NOTNULL;
default: { return 0; }
}
}
/*
* Used during parsing query sql. Since the query sql usually small in length, error position
* is not needed in the final error message.
@ -778,7 +820,7 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu
const char* msg1 = "sliding value no larger than the interval value";
const char* msg2 = "sliding value can not less than 1% of interval value";
const char* msg3 = "does not support sliding when interval is natural month/year";
const char* msg4 = "sliding not support yet in ordinary query";
// const char* msg4 = "sliding not support yet in ordinary query";
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
SSqlCmd* pCmd = &pSql->cmd;
@ -814,9 +856,9 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
// if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
// return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
// }
return TSDB_CODE_SUCCESS;
}
@ -864,7 +906,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
return TSDB_CODE_SUCCESS;
}
static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) {
assert(pFieldList != NULL);
const char* msg = "illegal number of columns";
@ -876,20 +918,22 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
const char* msg6 = "invalid column name";
// number of fields no less than 2
if (pFieldList->nField <= 1 || pFieldList->nField > TSDB_MAX_COLUMNS) {
size_t numOfCols = taosArrayGetSize(pFieldList);
if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
return false;
}
// first column must be timestamp
if (pFieldList->p[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
TAOS_FIELD* pField = taosArrayGet(pFieldList, 0);
if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return false;
}
int32_t nLen = 0;
for (int32_t i = 0; i < pFieldList->nField; ++i) {
TAOS_FIELD* pField = &pFieldList->p[i];
for (int32_t i = 0; i < numOfCols; ++i) {
pField = taosArrayGet(pFieldList, i);
if (pField->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
@ -913,7 +957,7 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
}
// field name must be unique
if (has(pFieldList, i + 1, pFieldList->p[i].name) == true) {
if (has(pFieldList, i + 1, pField->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
@ -930,7 +974,7 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
return true;
}
static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd) {
static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd) {
assert(pTagsList != NULL);
const char* msg1 = "invalid number of tag columns";
@ -942,18 +986,21 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
const char* msg7 = "invalid binary/nchar tag length";
// number of fields at least 1
if (pTagsList->nField < 1 || pTagsList->nField > TSDB_MAX_TAGS) {
size_t numOfTags = taosArrayGetSize(pTagsList);
if (numOfTags < 1 || numOfTags > TSDB_MAX_TAGS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return false;
}
int32_t nLen = 0;
for (int32_t i = 0; i < pTagsList->nField; ++i) {
if (pTagsList->p[i].bytes == 0) {
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (p->bytes == 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
}
nLen += pTagsList->p[i].bytes;
nLen += p->bytes;
}
// max tag row length must be less than TSDB_MAX_TAGS_LEN
@ -963,37 +1010,41 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
}
// field name must be unique
for (int32_t i = 0; i < pTagsList->nField; ++i) {
if (has(pFieldList, 0, pTagsList->p[i].name) == true) {
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (has(pFieldList, 0, p->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
}
/* timestamp in tag is not allowed */
for (int32_t i = 0; i < pTagsList->nField; ++i) {
if (pTagsList->p[i].type == TSDB_DATA_TYPE_TIMESTAMP) {
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagsList, i);
if (p->type == TSDB_DATA_TYPE_TIMESTAMP) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
return false;
}
if (pTagsList->p[i].type < TSDB_DATA_TYPE_BOOL || pTagsList->p[i].type > TSDB_DATA_TYPE_NCHAR) {
if (p->type < TSDB_DATA_TYPE_BOOL || p->type > TSDB_DATA_TYPE_NCHAR) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
return false;
}
if ((pTagsList->p[i].type == TSDB_DATA_TYPE_BINARY && pTagsList->p[i].bytes <= 0) ||
(pTagsList->p[i].type == TSDB_DATA_TYPE_NCHAR && pTagsList->p[i].bytes <= 0)) {
if ((p->type == TSDB_DATA_TYPE_BINARY && p->bytes <= 0) ||
(p->type == TSDB_DATA_TYPE_NCHAR && p->bytes <= 0)) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
}
if (validateColumnName(pTagsList->p[i].name) != TSDB_CODE_SUCCESS) {
if (validateColumnName(p->name) != TSDB_CODE_SUCCESS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
return false;
}
if (has(pTagsList, i + 1, pTagsList->p[i].name) == true) {
if (has(pTagsList, i + 1, p->name) == true) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
}
@ -1140,9 +1191,10 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
}
/* is contained in pFieldList or not */
static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) {
for (int32_t j = startIdx; j < pFieldList->nField; ++j) {
TAOS_FIELD* field = pFieldList->p + j;
static bool has(SArray* pFieldList, int32_t startIdx, const char* name) {
size_t numOfCols = taosArrayGetSize(pFieldList);
for (int32_t j = startIdx; j < numOfCols; ++j) {
TAOS_FIELD* field = taosArrayGet(pFieldList, j);
if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true;
}
@ -1726,7 +1778,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
int16_t functionID = 0;
if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -1862,7 +1914,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
int32_t intermediateResSize = 0;
int16_t functionID = 0;
if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -1933,7 +1985,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
bool requireAllFields = (pItem->pNode->pParam == NULL);
int16_t functionID = 0;
if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
@ -2122,7 +2174,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
* for dp = 100, it is max,
*/
int16_t functionId = 0;
if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) {
if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
@ -2139,7 +2191,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
}
int16_t functionId = 0;
if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) {
if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -2398,7 +2450,7 @@ int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo*
return doGetColumnIndexByName(pCmd, &tmpToken, pQueryInfo, pIndex);
}
int32_t changeFunctionID(int32_t optr, int16_t* functionId) {
int32_t convertFunctionId(int32_t optr, int16_t* functionId) {
switch (optr) {
case TK_COUNT:
*functionId = TSDB_FUNC_COUNT;
@ -2716,7 +2768,10 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo)
static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
int32_t startIdx = 0;
size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo);
assert(numOfExpr > 0);
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx);
int32_t functionID = pExpr->functionId;
@ -2755,7 +2810,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
return true;
}
int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) {
const char* msg1 = "too many columns in group by clause";
const char* msg2 = "invalid column name in group by clause";
const char* msg3 = "columns from one table allowed as group by columns";
@ -2775,8 +2830,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
}
pQueryInfo->groupbyExpr.numOfGroupCols = pList->nExpr;
if (pList->nExpr > TSDB_MAX_TAGS) {
pQueryInfo->groupbyExpr.numOfGroupCols = (int16_t)taosArrayGetSize(pList);
if (pQueryInfo->groupbyExpr.numOfGroupCols > TSDB_MAX_TAGS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@ -2789,9 +2844,12 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
SSchema s = tscGetTbnameColumnSchema();
int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
for (int32_t i = 0; i < pList->nExpr; ++i) {
tVariant* pVar = &pList->a[i].pVar;
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
tVariantListItem * pItem = taosArrayGet(pList, i);
tVariant* pVar = &pItem->pVar;
SStrToken token = {pVar->nLen, pVar->nType, pVar->pz};
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
@ -2852,7 +2910,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
taosArrayPush(pGroupExpr->columnInfo, &colIndex);
pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
if (i == 0 && pList->nExpr > 1) {
if (i == 0 && num > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
}
@ -4366,8 +4424,8 @@ int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
}
int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
tVariantList* pFillToken = pQuerySQL->fillType;
tVariantListItem* pItem = &pFillToken->a[0];
SArray* pFillToken = pQuerySQL->fillType;
tVariantListItem* pItem = taosArrayGet(pFillToken, 0);
const int32_t START_INTERPO_COL_IDX = 1;
@ -4407,12 +4465,13 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
} else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) {
pQueryInfo->fillType = TSDB_FILL_SET_VALUE;
if (pFillToken->nExpr == 1) {
size_t num = taosArrayGetSize(pFillToken);
if (num == 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
int32_t startPos = 1;
int32_t numOfFillVal = pFillToken->nExpr - 1;
int32_t numOfFillVal = (int32_t)(num - 1);
/* for point interpolation query, we do not have the timestamp column */
if (tscIsPointInterpQuery(pQueryInfo)) {
@ -4422,7 +4481,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
numOfFillVal = (int32_t)size;
}
} else {
numOfFillVal = (pFillToken->nExpr > (int32_t)size) ? (int32_t)size : pFillToken->nExpr;
numOfFillVal = (int16_t)((num > (int32_t)size) ? (int32_t)size : num);
}
int32_t j = 1;
@ -4435,14 +4494,15 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
continue;
}
int32_t ret = tVariantDump(&pFillToken->a[j].pVar, (char*)&pQueryInfo->fillVal[i], pFields->type, true);
tVariant* p = taosArrayGet(pFillToken, j);
int32_t ret = tVariantDump(p, (char*)&pQueryInfo->fillVal[i], pFields->type, true);
if (ret != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
}
if ((pFillToken->nExpr < size) || ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1];
if ((num < size) || ((num - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
tVariantListItem* lastItem = taosArrayGetLast(pFillToken);
for (int32_t i = numOfFillVal; i < size; ++i) {
TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
@ -4491,7 +4551,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return TSDB_CODE_SUCCESS;
}
tVariantList* pSortorder = pQuerySql->pSortOrder;
SArray* pSortorder = pQuerySql->pSortOrder;
/*
* for table query, there is only one or none order option is allowed, which is the
@ -4499,18 +4559,19 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
*
* for super table query, the order option must be less than 3.
*/
size_t size = taosArrayGetSize(pSortorder);
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
if (pSortorder->nExpr > 1) {
if (size > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
}
} else {
if (pSortorder->nExpr > 2) {
if (size > 2) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
// handle the first part of order by
tVariant* pVar = &pSortorder->a[0].pVar;
tVariant* pVar = taosArrayGet(pSortorder, 0);
// e.g., order by 1 asc, return directly with out further check.
if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) {
@ -4553,10 +4614,13 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
assert(!(orderByTags && orderByTS));
}
if (pSortorder->nExpr == 1) {
size_t s = taosArrayGetSize(pSortorder);
if (s == 1) {
if (orderByTags) {
pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder;
tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
pQueryInfo->groupbyExpr.orderType = p1->sortOrder;
} else if (isTopBottomQuery(pQueryInfo)) {
/* order of top/bottom query in interval is not valid */
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0);
@ -4567,11 +4631,14 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
pQueryInfo->order.order = p1->sortOrder;
pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
return TSDB_CODE_SUCCESS;
} else {
pQueryInfo->order.order = pSortorder->a[0].sortOrder;
tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
pQueryInfo->order.order = p1->sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
// orderby ts query on super table
@ -4581,16 +4648,18 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
}
}
if (pSortorder->nExpr == 2) {
if (s == 2) {
tVariantListItem *pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
if (orderByTags) {
pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder;
pQueryInfo->groupbyExpr.orderType = pItem->sortOrder;
} else {
pQueryInfo->order.order = pSortorder->a[0].sortOrder;
pQueryInfo->order.order = pItem->sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
}
tVariant* pVar2 = &pSortorder->a[1].pVar;
pItem = taosArrayGet(pQuerySql->pSortOrder, 1);
tVariant* pVar2 = &pItem->pVar;
SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz};
if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
@ -4599,7 +4668,8 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
} else {
pQueryInfo->order.order = pSortorder->a[1].sortOrder;
tVariantListItem* p1 = taosArrayGet(pSortorder, 1);
pQueryInfo->order.order = p1->sortOrder;
pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
}
}
@ -4623,12 +4693,14 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
pQueryInfo->order.order = pItem->sortOrder;
pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
return TSDB_CODE_SUCCESS;
}
pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
pQueryInfo->order.order = pItem->sortOrder;
}
return TSDB_CODE_SUCCESS;
@ -4696,27 +4768,28 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
tFieldList* pFieldList = pAlterSQL->pAddColumns;
if (pFieldList->nField > 1) {
SArray* pFieldList = pAlterSQL->pAddColumns;
if (taosArrayGetSize(pFieldList) > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
if (!validateOneTags(pCmd, &pFieldList->p[0])) {
TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
if (!validateOneTags(pCmd, p)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
if (tscGetNumOfTags(pTableMeta) == 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
// numOfTags == 1
if (pAlterSQL->varList->nExpr > 1) {
if (taosArrayGetSize(pAlterSQL->varList) > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
}
tVariantListItem* pItem = &pAlterSQL->varList->a[0];
tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0);
if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
@ -4741,13 +4814,13 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
tVariantList* pVarList = pAlterSQL->varList;
if (pVarList->nExpr > 2) {
SArray* pVarList = pAlterSQL->varList;
if (taosArrayGetSize(pVarList) > 2) {
return TSDB_CODE_TSC_INVALID_SQL;
}
tVariantListItem* pSrcItem = &pAlterSQL->varList->a[0];
tVariantListItem* pDstItem = &pAlterSQL->varList->a[1];
tVariantListItem* pSrcItem = taosArrayGet(pAlterSQL->varList, 0);
tVariantListItem* pDstItem = taosArrayGet(pAlterSQL->varList, 1);
if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
@ -4770,13 +4843,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19);
}
tVariantListItem* pItem = taosArrayGet(pVarList, 0);
char name[TSDB_COL_NAME_LEN] = {0};
strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen);
strncpy(name, pItem->pVar.pz, pItem->pVar.nLen);
TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
pItem = taosArrayGet(pVarList, 1);
memset(name, 0, tListLen(name));
strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen);
strncpy(name, pItem->pVar.pz, pItem->pVar.nLen);
f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
@ -4784,12 +4861,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
// the following is used to handle tags value for table created according to super table
pCmd->command = TSDB_SQL_UPDATE_TAGS_VAL;
tVariantList* pVarList = pAlterSQL->varList;
tVariant* pTagName = &pVarList->a[0].pVar;
SArray* pVarList = pAlterSQL->varList;
tVariantListItem* item = taosArrayGet(pVarList, 0);
int16_t numOfTags = tscGetNumOfTags(pTableMeta);
SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen};
SStrToken name = {.type = TK_STRING, .z = item->pVar.pz, .n = item->pVar.nLen};
if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -4798,8 +4875,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg12);
}
tVariantListItem* pItem = taosArrayGet(pVarList, 1);
SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex);
if (tVariantDump(&pVarList->a[1].pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13);
}
@ -4844,7 +4922,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
// copy the tag value to msg body
tVariantDump(&pVarList->a[1].pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true);
pItem = taosArrayGet(pVarList, 1);
tVariantDump(&pItem->pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true);
int32_t len = 0;
if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) {
@ -4859,27 +4938,29 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
pUpdateMsg->head.contLen = htonl(total);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
tFieldList* pFieldList = pAlterSQL->pAddColumns;
if (pFieldList->nField > 1) {
SArray* pFieldList = pAlterSQL->pAddColumns;
if (taosArrayGetSize(pFieldList) > 1) {
const char* msg = "only support add one column";
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
if (!validateOneColumn(pCmd, &pFieldList->p[0])) {
TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
if (!validateOneColumn(pCmd, p)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
} else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { //
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg15);
}
if (pAlterSQL->varList->nExpr > 1) {
size_t size = taosArrayGetSize(pAlterSQL->varList);
if (size > 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg16);
}
tVariantListItem* pItem = &pAlterSQL->varList->a[0];
tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0);
SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen};
@ -5250,21 +5331,28 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p
pMsg->daysToKeep1 = htonl(-1);
pMsg->daysToKeep2 = htonl(-1);
tVariantList* pKeep = pCreateDb->keep;
SArray* pKeep = pCreateDb->keep;
if (pKeep != NULL) {
switch (pKeep->nExpr) {
case 1:
pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
size_t s = taosArrayGetSize(pKeep);
tVariantListItem* p0 = taosArrayGet(pKeep, 0);
switch (s) {
case 1: {
pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
}
break;
case 2: {
pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
pMsg->daysToKeep1 = htonl((int32_t)pKeep->a[1].pVar.i64Key);
tVariantListItem* p1 = taosArrayGet(pKeep, 1);
pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key);
break;
}
case 3: {
pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
pMsg->daysToKeep1 = htonl((int32_t)pKeep->a[1].pVar.i64Key);
pMsg->daysToKeep2 = htonl((int32_t)pKeep->a[2].pVar.i64Key);
tVariantListItem* p1 = taosArrayGet(pKeep, 1);
tVariantListItem* p2 = taosArrayGet(pKeep, 2);
pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key);
pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64Key);
break;
}
default: { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); }
@ -5951,8 +6039,8 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p
SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo;
tFieldList* pFieldList = pCreateTable->colInfo.pColumns;
tFieldList* pTagList = pCreateTable->colInfo.pTagColumns;
SArray* pFieldList = pCreateTable->colInfo.pColumns;
SArray* pTagList = pCreateTable->colInfo.pTagColumns;
assert(pFieldList != NULL);
@ -5974,18 +6062,23 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p
}
int32_t col = 0;
for (; col < pFieldList->nField; ++col) {
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[col]);
size_t numOfFields = taosArrayGetSize(pFieldList);
for (; col < numOfFields; ++col) {
TAOS_FIELD* p = taosArrayGet(pFieldList, col);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
}
pCmd->numOfCols = (int16_t)pFieldList->nField;
pCmd->numOfCols = (int16_t)numOfFields;
if (pTagList != NULL) { // create super table[optional]
for (int32_t i = 0; i < pTagList->nField; ++i) {
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]);
size_t numOfTags = taosArrayGetSize(pTagList);
for (int32_t i = 0; i < numOfTags; ++i) {
TAOS_FIELD* p = taosArrayGet(pTagList, i);
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
}
pCmd->count = pTagList->nField;
pCmd->count =(int32_t) numOfTags;
}
return TSDB_CODE_SUCCESS;
@ -6026,14 +6119,15 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
// get meter meta from mnode
tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name));
tVariantList* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
SArray* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
code = tscGetTableMeta(pSql, pStableMeterMetaInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != pList->nExpr) {
size_t size = taosArrayGetSize(pList);
if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != size) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
@ -6047,18 +6141,19 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
int32_t ret = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pList->nExpr; ++i) {
for (int32_t i = 0; i < size; ++i) {
SSchema* pSchema = &pTagSchema[i];
tVariantListItem* pItem = taosArrayGet(pList, i);
char tagVal[TSDB_MAX_TAGS_LEN];
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
if (pList->a[i].pVar.nLen > pSchema->bytes) {
if (pItem->pVar.nLen > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true);
ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
// check again after the convert since it may be converted from binary to nchar.
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
@ -6125,13 +6220,13 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
if (pSrcMeterName == NULL || pSrcMeterName->nExpr == 0) {
SArray* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
if (pSrcMeterName == NULL || taosArrayGetSize(pSrcMeterName) == 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
tVariant* pVar = &pSrcMeterName->a[0].pVar;
SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING};
tVariantListItem* p1 = taosArrayGet(pSrcMeterName, 0);
SStrToken srcToken = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@ -6196,7 +6291,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
tVariantListItem* pItem = &pQuerySql->fillType->a[0];
tVariantListItem* pItem = taosArrayGet(pQuerySql->fillType, 0);
if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) ||
(strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) {
@ -6211,7 +6306,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
}
int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
assert(pQuerySql != NULL && (pQuerySql->from == NULL || pQuerySql->from->nExpr > 0));
assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0));
const char* msg0 = "invalid table name";
const char* msg2 = "point interpolation query needs timestamp";
@ -6253,19 +6348,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySql);
}
if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM * 2) {
size_t fromSize = taosArrayGetSize(pQuerySql->from);
if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
pQueryInfo->command = TSDB_SQL_SELECT;
if (pQuerySql->from->nExpr > 4) {
if (fromSize > 4) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
}
// set all query tables, which are maybe more than one.
for (int32_t i = 0; i < pQuerySql->from->nExpr; ) {
tVariant* pTableItem = &pQuerySql->from->a[i].pVar;
for (int32_t i = 0; i < fromSize; ) {
tVariantListItem* item = taosArrayGet(pQuerySql->from, i);
tVariant* pTableItem = &item->pVar;
if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
@ -6290,21 +6387,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return code;
}
tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar;
if (pTableItem1->nType != TSDB_DATA_TYPE_BINARY) {
tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i);
if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
}
SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING};
SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
}
// has no table alias name
if (memcmp(pTableItem->pz, pTableItem1->pz, pTableItem1->nLen) == 0) {
if (memcmp(pTableItem->pz, p1->pVar.pz, p1->pVar.nLen) == 0) {
extractTableName(pTableMetaInfo1->name, pTableMetaInfo1->aliasName);
} else {
tstrncpy(pTableMetaInfo1->aliasName, pTableItem1->pz, sizeof(pTableMetaInfo1->aliasName));
tstrncpy(pTableMetaInfo1->aliasName, p1->pVar.pz, sizeof(pTableMetaInfo1->aliasName));
}
code = tscGetTableMeta(pSql, pTableMetaInfo1);
@ -6315,7 +6412,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
i += 2;
}
assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr / 2);
assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySql->from) / 2);
bool isSTable = false;
if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@ -6349,12 +6446,12 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
}
} else { // set the time rang
if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed.
if (taosArrayGetSize(pQuerySql->from) > 2) { // it is a join query, no wher clause is not allowed.
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
}
}
int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2);
int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2);
if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
@ -6537,7 +6634,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
(*pExpr)->_node.pRight = pRight;
SStrToken t = {.type = pSqlExpr->nSQLOptr};
(*pExpr)->_node.optr = getBinaryExprOptr(&t);
(*pExpr)->_node.optr = convertOptr(&t);
assert((*pExpr)->_node.optr != 0);

10
src/client/src/tscSchemaUtil.c
View File

@ -130,16 +130,6 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
return NULL;
}
struct SSchema tscGetTbnameColumnSchema() {
struct SSchema s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
.bytes = TSDB_TABLE_NAME_LEN
};
strcpy(s.name, TSQL_TBNAME_L);
return s;
}
static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
corVgroupInfo->version = 0;
corVgroupInfo->inUse = 0;

10
src/client/src/tscServer.c
View File

@ -241,11 +241,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.code = 0
};
// NOTE: the rpc context should be acquired before sending data to server.
// Otherwise, the pSql object may have been released already during the response function, which is
// processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
// cause crash.
pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid);
return TSDB_CODE_SUCCESS;
}
@ -361,7 +357,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen);
}
} else {
pRes->pRsp = NULL;
tfree(pRes->pRsp);
}
/*
@ -892,7 +888,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t msgLen = (int32_t)(pMsg - pCmd->payload);
tscDebug("%p msg built success,len:%d bytes", pSql, msgLen);
tscDebug("%p msg built success, len:%d bytes", pSql, msgLen);
pCmd->payloadLen = msgLen;
pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY;

2
src/client/src/tscSystem.c
View File

@ -105,6 +105,7 @@ void taos_init_imp(void) {
taosReadGlobalCfg();
taosCheckGlobalCfg();
rpcInit();
tscDebug("starting to initialize TAOS client ...");
tscDebug("Local End Point is:%s", tsLocalEp);
}
@ -179,6 +180,7 @@ void taos_cleanup(void) {
taosCloseRef(tscRefId);
taosCleanupKeywordsTable();
taosCloseLog();
if (tscEmbedded == 0) rpcCleanup();
m = tscTmr;
if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {

7
src/client/src/tscUtil.c
View File

@ -71,7 +71,8 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
}
bool tscQueryTags(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
int32_t functId = pExpr->functionId;
@ -201,13 +202,9 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
size_t size = tscSqlExprNumOfExprs(pQueryInfo);
for (int32_t i = 0; i < size; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
assert(pExpr != NULL);
// if (pExpr == NULL) {
// return false;
// }
int32_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TAG) {

2
src/common/inc/tname.h
View File

@ -35,6 +35,6 @@ bool tscValidateTableNameLength(size_t len);
SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);
// int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision);
SSchema tscGetTbnameColumnSchema();
#endif // TDENGINE_NAME_H

11
src/common/src/tname.c
View File

@ -188,3 +188,14 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
pToken->z = r;
}
}
SSchema tscGetTbnameColumnSchema() {
struct SSchema s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
.bytes = TSDB_TABLE_NAME_LEN
};
strcpy(s.name, TSQL_TBNAME_L);
return s;
}

34
src/common/src/ttypes.c
View File

@ -355,32 +355,6 @@ bool isValidDataType(int32_t type) {
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
}
//bool isNull(const char *val, int32_t type) {
// switch (type) {
// case TSDB_DATA_TYPE_BOOL:
// return *(uint8_t *)val == TSDB_DATA_BOOL_NULL;
// case TSDB_DATA_TYPE_TINYINT:
// return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL;
// case TSDB_DATA_TYPE_SMALLINT:
// return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL;
// case TSDB_DATA_TYPE_INT:
// return *(uint32_t *)val == TSDB_DATA_INT_NULL;
// case TSDB_DATA_TYPE_BIGINT:
// case TSDB_DATA_TYPE_TIMESTAMP:
// return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL;
// case TSDB_DATA_TYPE_FLOAT:
// return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
// case TSDB_DATA_TYPE_DOUBLE:
// return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
// case TSDB_DATA_TYPE_NCHAR:
// return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
// case TSDB_DATA_TYPE_BINARY:
// return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
// default:
// return false;
// };
//}
void setVardataNull(char* val, int32_t type) {
if (type == TSDB_DATA_TYPE_BINARY) {
varDataSetLen(val, sizeof(int8_t));
@ -433,14 +407,10 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
*(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
}
break;
case TSDB_DATA_TYPE_NCHAR: // todo : without length?
for (int32_t i = 0; i < numOfElems; ++i) {
*(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
}
break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_BINARY:
for (int32_t i = 0; i < numOfElems; ++i) {
*(uint8_t *)(val + i * bytes) = TSDB_DATA_BINARY_NULL;
setVardataNull(val + i * bytes, type);
}
break;
default: {

2
src/dnode/src/dnodeMain.c
View File

@ -20,6 +20,7 @@
#include "tconfig.h"
#include "tglobal.h"
#include "twal.h"
#include "trpc.h"
#include "dnode.h"
#include "dnodeInt.h"
#include "dnodeMgmt.h"
@ -54,6 +55,7 @@ typedef struct {
} SDnodeComponent;
static const SDnodeComponent tsDnodeComponents[] = {
{"rpc", rpcInit, rpcCleanup},
{"storage", dnodeInitStorage, dnodeCleanupStorage},
{"dnodecfg", dnodeInitCfg, dnodeCleanupCfg},
{"dnodeeps", dnodeInitEps, dnodeCleanupEps},

4
src/dnode/src/dnodePeer.c
View File

@ -169,7 +169,7 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
}
void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
rpcSendRequest(tsClientRpc, epSet, rpcMsg);
rpcSendRequest(tsClientRpc, epSet, rpcMsg, NULL);
}
void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
@ -180,4 +180,4 @@ void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp);
}
}

12
src/dnode/src/dnodeVWrite.c
View File

@ -212,7 +212,7 @@ static void *dnodeProcessVWriteQueue(void *param) {
pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
if (pWrite->code <= 0) pWrite->processedCount = 1;
if (pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
if (pWrite->code == 0 && pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code));
}
@ -225,11 +225,13 @@ static void *dnodeProcessVWriteQueue(void *param) {
taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
if (qtype == TAOS_QTYPE_RPC) {
dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
} else if (qtype == TAOS_QTYPE_FWD) {
vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
taosFreeQitem(pWrite);
vnodeRelease(pVnode);
} else {
if (qtype == TAOS_QTYPE_FWD) {
vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
}
if (pWrite->rspRet.rsp) {
rpcFreeCont(pWrite->rspRet.rsp);
}
taosFreeQitem(pWrite);
vnodeRelease(pVnode);
}

83
src/inc/taosdef.h
View File

@ -61,13 +61,23 @@ typedef struct tstr {
// Bytes for each type.
extern const int32_t TYPE_BYTES[11];
// TODO: replace and remove code below
#define CHAR_BYTES sizeof(char)
#define SHORT_BYTES sizeof(int16_t)
#define INT_BYTES sizeof(int32_t)
#define LONG_BYTES sizeof(int64_t)
#define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double)
#define CHAR_BYTES sizeof(char)
#define SHORT_BYTES sizeof(int16_t)
#define INT_BYTES sizeof(int32_t)
#define LONG_BYTES sizeof(int64_t)
#define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
#define TSDB_KEYSIZE sizeof(TSKEY)
#if LINUX
#define TSDB_NCHAR_SIZE sizeof(wchar_t)
#else
#define TSDB_NCHAR_SIZE sizeof(int32_t)
#endif
// NULL definition
#define TSDB_DATA_BOOL_NULL 0x02
@ -102,10 +112,12 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_TIME_PRECISION_MILLI 0
#define TSDB_TIME_PRECISION_MICRO 1
#define TSDB_TIME_PRECISION_NANO 2
#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
#define TSDB_TIME_PRECISION_MILLI_STR "ms"
#define TSDB_TIME_PRECISION_MICRO_STR "us"
#define TSDB_TIME_PRECISION_NANO_STR "ns"
#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
#define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member)
#define T_APPEND_MEMBER(dst, ptr, type, member) \
@ -119,15 +131,6 @@ do { \
(src) = (void *)((char *)src + sizeof(type));\
} while(0)
#define TSDB_KEYSIZE sizeof(TSKEY)
#if LINUX
#define TSDB_NCHAR_SIZE sizeof(wchar_t)
#else
#define TSDB_NCHAR_SIZE 4
#endif
//#define TSDB_CHAR_TERMINATED_SPACE 1
#define GET_INT8_VAL(x) (*(int8_t *)(x))
#define GET_INT16_VAL(x) (*(int16_t *)(x))
#define GET_INT32_VAL(x) (*(int32_t *)(x))
@ -173,7 +176,6 @@ typedef struct tDataTypeDescriptor {
} tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypeDesc[11];
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
bool isValidDataType(int32_t type);
//bool isNull(const char *val, int32_t type);
@ -267,10 +269,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_AUTH_LEN 16
#define TSDB_KEY_LEN 16
#define TSDB_VERSION_LEN 12
#define TSDB_STREET_LEN 64
#define TSDB_CITY_LEN 20
#define TSDB_STATE_LEN 20
#define TSDB_COUNTRY_LEN 20
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 96
#define TSDB_LABEL_LEN 8
@ -393,27 +391,27 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
* 1. ordinary sub query for select * from super_table
* 2. all sqlobj generated by createSubqueryObj with this flag
*/
#define TSDB_QUERY_TYPE_SUBQUERY 0x02u
#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table
#define TSDB_QUERY_TYPE_SUBQUERY 0x02u
#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table
#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side
#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table
#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query
#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query
#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage
#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side
#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table
#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query
#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query
#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage
#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type
#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type
#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type
#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type
#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0)
#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type))
#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type))
#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE)
#define TSDB_ORDER_ASC 1
#define TSDB_ORDER_DESC 2
#define TSDB_ORDER_ASC 1
#define TSDB_ORDER_DESC 2
#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1
#define TSDB_DEFAULT_MNODES_HASH_SIZE 5
@ -425,10 +423,15 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_DEFAULT_STABLES_HASH_SIZE 100
#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000
#define TSDB_PORT_DNODESHELL 0
#define TSDB_PORT_DNODEDNODE 5
#define TSDB_PORT_SYNC 10
#define TSDB_PORT_HTTP 11
#define TSDB_PORT_DNODESHELL 0
#define TSDB_PORT_DNODEDNODE 5
#define TSDB_PORT_SYNC 10
#define TSDB_PORT_HTTP 11
#define TSDB_PORT_ARBITRATOR 12
#define TSDB_PORT_DNODESHELL 0
#define TSDB_PORT_DNODEDNODE 5
#define TSDB_PORT_SYNC 10
#define TSDB_PORT_HTTP 11
#define TSDB_PORT_ARBITRATOR 12
#define TSDB_MAX_WAL_SIZE (1024*1024)
@ -462,8 +465,8 @@ typedef enum {
TSDB_CHECK_ITEM_MEM,
TSDB_CHECK_ITEM_CPU,
TSDB_CHECK_ITEM_DISK,
TSDB_CHECK_ITEM_OS,
TSDB_CHECK_ITEM_ACCESS,
TSDB_CHECK_ITEM_OS,
TSDB_CHECK_ITEM_ACCESS,
TSDB_CHECK_ITEM_VERSION,
TSDB_CHECK_ITEM_DATAFILE,
TSDB_CHECK_ITEM_MAX

16
src/inc/taosmsg.h
View File

@ -481,15 +481,15 @@ typedef struct {
int64_t limit;
int64_t offset;
uint32_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t numOfOutput; // final output columns numbers
int16_t tagNameRelType; // relation of tag criteria and tbname criteria
int16_t fillType; // interpolate type
uint64_t fillVal; // default value array list
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
int32_t tsOrder; // ts comp block order
int32_t numOfTags; // number of tags columns involved
int16_t fillType; // interpolate type
uint64_t fillVal; // default value array list
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
int32_t tsOrder; // ts comp block order
int32_t numOfTags; // number of tags columns involved
SColumnInfo colList[];
} SQueryTableMsg;

4
src/inc/trpc.h
View File

@ -78,12 +78,14 @@ typedef struct SRpcInit {
int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
} SRpcInit;
int32_t rpcInit();
void rpcCleanup();
void *rpcOpen(const SRpcInit *pRpc);
void rpcClose(void *);
void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen);
int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);

6
src/inc/tsdb.h
View File

@ -46,7 +46,7 @@ extern "C" {
typedef struct {
void *appH;
void *cqH;
int (*notifyStatus)(void *, int status);
int (*notifyStatus)(void *, int status, int eno);
int (*eventCallBack)(void *);
void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
void (*cqDropFunc)(void *handle);
@ -126,7 +126,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
// the TSDB repository info
typedef struct STsdbRepoInfo {
STsdbCfg tsdbCfg;
int64_t version; // version of the repository
uint64_t version; // version of the repository
int64_t tsdbTotalDataSize; // the original inserted data size
int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository
// TODO: Other informations to add
@ -136,7 +136,7 @@ STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
// the meter information report structure
typedef struct {
STableCfg tableCfg;
int64_t version;
uint64_t version;
int64_t tableTotalDataSize; // In bytes
int64_t tableTotalDiskSize; // In bytes
} STableInfo;

2
src/inc/ttokendef.h
View File

@ -16,7 +16,6 @@
#ifndef TDENGINE_TTOKENDEF_H
#define TDENGINE_TTOKENDEF_H
#define TK_ID 1
#define TK_BOOL 2
#define TK_TINYINT 3
@ -224,6 +223,7 @@
#define TK_INTO 205
#define TK_VALUES 206
#define TK_SPACE 300
#define TK_COMMENT 301
#define TK_ILLEGAL 302

36
src/inc/ttype.h Normal file
View File

@ -0,0 +1,36 @@
#ifndef TDENGINE_TTYPE_H
#define TDENGINE_TTYPE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "taosdef.h"
#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
switch (_type) { \
case TSDB_DATA_TYPE_TINYINT: \
(_v) = (_finalType)GET_INT8_VAL(_data); \
break; \
case TSDB_DATA_TYPE_SMALLINT: \
(_v) = (_finalType)GET_INT16_VAL(_data); \
break; \
case TSDB_DATA_TYPE_BIGINT: \
(_v) = (_finalType)(GET_INT64_VAL(_data)); \
break; \
case TSDB_DATA_TYPE_FLOAT: \
(_v) = (_finalType)GET_FLOAT_VAL(_data); \
break; \
case TSDB_DATA_TYPE_DOUBLE: \
(_v) = (_finalType)GET_DOUBLE_VAL(_data); \
break; \
default: \
(_v) = (_finalType)GET_INT32_VAL(_data); \
break; \
};
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TTYPE_H

1
src/kit/CMakeLists.txt
View File

@ -3,3 +3,4 @@ PROJECT(TDengine)
ADD_SUBDIRECTORY(shell)
ADD_SUBDIRECTORY(taosdemo)
ADD_SUBDIRECTORY(taosdump)

2
src/kit/shell/src/shellLinux.c
View File

@ -46,7 +46,7 @@ static struct argp_option options[] = {
{"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."},
{"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."},
{"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."},
{"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, valid option: client | server."},
{"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is NULL, options: client|clients|server."},
{"endport", 'e', "ENDPORT", 0, "Net test end port, default is 6042."},
{"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."},
{0}};

16
src/kit/taosdump/CMakeLists.txt Normal file
View File

@ -0,0 +1,16 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(. SRC)
IF (TD_LINUX)
ADD_EXECUTABLE(taosdump ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdump taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(taosdump taos)
ENDIF ()
ENDIF ()

2246
src/kit/taosdump/taosdump.c Normal file
View File

File diff suppressed because it is too large Load Diff

48
src/kit/taosdump/taosdump.sh Executable file
View File

@ -0,0 +1,48 @@
taos1_6="/root/mnt/work/test/td1.6/build/bin/taos"
taosdump1_6="/root/mnt/work/test/td1.6/build/bin/taosdump"
taoscfg1_6="/root/mnt/work/test/td1.6/test/cfg"
taos2_0="/root/mnt/work/test/td2.0/build/bin/taos"
taosdump2_0="/root/mnt/work/test/td2.0/build/bin/taosdump"
taoscfg2_0="/root/mnt/work/test/td2.0/test/cfg"
data_dir="/root/mnt/work/test/td1.6/output"
table_list="/root/mnt/work/test/td1.6/tables"
DBNAME="test"
NTABLES=$(wc -l ${table_list} | awk '{print $1;}')
NTABLES_PER_DUMP=101
mkdir -p ${data_dir}
i=0
round=0
command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 -T 20 ${DBNAME}"
while IFS= read -r line
do
i=$((i+1))
command="${command} ${line}"
if [[ "$i" -eq ${NTABLES_PER_DUMP} ]]; then
round=$((round+1))
echo "Starting round ${round} dump out..."
rm -f ${data_dir}/*
${command}
echo "Starting round ${round} dump in..."
${taosdump2_0} -c ${taoscfg2_0} -i ${data_dir}
# Reset variables
# command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 ${DBNAME}"
command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 -T 20 ${DBNAME}"
i=0
fi
done < "${table_list}"
if [[ ${i} -ne "0" ]]; then
round=$((round+1))
echo "Starting round ${round} dump out..."
rm -f ${data_dir}/*
${command}
echo "Starting round ${round} dump in..."
${taosdump2_0} -c ${taoscfg2_0} -i ${data_dir}
fi

2
src/mnode/src/mnodeSdb.c
View File

@ -71,7 +71,7 @@ typedef struct _SSdbTable {
typedef struct {
ESyncRole role;
ESdbStatus status;
int64_t version;
uint64_t version;
int64_t sync;
void * wal;
SSyncCfg cfg;

12
src/query/inc/qAst.h
View File

@ -74,21 +74,19 @@ typedef struct tExprNode {
};
} tExprNode;
void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param);
void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*cb)(void *, const char*, int32_t));
uint8_t getBinaryExprOptr(SStrToken *pToken);
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
tExprNode* exprTreeFromBinary(const void* data, size_t size);
tExprNode* exprTreeFromTableName(const char* tbnameCond);
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
#ifdef __cplusplus
}
#endif

50
src/query/inc/qFill.h
View File

@ -28,6 +28,7 @@ typedef struct {
STColumn col; // column info
int16_t functionId; // sql function id
int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN
int16_t tagIndex; // index of current tag in SFillTagColInfo array list
union {int64_t i; double d;} fillVal;
} SFillColInfo;
@ -37,26 +38,29 @@ typedef struct {
} SFillTagColInfo;
typedef struct SFillInfo {
TSKEY start; // start timestamp
TSKEY endKey; // endKey for fill
int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
int32_t fillType; // fill type
int32_t numOfRows; // number of rows in the input data block
int32_t rowIdx; // rowIdx
int32_t numOfTotal; // number of filled rows in one round
int32_t numOfCurrent; // number of filled rows in current results
TSKEY start; // start timestamp
TSKEY end; // endKey for fill
TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure.
int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
int32_t type; // fill type
int32_t numOfRows; // number of rows in the input data block
int32_t index; // active row index
int32_t numOfTotal; // number of filled rows in one round
int32_t numOfCurrent; // number of filled rows in current results
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
SFillTagColInfo* pTags; // tags value for filling gap
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
SInterval interval;
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t capacityInRows; // data buffer size in rows
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t alloc; // data buffer size in rows
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
SFillTagColInfo* pTags; // tags value for filling gap
void* handle; // for dubug purpose
} SFillInfo;
typedef struct SPoint {
@ -66,7 +70,7 @@ typedef struct SPoint {
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pFillCol);
SFillColInfo* pFillCol, void* handle);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
@ -74,17 +78,17 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput);
void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput);
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput);
int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int64_t getNumOfResWithFill(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
#ifdef __cplusplus
}

99
src/query/inc/qSqlparser.h
View File

@ -20,10 +20,10 @@
extern "C" {
#endif
#include <tstrbuild.h>
#include "taos.h"
#include "taosmsg.h"
#include "tstoken.h"
#include "tstrbuild.h"
#include "tvariant.h"
#define ParseTOKENTYPE SStrToken
@ -37,12 +37,6 @@ extern char tTokenTypeSwitcher[13];
(x) = tTokenTypeSwitcher[(x)]; \
} \
} while (0)
typedef struct tFieldList {
int32_t nField;
int32_t nAlloc;
TAOS_FIELD *p;
} tFieldList;
typedef struct SLimitVal {
int64_t limit;
@ -59,12 +53,6 @@ typedef struct tVariantListItem {
uint8_t sortOrder;
} tVariantListItem;
typedef struct tVariantList {
int32_t nExpr; /* Number of expressions on the list */
int32_t nAlloc; /* Number of entries allocated below */
tVariantListItem *a; /* One entry for each expression */
} tVariantList;
typedef struct SIntervalVal {
SStrToken interval;
SStrToken offset;
@ -72,16 +60,16 @@ typedef struct SIntervalVal {
typedef struct SQuerySQL {
struct tSQLExprList *pSelection; // select clause
tVariantList * from; // from clause
SArray * from; // from clause SArray<tVariantListItem>
struct tSQLExpr * pWhere; // where clause [optional]
tVariantList * pGroupby; // groupby clause, only for tags[optional]
tVariantList * pSortOrder; // orderby [optional]
SArray * pGroupby; // groupby clause, only for tags[optional], SArray<tVariantListItem>
SArray * pSortOrder; // orderby [optional], SArray<tVariantListItem>
SStrToken interval; // interval [optional]
SStrToken offset; // offset window [optional]
SStrToken sliding; // sliding window [optional]
SLimitVal limit; // limit offset [optional]
SLimitVal slimit; // group limit offset [optional]
tVariantList * fillType; // fill type[optional]
SArray * fillType; // fill type[optional], SArray<tVariantListItem>
SStrToken selectToken; // sql string
} SQuerySQL;
@ -91,26 +79,25 @@ typedef struct SCreateTableSQL {
int8_t type; // create normal table/from super table/ stream
struct {
tFieldList *pTagColumns; // for normal table, pTagColumns = NULL;
tFieldList *pColumns;
SArray *pTagColumns; // SArray<TAOS_FIELD>
SArray *pColumns; // SArray<TAOS_FIELD>
} colInfo;
struct {
SStrToken stableName; // super table name, for using clause
tVariantList *pTagVals; // create by using metric, tag value
STagData tagdata;
SStrToken stableName; // super table name, for using clause
SArray *pTagVals; // create by using metric, tag value
STagData tagdata;
} usingInfo;
SQuerySQL *pSelect;
SQuerySQL *pSelect;
} SCreateTableSQL;
typedef struct SAlterTableSQL {
SStrToken name;
int16_t type;
STagData tagData;
tFieldList * pAddColumns;
tVariantList *varList; // set t=val or: change src dst
SArray *pAddColumns; // SArray<TAOS_FIELD>
SArray *varList; // set t=val or: change src dst, SArray<tVariantListItem>
} SAlterTableSQL;
typedef struct SCreateDBInfo {
@ -131,7 +118,7 @@ typedef struct SCreateDBInfo {
bool ignoreExists;
int8_t update;
tVariantList *keep;
SArray *keep;
} SCreateDBInfo;
typedef struct SCreateAcctSQL {
@ -169,7 +156,7 @@ typedef struct tDCLSQL {
SCreateDBInfo dbOpt;
SCreateAcctSQL acctOpt;
SShowInfo showOpt;
SStrToken ip;
SStrToken ip;
};
SUserInfo user;
@ -182,33 +169,32 @@ typedef struct SSubclauseInfo { // "UNION" multiple select sub-clause
} SSubclauseInfo;
typedef struct SSqlInfo {
int32_t type;
bool valid;
int32_t type;
bool valid;
union {
SCreateTableSQL *pCreateTableInfo;
SAlterTableSQL * pAlterInfo;
tDCLSQL * pDCLInfo;
SAlterTableSQL *pAlterInfo;
tDCLSQL *pDCLInfo;
};
SSubclauseInfo subclauseInfo;
char pzErrMsg[256];
SSubclauseInfo subclauseInfo;
char pzErrMsg[256];
} SSqlInfo;
typedef struct tSQLExpr {
// TK_FUNCTION: sql function, TK_LE: less than(binary expr)
uint32_t nSQLOptr;
uint32_t nSQLOptr; // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
// the full sql string of function(col, param), which is actually the raw
// field name, since the function name is kept in nSQLOptr already
SStrToken operand;
SStrToken colInfo; // field id
tVariant val; // value only for string, float, int
SStrToken operand;
SStrToken colInfo; // field id
tVariant val; // value only for string, float, int
SStrToken token; // original sql expr string
struct tSQLExpr *pLeft; // left child
struct tSQLExpr *pRight; // right child
struct tSQLExprList *pParam; // function parameters
SStrToken token; // original sql expr string
} tSQLExpr;
// used in select clause. select <tSQLExprList> from xxx
@ -239,16 +225,9 @@ void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *);
*/
void ParseFree(void *p, void (*freeProc)(void *));
tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder);
tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder);
void tVariantListDestroy(tVariantList *pList);
tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField);
void tFieldListDestroy(tFieldList *pList);
SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder);
SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder);
tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType);
@ -258,17 +237,16 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken
void tSQLExprListDestroy(tSQLExprList *pList);
SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName,
tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type);
SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pMetricName,
SArray *pTagVals, SQuerySQL *pSelect, int32_t type);
void tSQLExprNodeDestroy(tSQLExpr *pExpr);
tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr);
void tSQLExprNodeDestroy(tSQLExpr *pExpr);
SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type);
SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type);
void destroyAllSelectClause(SSubclauseInfo *pSql);
void doDestroyQuerySql(SQuerySQL *pSql);
@ -310,9 +288,6 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken);
void *ParseAlloc(void *(*mallocProc)(size_t));
// convert the sql filter expression into binary data
int32_t tSQLExprToBinary(tSQLExpr* pExpr, SStringBuilder* sb);
enum {
TSQL_NODE_TYPE_EXPR = 0x1,
TSQL_NODE_TYPE_ID = 0x2,

54
src/query/inc/sql.y
View File

@ -223,8 +223,8 @@ acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K
Y.stat = M;
}
%type keep {tVariantList*}
%destructor keep {tVariantListDestroy($$);}
%type keep {SArray*}
%destructor keep {taosArrayDestroy($$);}
keep(Y) ::= KEEP tagitemlist(X). { Y = X; }
cache(Y) ::= CACHE INTEGER(X). { Y = X; }
@ -327,10 +327,10 @@ create_table_args(A) ::= AS select(S). {
}
%type column{TAOS_FIELD}
%type columnlist{tFieldList*}
%destructor columnlist {tFieldListDestroy($$);}
columnlist(A) ::= columnlist(X) COMMA column(Y). {A = tFieldListAppend(X, &Y); }
columnlist(A) ::= column(X). {A = tFieldListAppend(NULL, &X);}
%type columnlist{SArray*}
%destructor columnlist {taosArrayDestroy($$);}
columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; }
columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(A, &X);}
// The information used for a column is the name and type of column:
// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x)
@ -338,8 +338,8 @@ column(A) ::= ids(X) typename(Y). {
tSQLSetColumnInfo(&A, &X, &Y);
}
%type tagitemlist {tVariantList*}
%destructor tagitemlist {tVariantListDestroy($$);}
%type tagitemlist {SArray*}
%destructor tagitemlist {taosArrayDestroy($$);}
%type tagitem {tVariant}
tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); }
@ -432,11 +432,11 @@ as(X) ::= ids(Y). { X = Y; }
as(X) ::= . { X.n = 0; }
// A complete FROM clause.
%type from {tVariantList*}
%type from {SArray*}
// current not support query from no-table
from(A) ::= FROM tablelist(X). {A = X;}
%type tablelist {tVariantList*}
%type tablelist {SArray*}
tablelist(A) ::= ids(X) cpxName(Y). {
toTSDBType(X.type);
X.n += Y.n;
@ -476,8 +476,8 @@ interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0;
interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP. {N.interval = E; N.offset = O;}
interval_opt(N) ::= . {memset(&N, 0, sizeof(N));}
%type fill_opt {tVariantList*}
%destructor fill_opt {tVariantListDestroy($$);}
%type fill_opt {SArray*}
%destructor fill_opt {taosArrayDestroy($$);}
fill_opt(N) ::= . {N = 0; }
fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. {
tVariant A = {0};
@ -497,11 +497,11 @@ fill_opt(N) ::= FILL LP ID(Y) RP. {
sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; }
sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; }
%type orderby_opt {tVariantList*}
%destructor orderby_opt {tVariantListDestroy($$);}
%type orderby_opt {SArray*}
%destructor orderby_opt {taosArrayDestroy($$);}
%type sortlist {tVariantList*}
%destructor sortlist {tVariantListDestroy($$);}
%type sortlist {SArray*}
%destructor sortlist {taosArrayDestroy($$);}
%type sortitem {tVariant}
%destructor sortitem {tVariantDestroy(&$$);}
@ -531,10 +531,10 @@ sortorder(A) ::= DESC. {A = TSDB_ORDER_DESC;}
sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend order
//group by clause
%type groupby_opt {tVariantList*}
%destructor groupby_opt {tVariantListDestroy($$);}
%type grouplist {tVariantList*}
%destructor grouplist {tVariantListDestroy($$);}
%type groupby_opt {SArray*}
%destructor groupby_opt {taosArrayDestroy($$);}
%type grouplist {SArray*}
%destructor grouplist {taosArrayDestroy($$);}
groupby_opt(A) ::= . {A = 0;}
groupby_opt(A) ::= GROUP BY grouplist(X). {A = X;}
@ -556,11 +556,11 @@ having_opt(A) ::= HAVING expr(X). {A = X;}
//limit-offset subclause
%type limit_opt {SLimitVal}
limit_opt(A) ::= . {A.limit = -1; A.offset = 0;}
limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;}
limit_opt(A) ::= LIMIT signed(X). {printf("aa1, %d\n", X); A.limit = X; A.offset = 0;}
limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y).
{A.limit = X; A.offset = Y;}
{printf("aa2\n, %d\n", X); A.limit = X; A.offset = Y;}
limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y).
{A.limit = Y; A.offset = X;}
{printf("aa3\n, %d\n", X); A.limit = Y; A.offset = X;}
%type slimit_opt {SLimitVal}
slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;}
@ -657,7 +657,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). {
X.n += F.n;
toTSDBType(A.type);
tVariantList* K = tVariantListAppendToken(NULL, &A, -1);
SArray* K = tVariantListAppendToken(NULL, &A, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@ -673,7 +673,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). {
X.n += Z.n;
toTSDBType(Y.type);
tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@ -683,7 +683,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). {
X.n += F.n;
toTSDBType(Y.type);
tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
toTSDBType(Z.type);
A = tVariantListAppendToken(A, &Z, -1);
@ -696,7 +696,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). {
X.n += F.n;
toTSDBType(Y.type);
tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
SArray* A = tVariantListAppendToken(NULL, &Y, -1);
A = tVariantListAppend(A, &Z, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);

444
src/query/src/qAst.c
View File

@ -13,12 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "exception.h"
#include "qAst.h"
#include "qSqlparser.h"
#include "qSyntaxtreefunction.h"
#include "taosdef.h"
#include "taosmsg.h"
@ -30,200 +28,19 @@
#include "tskiplist.h"
#include "tsqlfunction.h"
#include "tstoken.h"
#include "ttokendef.h"
#include "tulog.h"
#include "tschemautil.h"
/*
*
* @date 2018-2-15
* @version 0.2 operation for column filter
*
* @Description parse tag query expression to build ast
* ver 0.2, filter the result on first column with high priority to limit the candidate set
* ver 0.3, pipeline filter in the form of: (a+2)/9 > 14
*
*/
static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken);
typedef struct {
char* v;
int32_t optr;
} SEndPoint;
static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i);
static void destroySyntaxTree(tExprNode *);
typedef struct {
SEndPoint* start;
SEndPoint* end;
} SQueryCond;
static uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight);
/*
* Check the filter value type on the right hand side based on the column id on the left hand side,
* the filter value type must be identical to field type for relational operation
* As for binary arithmetic operation, it is not necessary to do so.
*/
static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, uint8_t *optr) {
if (*optr >= TSDB_RELATION_LESS && *optr <= TSDB_RELATION_LIKE) {
// make sure that the type of data on both sides of relational comparision are identical
if ((*pLeft)->nodeType == TSQL_NODE_VALUE) {
tVariantTypeSetType((*pLeft)->pVal, (*pRight)->pSchema->type);
} else if ((*pRight)->nodeType == TSQL_NODE_VALUE) {
tVariantTypeSetType((*pRight)->pVal, (*pLeft)->pSchema->type);
}
} else if (*optr >= TSDB_BINARY_OP_ADD && *optr <= TSDB_BINARY_OP_REMAINDER) {
if ((*pLeft)->nodeType == TSQL_NODE_VALUE) {
/* convert to int/bigint may cause the precision loss */
tVariantTypeSetType((*pLeft)->pVal, TSDB_DATA_TYPE_DOUBLE);
} else if ((*pRight)->nodeType == TSQL_NODE_VALUE) {
/* convert to int/bigint may cause the precision loss */
tVariantTypeSetType((*pRight)->pVal, TSDB_DATA_TYPE_DOUBLE);
}
}
/*
* for expressions that are suitable for switch principle,
* switch left and left and right hand side in expr if possible
*/
if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) {
if (*optr >= TSDB_RELATION_GREATER && *optr <= TSDB_RELATION_GREATER_EQUAL && *optr != TSDB_RELATION_EQUAL) {
SWAP(*pLeft, *pRight, tExprNode *);
}
switch (*optr) {
case TSDB_RELATION_GREATER:
(*optr) = TSDB_RELATION_LESS;
break;
case TSDB_RELATION_LESS:
(*optr) = TSDB_RELATION_GREATER;
break;
case TSDB_RELATION_GREATER_EQUAL:
(*optr) = TSDB_RELATION_LESS_EQUAL;
break;
case TSDB_RELATION_LESS_EQUAL:
(*optr) = TSDB_RELATION_GREATER_EQUAL;
break;
default:;
// for other type of operations, do nothing
}
}
}
static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) {
/* if the token is not a value, return false */
if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID &&
pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) {
return NULL;
}
size_t nodeSize = sizeof(tExprNode);
tExprNode *pNode = NULL;
if (pToken->type == TK_ID || pToken->type == TK_TBNAME) {
int32_t i = 0;
if (pToken->type == TK_ID) {
do {
SStrToken tableToken = {0};
extractTableNameFromToken(pToken, &tableToken);
size_t len = strlen(pSchema[i].name);
if (strncmp(pToken->z, pSchema[i].name, pToken->n) == 0 && pToken->n == len) break;
} while (++i < numOfCols);
if (i == numOfCols) { // column name is not valid, parse the expression failed
return NULL;
}
}
nodeSize += sizeof(SSchema);
pNode = calloc(1, nodeSize);
pNode->pSchema = (struct SSchema *)((char *)pNode + sizeof(tExprNode));
pNode->nodeType = TSQL_NODE_COL;
if (pToken->type == TK_ID) {
memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema));
} else {
pNode->pSchema->type = TSDB_DATA_TYPE_BINARY;
pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
strcpy(pNode->pSchema->name, TSQL_TBNAME_L);
pNode->pSchema->colId = -1;
}
} else {
nodeSize += sizeof(tVariant);
pNode = calloc(1, nodeSize);
pNode->pVal = (tVariant *)((char *)pNode + sizeof(tExprNode));
toTSDBType(pToken->type);
tVariantCreate(pNode->pVal, pToken);
pNode->nodeType = TSQL_NODE_VALUE;
}
return pNode;
}
uint8_t getBinaryExprOptr(SStrToken *pToken) {
switch (pToken->type) {
case TK_LT:
return TSDB_RELATION_LESS;
case TK_LE:
return TSDB_RELATION_LESS_EQUAL;
case TK_GT:
return TSDB_RELATION_GREATER;
case TK_GE:
return TSDB_RELATION_GREATER_EQUAL;
case TK_NE:
return TSDB_RELATION_NOT_EQUAL;
case TK_AND:
return TSDB_RELATION_AND;
case TK_OR:
return TSDB_RELATION_OR;
case TK_EQ:
return TSDB_RELATION_EQUAL;
case TK_PLUS:
return TSDB_BINARY_OP_ADD;
case TK_MINUS:
return TSDB_BINARY_OP_SUBTRACT;
case TK_STAR:
return TSDB_BINARY_OP_MULTIPLY;
case TK_SLASH:
case TK_DIVIDE:
return TSDB_BINARY_OP_DIVIDE;
case TK_REM:
return TSDB_BINARY_OP_REMAINDER;
case TK_LIKE:
return TSDB_RELATION_LIKE;
case TK_ISNULL:
return TSDB_RELATION_ISNULL;
case TK_NOTNULL:
return TSDB_RELATION_NOTNULL;
default: { return 0; }
}
}
// previous generated expr is reduced as the left child
static tExprNode *parseRemainStr(char *pstr, tExprNode *pExpr, SSchema *pSchema, int32_t optr,
int32_t numOfCols, int32_t *i) {
// set the previous generated node as the left child of new root
pExpr->nodeType = TSQL_NODE_EXPR;
// remain is the right child
tExprNode *pRight = createSyntaxTree(pSchema, numOfCols, pstr, i);
if (pRight == NULL || (pRight->nodeType == TSQL_NODE_COL && pExpr->nodeType != TSQL_NODE_VALUE) ||
(pExpr->nodeType == TSQL_NODE_VALUE && pRight->nodeType != TSQL_NODE_COL)) {
tExprNodeDestroy(pExpr, NULL);
tExprNodeDestroy(pRight, NULL);
return NULL;
}
tExprNode *pNewExpr = (tExprNode *)calloc(1, sizeof(tExprNode));
uint8_t k = optr;
reviseBinaryExprIfNecessary(&pExpr, &pRight, &k);
pNewExpr->_node.pLeft = pExpr;
pNewExpr->_node.pRight = pRight;
pNewExpr->_node.optr = k;
pNewExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pExpr, pRight);
pNewExpr->nodeType = TSQL_NODE_EXPR;
return pNewExpr;
}
uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
if (pLeft->nodeType == TSQL_NODE_COL) {
// if left node is the primary column,return true
return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
@ -236,103 +53,6 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef
}
}
static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) {
SStrToken t0 = tStrGetToken(str, i, false, 0, NULL);
if (t0.n == 0) {
return NULL;
}
tExprNode *pLeft = NULL;
if (t0.type == TK_LP) { // start new left child branch
pLeft = createSyntaxTree(pSchema, numOfCols, str, i);
} else {
if (t0.type == TK_RP) {
return NULL;
}
pLeft = tExprNodeCreate(pSchema, numOfCols, &t0);
}
if (pLeft == NULL) {
return NULL;
}
t0 = tStrGetToken(str, i, false, 0, NULL);
if (t0.n == 0 || t0.type == TK_RP) {
if (pLeft->nodeType != TSQL_NODE_EXPR) { // if left is not the expr, it is not a legal expr
tExprNodeDestroy(pLeft, NULL);
return NULL;
}
return pLeft;
}
// get the operator of expr
uint8_t optr = getBinaryExprOptr(&t0);
if (optr == 0) {
uError("not support binary operator:%d", t0.type);
tExprNodeDestroy(pLeft, NULL);
return NULL;
}
assert(pLeft != NULL);
tExprNode *pRight = NULL;
if (t0.type == TK_AND || t0.type == TK_OR || t0.type == TK_LP) {
pRight = createSyntaxTree(pSchema, numOfCols, str, i);
} else {
/*
* In case that pLeft is a field identification,
* we parse the value in expression according to queried field type,
* if we do not get the information, in case of value of field presented first,
* we revised the value after the binary expression is completed.
*/
t0 = tStrGetToken(str, i, true, 0, NULL);
if (t0.n == 0) {
tExprNodeDestroy(pLeft, NULL); // illegal expression
return NULL;
}
if (t0.type == TK_LP) {
pRight = createSyntaxTree(pSchema, numOfCols, str, i);
} else {
pRight = tExprNodeCreate(pSchema, numOfCols, &t0);
}
}
if (pRight == NULL) {
tExprNodeDestroy(pLeft, NULL);
return NULL;
}
/* create binary expr as the child of new parent node */
tExprNode *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode));
reviseBinaryExprIfNecessary(&pLeft, &pRight, &optr);
pExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight);
pExpr->_node.pLeft = pLeft;
pExpr->_node.pRight = pRight;
pExpr->_node.optr = optr;
t0 = tStrGetToken(str, i, true, 0, NULL);
if (t0.n == 0 || t0.type == TK_RP) {
pExpr->nodeType = TSQL_NODE_EXPR;
return pExpr;
} else {
uint8_t localOptr = getBinaryExprOptr(&t0);
if (localOptr == 0) {
uError("not support binary operator:%d", t0.type);
free(pExpr);
return NULL;
}
return parseRemainStr(str, pExpr, pSchema, localOptr, numOfCols, i);
}
}
static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); }
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
if (pNode == NULL) {
return;
@ -372,16 +92,6 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
*pExpr = NULL;
}
typedef struct {
char* v;
int32_t optr;
} SEndPoint;
typedef struct {
SEndPoint* start;
SEndPoint* end;
} SQueryCond;
// todo check for malloc failure
static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
int32_t optr = queryColInfo->optr;
@ -395,13 +105,10 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
pCond->end = calloc(1, sizeof(SEndPoint));
pCond->end->optr = queryColInfo->optr;
pCond->end->v = queryColInfo->q;
} else if (optr == TSDB_RELATION_IN) {
printf("relation is in\n");
assert(0);
} else if (optr == TSDB_RELATION_LIKE) {
printf("relation is like\n");
} else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) {
assert(0);
}
return TSDB_CODE_SUCCESS;
}
@ -529,99 +236,6 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
tSkipListDestroyIter(iter);
}
int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
// assert(pFinalRes->pRes == 0);
//
// pFinalRes->pRes = calloc((size_t)(pLeft->num + pRight->num), POINTER_BYTES);
// pFinalRes->num = 0;
//
// // sort according to address
// tSkipListNode **pLeftNodes = (tSkipListNode **)pLeft->pRes;
// qsort(pLeftNodes, pLeft->num, sizeof(pLeft->pRes[0]), compareByAddr);
//
// tSkipListNode **pRightNodes = (tSkipListNode **)pRight->pRes;
// qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr);
//
// int32_t i = 0, j = 0;
//
// // merge two sorted arrays in O(n) time
// while (i < pLeft->num && j < pRight->num) {
// int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j];
//
// if (ret < 0) {
// pFinalRes->pRes[pFinalRes->num++] = pLeftNodes[i++];
// } else if (ret > 0) {
// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
// } else { // pNode->key > pkey[i]
// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
// i++;
// }
// }
//
// while (i < pLeft->num) {
// pFinalRes->pRes[pFinalRes->num++] = pLeftNodes[i++];
// }
//
// while (j < pRight->num) {
// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
// }
//
// return pFinalRes->num;
return 0;
}
int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
// int64_t num = MIN(pLeft->num, pRight->num);
//
// assert(pFinalRes->pRes == 0);
//
// pFinalRes->pRes = calloc(num, POINTER_BYTES);
// pFinalRes->num = 0;
//
// // sort according to address
// tSkipListNode **pLeftNodes = (tSkipListNode **)pLeft->pRes;
// qsort(pLeftNodes, pLeft->num, sizeof(pLeft->pRes[0]), compareByAddr);
//
// tSkipListNode **pRightNodes = (tSkipListNode **)pRight->pRes;
// qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr);
//
// int32_t i = 0, j = 0;
// // merge two sorted arrays in O(n) time
// while (i < pLeft->num && j < pRight->num) {
// int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j];
//
// if (ret < 0) {
// i++;
// } else if (ret > 0) {
// j++;
// } else { // pNode->key > pkey[i]
// pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j];
// i++;
// j++;
// }
// }
//
// return pFinalRes->num;
return 0;
}
/*
* traverse the result and apply the function to each item to check if the item is qualified or not
*/
static UNUSED_FUNC void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) {
assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL);
// scan the result array list and check for each item in the list
for (int32_t i = 0; i < taosArrayGetSize(pResult); ++i) {
void* item = taosArrayGet(pResult, i);
if (fp(item, pExpr->_node.info)) {
i++;
} else {
taosArrayRemove(pResult, i);
}
}
}
static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) {
tExprNode *pLeft = pExpr->_node.pLeft;
tExprNode *pRight = pExpr->_node.pRight;
@ -649,32 +263,6 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p
return param->nodeFilterFn(pItem, pExpr->_node.info);
}
/**
* Apply the filter expression on non-indexed tag columns to each element in the result list, which is generated
* by filtering on indexed tag column. So the whole result set only needs to be iterated once to generate
* result that is satisfied to the filter expression, no matter how the filter expression consisting of.
*
* @param pExpr filter expression on non-indexed tag columns.
* @param pResult results from filter on the indexed tag column, which is usually the first tag column
* @param pSchema tag schemas
* @param fp filter callback function
*/
static UNUSED_FUNC void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) {
size_t size = taosArrayGetSize(pResult);
SArray* array = taosArrayInit(size, POINTER_BYTES);
for (int32_t i = 0; i < size; ++i) {
void *pItem = taosArrayGetP(pResult, i);
if (filterItem(pExpr, pItem, param)) {
taosArrayPush(array, &pItem);
}
}
taosArrayCopy(pResult, array);
taosArrayDestroy(array);
}
static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
@ -750,7 +338,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
//apply the hierarchical expression to every node in skiplist for find the qualified nodes
tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
#if 0
#if 0
/*
* (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here
*
@ -972,6 +560,7 @@ tExprNode* exprTreeFromBinary(const void* data, size_t size) {
if (size == 0) {
return NULL;
}
SBufferReader br = tbufInitReader(data, size, false);
return exprTreeFromBinaryImpl(&br);
}
@ -995,10 +584,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
left->pSchema = pSchema;
pSchema->type = TSDB_DATA_TYPE_BINARY;
pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
strcpy(pSchema->name, TSQL_TBNAME_L);
pSchema->colId = -1;
*pSchema = tscGetTbnameColumnSchema();
tExprNode* right = exception_calloc(1, sizeof(tExprNode));
expr->_node.pRight = right;

98
src/query/src/qExecutor.c
View File

@ -170,8 +170,6 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
tw->ekey -= 1;
}
#define GET_NEXT_TIMEWINDOW(_q, tw) getNextTimeWindow((_q), (tw))
#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
@ -827,7 +825,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) {
SQuery *pQuery = pRuntimeEnv->pQuery;
GET_NEXT_TIMEWINDOW(pQuery, pNext);
getNextTimeWindow(pQuery, pNext);
// next time window is not in current block
if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@ -1342,9 +1340,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
int32_t index = pWindowResInfo->curIndex;
while (1) {
GET_NEXT_TIMEWINDOW(pQuery, &nextWin);
getNextTimeWindow(pQuery, &nextWin);
if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
(nextWin.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
break;
}
@ -2202,7 +2200,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
}
while(1) {
GET_NEXT_TIMEWINDOW(pQuery, &w);
getNextTimeWindow(pQuery, &w);
if (w.skey > pBlockInfo->window.ekey) {
break;
}
@ -2221,7 +2219,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
}
while(1) {
GET_NEXT_TIMEWINDOW(pQuery, &w);
getNextTimeWindow(pQuery, &w);
if (w.ekey < pBlockInfo->window.skey) {
break;
}
@ -2536,7 +2534,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
setQueryStatus(pQuery, QUERY_COMPLETED);
}
if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) {
if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) {
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window
@ -4116,7 +4114,7 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) {
* first result row in the actual result set will fill nothing.
*/
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
int32_t numOfTotal = (int32_t)getFilledNumOfRes(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity);
int32_t numOfTotal = (int32_t)getNumOfResWithFill(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity);
return numOfTotal > 0;
}
@ -4174,7 +4172,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
while (1) {
int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity);
int32_t ret = (int32_t)taosFillResultDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity);
// todo apply limit output function
/* reached the start position of according to offset value, return immediately */
@ -4354,14 +4352,17 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery);
while (pQuery->limit.offset > 0) {
STimeWindow tw = win;
if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
pQuery->limit.offset -= 1;
pWindowResInfo->prevSKey = win.skey;
}
STimeWindow tw = win;
GET_NEXT_TIMEWINDOW(pQuery, &tw);
getNextTimeWindow(pQuery, &tw);
} else { // current window does not ended in current data block, try next data block
getNextTimeWindow(pQuery, &tw);
}
if (pQuery->limit.offset == 0) {
if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@ -4414,16 +4415,60 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
tw = win;
int32_t startPos =
getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
assert(startPos >= 0);
if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
pQuery->limit.offset -= 1;
}
// set the abort info
pQuery->pos = startPos;
pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
pWindowResInfo->prevSKey = tw.skey;
win = tw;
if (pQuery->limit.offset == 0) {
if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
// load the data block and check data remaining in current data block
// TODO optimize performance
SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
tw = win;
int32_t startPos =
getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
assert(startPos >= 0);
// set the abort info
pQuery->pos = startPos;
// reset the query start timestamp
pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
pQuery->window.skey = pTableQueryInfo->win.skey;
*start = pTableQueryInfo->win.skey;
pWindowResInfo->prevSKey = tw.skey;
int32_t index = pRuntimeEnv->windowResInfo.curIndex;
int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index
qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64,
GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey);
return true;
} else { // do nothing
*start = tw.skey;
pQuery->window.skey = tw.skey;
pWindowResInfo->prevSKey = tw.skey;
return true;
}
} else {
tw = win;
int32_t startPos =
getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
assert(startPos >= 0);
// set the abort info
pQuery->pos = startPos;
pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
pWindowResInfo->prevSKey = tw.skey;
win = tw;
}
} else {
break; // offset is not 0, and next time window begins or ends in the next block.
}
@ -4519,6 +4564,7 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
pFillCol[i].col.bytes = pExprInfo->bytes;
pFillCol[i].col.type = (int8_t)pExprInfo->type;
pFillCol[i].col.offset = offset;
pFillCol[i].tagIndex = -2;
pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query
pFillCol[i].functionId = pExprInfo->base.functionId;
pFillCol[i].fillVal.i = pQuery->fillVal[i];
@ -4532,7 +4578,6 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
int32_t code = TSDB_CODE_SUCCESS;
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery);
@ -4540,7 +4585,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
setScanLimitationByResultBuffer(pQuery);
code = setupQueryHandle(tsdb, pQInfo, isSTableQuery);
int32_t code = setupQueryHandle(tsdb, pQInfo, isSTableQuery);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -4628,7 +4673,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, pQuery->numOfOutput,
pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision,
pQuery->fillType, pColInfo);
pQuery->fillType, pColInfo, pQInfo);
}
setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
@ -5430,7 +5475,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
break;
} else {
taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey);
taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata);
numOfFilled = 0;
pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled);
@ -6926,7 +6971,6 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
// here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
doFreeQueryHandle(pQInfo);
*continueExec = false;
(*pRsp)->completed = 1; // notify no more result to client
} else {

571
src/query/src/qFill.c
View File

@ -13,31 +13,80 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qFill.h"
#include "os.h"
#include "qExtbuffer.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tsqlfunction.h"
#include "ttype.h"
#include "qFill.h"
#include "qExtbuffer.h"
#include "queryLog.h"
#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
// there are no duplicated tags in the SFillTagColInfo list
static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
int32_t rowsize = 0;
int32_t k = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity);
if (TSDB_COL_IS_TAG(pColInfo->flag)) {
bool exists = false;
int32_t index = -1;
for (int32_t j = 0; j < k; ++j) {
if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
exists = true;
index = j;
break;
}
}
if (!exists) {
SSchema* pSchema = &pFillInfo->pTags[k].col;
pSchema->colId = pColInfo->col.colId;
pSchema->type = pColInfo->col.type;
pSchema->bytes = pColInfo->col.bytes;
pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
pColInfo->tagIndex = k;
k += 1;
} else {
pColInfo->tagIndex = index;
}
}
rowsize += pColInfo->col.bytes;
}
assert(k <= pFillInfo->numOfTags);
return rowsize;
}
SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol) {
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pCol, void* handle) {
if (fillType == TSDB_FILL_NONE) {
return NULL;
}
SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
taosResetFillInfo(pFillInfo, skey);
pFillInfo->order = order;
pFillInfo->fillType = fillType;
pFillInfo->pFillCol = pFillCol;
pFillInfo->order = order;
pFillInfo->type = fillType;
pFillInfo->pFillCol = pCol;
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->alloc = capacity;
pFillInfo->handle = handle;
pFillInfo->interval.interval = slidingTime;
pFillInfo->interval.intervalUnit = slidingUnit;
@ -47,45 +96,21 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
if (numOfTags > 0) {
pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo));
for(int32_t i = 0; i < numOfTags; ++i) {
pFillInfo->pTags[i].col.colId = -2;
for (int32_t i = 0; i < numOfTags; ++i) {
pFillInfo->pTags[i].col.colId = -2; // TODO
}
}
int32_t rowsize = 0;
int32_t k = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity);
pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
assert(pFillInfo->rowSize > 0);
if (TSDB_COL_IS_TAG(pColInfo->flag)) {
bool exists = false;
for(int32_t j = 0; j < k; ++j) {
if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
exists = true;
break;
}
}
if (!exists) {
pFillInfo->pTags[k].col.colId = pColInfo->col.colId;
pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
k += 1;
}
}
rowsize += pColInfo->col.bytes;
}
pFillInfo->rowSize = rowsize;
pFillInfo->capacityInRows = capacity;
return pFillInfo;
}
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->start = startTimestamp;
pFillInfo->rowIdx = -1;
pFillInfo->currentKey = startTimestamp;
pFillInfo->index = -1;
pFillInfo->numOfRows = 0;
pFillInfo->numOfCurrent = 0;
pFillInfo->numOfTotal = 0;
@ -112,20 +137,20 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
if (pFillInfo->fillType == TSDB_FILL_NONE) {
if (pFillInfo->type == TSDB_FILL_NONE) {
return;
}
pFillInfo->endKey = endKey;
if (pFillInfo->order != TSDB_ORDER_ASC) {
pFillInfo->endKey = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
pFillInfo->end = endKey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
}
pFillInfo->rowIdx = 0;
pFillInfo->index = 0;
pFillInfo->numOfRows = numOfRows;
// ensure the space
if (pFillInfo->capacityInRows < numOfRows) {
if (pFillInfo->alloc < numOfRows) {
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes);
assert(tmp != NULL); // todo handle error
@ -136,42 +161,38 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
}
}
void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput) {
// copy the data into source data buffer
// copy the data into source data buffer
void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) {
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
}
}
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput) {
void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) {
assert(pFillInfo->numOfRows == pInput->num);
for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* data = pInput->data + pCol->col.offset * pInput->num;
const char* data = pInput->data + pCol->col.offset * pInput->num;
memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes));
if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer
for (int32_t j = 0; j < pFillInfo->numOfTags; ++j) {
SFillTagColInfo* pTag = &pFillInfo->pTags[j];
if (pTag->col.colId == pCol->col.colId) {
memcpy(pTag->tagVal, data, pCol->col.bytes);
break;
}
}
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
memcpy(pTag->tagVal, data, pCol->col.bytes);
}
}
}
int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t getNumOfResWithFill(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t* tsList = (int64_t*) pFillInfo->pData[0];
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
int64_t numOfRes = -1;
@ -179,20 +200,20 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = taosTimeCountInterval(
lastKey,
pFillInfo->start,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
numOfRes = taosTimeCountInterval(
ekey1,
pFillInfo->start,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
@ -203,315 +224,283 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
}
int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
if (pFillInfo->rowIdx == -1 || pFillInfo->numOfRows == 0) {
if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
return 0;
}
return pFillInfo->numOfRows - pFillInfo->rowIdx;
return pFillInfo->numOfRows - pFillInfo->index;
}
// todo: refactor
static double linearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
}
#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
switch (type) {
case TSDB_DATA_TYPE_INT: {
*(int32_t*)point->val = (int32_t)linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, (double)point1->key,
(double)point2->key, (double)point->key);
break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(float*)point->val = (float)
linearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
break;
};
case TSDB_DATA_TYPE_DOUBLE: {
*(double*)point->val =
linearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
break;
};
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_BIGINT: {
*(int64_t*)point->val = (int64_t)linearInterpolationImpl((double)(*(int64_t*)point1->val), (double)(*(int64_t*)point2->val), (double)point1->key,
(double)point2->key, (double)point->key);
break;
};
case TSDB_DATA_TYPE_SMALLINT: {
*(int16_t*)point->val = (int16_t)linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, (double)point1->key,
(double)point2->key, (double)point->key);
break;
};
case TSDB_DATA_TYPE_TINYINT: {
*(int8_t*) point->val = (int8_t)
linearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
break;
};
default: {
// TODO: Deal with interpolation with bool and strings and timestamp
return -1;
}
double v1 = -1;
double v2 = -1;
GET_TYPED_DATA(v1, double, type, point1->val);
GET_TYPED_DATA(v2, double, type, point2->val);
double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
switch(type) {
case TSDB_DATA_TYPE_TINYINT: *(int8_t*) point->val = (int8_t) r;break;
case TSDB_DATA_TYPE_SMALLINT: *(int16_t*) point->val = (int16_t) r;break;
case TSDB_DATA_TYPE_INT: *(int32_t*) point->val = (int32_t) r;break;
case TSDB_DATA_TYPE_BIGINT: *(int64_t*) point->val = (int64_t) r;break;
case TSDB_DATA_TYPE_DOUBLE: *(double*) point->val = (double) r;break;
case TSDB_DATA_TYPE_FLOAT: *(float*) point->val = (float) r;break;
default:
assert(0);
}
return 0;
return TSDB_CODE_SUCCESS;
}
static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t num) {
static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) {
for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
SFillColInfo* pCol = &pFillInfo->pFillCol[j];
if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) {
continue;
}
char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, num);
char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, genRows);
for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
SFillTagColInfo* pTag = &pFillInfo->pTags[i];
if (pTag->col.colId == pCol->col.colId) {
assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
break;
}
}
assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
}
}
static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* num, char** srcData, int64_t ts,
bool outOfBound) {
char* prevValues = pFillInfo->prevValues;
char* nextValues = pFillInfo->nextValues;
static void setNullValueForRow(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfCol, int32_t rowIndex) {
// the first are always the timestamp column, so start from the second column.
for (int32_t i = 1; i < numOfCol; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* output = elePtrAt(data[i]->data, pCol->col.bytes, rowIndex);
setNull(output, pCol->col.type, pCol->col.bytes);
}
}
static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** srcData, int64_t ts, bool outOfBound) {
char* prev = pFillInfo->prevValues;
char* next = pFillInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, *num);
*(TSKEY*) val = pFillInfo->start;
int32_t numOfValCols = pFillInfo->numOfCols - pFillInfo->numOfTags;
// set the primary timestamp column value
int32_t index = pFillInfo->numOfCurrent;
char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, index);
*(TSKEY*) val = pFillInfo->currentKey;
// set the other values
if (pFillInfo->fillType == TSDB_FILL_PREV) {
char* p = FILL_IS_ASC_FILL(pFillInfo) ? prevValues : nextValues;
if (pFillInfo->type == TSDB_FILL_PREV) {
char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
if (p != NULL) {
for (int32_t i = 1; i < numOfValCols; ++i) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (isNull(p + pCol->col.offset, pCol->col.type)) {
if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(val1, pCol->col.type);
} else {
setNull(val1, pCol->col.type, pCol->col.bytes);
}
} else {
assignVal(val1, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i]->data, pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(val1, pCol->col.type);
} else {
setNull(val1, pCol->col.type, pCol->col.bytes);
}
}
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
setTagsValue(pFillInfo, data, *num);
} else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
// TODO : linear interpolation supports NULL value
if (prevValues != NULL && !outOfBound) {
for (int32_t i = 1; i < numOfValCols; ++i) {
if (prev != NULL && !outOfBound) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
int16_t type = pCol->col.type;
int16_t bytes = pCol->col.bytes;
char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(val1, pCol->col.type);
continue;
} else if (type == TSDB_DATA_TYPE_BOOL) {
char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, pCol->col.type, bytes);
continue;
}
point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
point = (SPoint){.key = pFillInfo->start, .val = val1};
taosGetLinearInterpolationVal(type, &point1, &point2, &point);
}
setTagsValue(pFillInfo, data, *num);
} else {
for (int32_t i = 1; i < numOfValCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(val1, pCol->col.type);
} else {
setNull(val1, pCol->col.type, pCol->col.bytes);
}
}
setTagsValue(pFillInfo, data, *num);
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else { /* fill the default value */
for (int32_t i = 1; i < numOfValCols; ++i) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
setTagsValue(pFillInfo, data, *num);
}
pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
setTagsValue(pFillInfo, data, index);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->numOfCurrent++;
(*num) += 1;
}
static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** nextValues) {
if (*nextValues != NULL) {
static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
if (*next != NULL) {
return;
}
*nextValues = calloc(1, pFillInfo->rowSize);
*next = calloc(1, pFillInfo->rowSize);
for (int i = 1; i < pFillInfo->numOfCols; i++) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(*nextValues + pCol->col.offset, pCol->col.type);
} else {
setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
}
setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
}
}
int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
int32_t num = 0;
static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
int32_t rowIndex = pFillInfo->index;
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
}
}
static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t outputRows) {
pFillInfo->numOfCurrent = 0;
char** prevValues = &pFillInfo->prevValues;
char** nextValues = &pFillInfo->nextValues;
char** srcData = pFillInfo->pData;
char** prev = &pFillInfo->prevValues;
char** next = &pFillInfo->nextValues;
int32_t numOfTags = pFillInfo->numOfTags;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
if (numOfRows == 0) {
/*
* These data are generated according to fill strategy, since the current timestamp is out of time window of
* real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
*/
while (num < outputRows) {
doFillResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, true);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
if (FILL_IS_ASC_FILL(pFillInfo)) {
assert(pFillInfo->currentKey >= pFillInfo->start);
} else {
while (1) {
int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->rowIdx];
assert(pFillInfo->currentKey <= pFillInfo->start);
}
if ((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
/* set the next value for interpolation */
initBeforeAfterDataBuf(pFillInfo, nextValues);
int32_t offset = pFillInfo->rowIdx;
for (int32_t i = 0; i < pFillInfo->numOfCols - numOfTags; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
memcpy(*nextValues + pCol->col.offset, srcData[i] + offset * pCol->col.bytes, pCol->col.bytes);
}
while (pFillInfo->numOfCurrent < outputRows) {
int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
/* set the next value for interpolation */
initBeforeAfterDataBuf(pFillInfo, next);
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
}
if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
// fill the gap between two actual input rows
while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
doFillOneRowResult(pFillInfo, data, srcData, ts, false);
}
if (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
while (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
doFillResultImpl(pFillInfo, data, &num, srcData, ts, false);
}
/* output buffer is full, abort */
if ((num == outputRows && FILL_IS_ASC_FILL(pFillInfo)) || (num < 0 && !FILL_IS_ASC_FILL(pFillInfo))) {
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
}
} else {
assert(pFillInfo->start == ts);
initBeforeAfterDataBuf(pFillInfo, prevValues);
// assign rows to dst buffer
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->rowIdx);
if (i == 0 ||
(pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
(pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
assignVal(val1, src, pCol->col.bytes, pCol->col.type);
memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
} else { // i > 0 and data is null , do interpolation
if (pFillInfo->fillType == TSDB_FILL_PREV) {
assignVal(val1, *prevValues + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
assignVal(val1, src, pCol->col.bytes, pCol->col.type);
memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
} else {
assignVal(val1, (char*) &pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
}
// set the tag value for final result
setTagsValue(pFillInfo, data, num);
pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->rowIdx += 1;
pFillInfo->numOfCurrent +=1;
num += 1;
}
if ((pFillInfo->rowIdx >= pFillInfo->numOfRows && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->rowIdx < 0 && !FILL_IS_ASC_FILL(pFillInfo)) || num >= outputRows) {
if (pFillInfo->rowIdx >= pFillInfo->numOfRows || pFillInfo->rowIdx < 0) {
pFillInfo->rowIdx = -1;
pFillInfo->numOfRows = 0;
/* the raw data block is exhausted, next value does not exists */
tfree(*nextValues);
}
// output buffer is full, abort
if (pFillInfo->numOfCurrent == outputRows) {
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return num;
return outputRows;
}
} else {
assert(pFillInfo->currentKey == ts);
initBeforeAfterDataBuf(pFillInfo, prev);
// assign rows to dst buffer
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i]->data, pCol->col.bytes, pFillInfo->numOfCurrent);
char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
(pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else { // i > 0 and data is null , do interpolation
if (pFillInfo->type == TSDB_FILL_PREV) {
assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else {
assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
}
// set the tag value for final result
setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->index += 1;
pFillInfo->numOfCurrent += 1;
}
if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
/* the raw data block is exhausted, next value does not exists */
if (pFillInfo->index >= pFillInfo->numOfRows) {
tfree(*next);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return pFillInfo->numOfCurrent;
}
}
return pFillInfo->numOfCurrent;
}
int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo); // todo use iterator?
static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) {
/*
* These data are generated according to fill strategy, since the current timestamp is out of the time window of
* real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
*/
pFillInfo->numOfCurrent = 0;
while (pFillInfo->numOfCurrent < resultCapacity) {
doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
assert(pFillInfo->numOfCurrent == resultCapacity);
return resultCapacity;
}
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
int64_t numOfRes = getNumOfResWithFill(pFillInfo, pFillInfo->end, capacity);
assert(numOfRes <= capacity);
// no data existed for fill operation now, append result according to the fill strategy
if (remain == 0) {
fillExternalResults(pFillInfo, output, numOfRes);
} else {
fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
assert(numOfRes == pFillInfo->numOfCurrent);
}
qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
pFillInfo->numOfTotal, pFillInfo->handle);
int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, pFillInfo->endKey, capacity);
int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
assert(numOfRes == rows);
return numOfRes;
}

206
src/query/src/qParserImpl.c
View File

@ -15,11 +15,9 @@
#include "os.h"
#include "qSqlparser.h"
#include "queryLog.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tcmdtype.h"
#include "tglobal.h"
#include "tstoken.h"
#include "tstrbuild.h"
#include "ttokendef.h"
@ -227,13 +225,12 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
tSQLExprDestroy(pLeft);
tSQLExprDestroy(pRight);
} else if ((pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE && pRight->val.nType == TSDB_DATA_TYPE_BIGINT) ||
(pRight->val.nType == TSDB_DATA_TYPE_DOUBLE && pLeft->val.nType == TSDB_DATA_TYPE_BIGINT)) {
} else if (pLeft->nSQLOptr == TK_FLOAT || pRight->nSQLOptr == TK_FLOAT) {
pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE;
pExpr->nSQLOptr = TK_FLOAT;
pExpr->nSQLOptr = TK_FLOAT;
double left = pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE ? pLeft->val.dKey : pLeft->val.i64Key;
double right = pRight->val.nType == TSDB_DATA_TYPE_DOUBLE ? pRight->val.dKey : pRight->val.i64Key;
double left = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64Key;
double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64Key;
switch (optrType) {
case TK_PLUS: {
@ -314,130 +311,57 @@ void tSQLExprDestroy(tSQLExpr *pExpr) {
tSQLExprNodeDestroy(pExpr);
}
static void *tVariantListExpand(tVariantList *pList) {
if (pList->nAlloc <= pList->nExpr) { //
int32_t newSize = (pList->nAlloc << 1) + 4;
void *ptr = realloc(pList->a, newSize * sizeof(pList->a[0]));
if (ptr == 0) {
return NULL;
}
pList->nAlloc = newSize;
pList->a = ptr;
}
assert(pList->a != 0);
return pList;
}
tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder) {
SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) {
if (pList == NULL) {
pList = calloc(1, sizeof(tVariantList));
}
if (tVariantListExpand(pList) == NULL) {
return pList;
}
if (pVar) {
tVariantListItem *pItem = &pList->a[pList->nExpr++];
/*
* Here we do not employ the assign function, since we need the pz attribute of structure
* , which is the point to char string, to free it!
*
* Otherwise, the original pointer may be lost, which causes memory leak.
*/
memcpy(pItem, pVar, sizeof(tVariant));
pItem->sortOrder = sortOrder;
}
return pList;
}
tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) {
if (pList == NULL || index >= pList->nExpr) {
return tVariantListAppend(NULL, pVar, sortOrder);
}
if (tVariantListExpand(pList) == NULL) {
return pList;
}
if (pVar) {
memmove(&pList->a[index + 1], &pList->a[index], sizeof(tVariantListItem) * (pList->nExpr - index));
tVariantListItem *pItem = &pList->a[index];
/*
* Here we do not employ the assign function, since we need the pz attribute of structure
* , which is the point to char string, to free it!
*
* Otherwise, the original pointer may be lost, which causes memory leak.
*/
memcpy(pItem, pVar, sizeof(tVariant));
pItem->sortOrder = sortOrder;
pList->nExpr++;
}
return pList;
}
void tVariantListDestroy(tVariantList *pList) {
if (pList == NULL) return;
for (int32_t i = 0; i < pList->nExpr; ++i) {
tVariantDestroy(&pList->a[i].pVar);
}
free(pList->a);
free(pList);
}
tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) {
if (pList == NULL) {
pList = calloc(1, sizeof(tVariantList));
}
if (tVariantListExpand(pList) == NULL) {
return pList;
pList = taosArrayInit(4, sizeof(tVariantListItem));
}
if (pToken) {
tVariant t = {0};
tVariantCreate(&t, pToken);
tVariantListItem item;
tVariantCreate(&item.pVar, pToken);
item.sortOrder = order;
tVariantListItem *pItem = &pList->a[pList->nExpr++];
memcpy(pItem, &t, sizeof(tVariant));
pItem->sortOrder = sortOrder;
taosArrayPush(pList, &item);
}
return pList;
}
tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField) {
if (pList == NULL) pList = calloc(1, sizeof(tFieldList));
if (pList->nAlloc <= pList->nField) { //
pList->nAlloc = (pList->nAlloc << 1) + 4;
pList->p = realloc(pList->p, pList->nAlloc * sizeof(pList->p[0]));
if (pList->p == 0) {
pList->nField = pList->nAlloc = 0;
return pList;
}
SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) {
if (pList == NULL) {
pList = taosArrayInit(4, sizeof(tVariantListItem));
}
assert(pList->p != 0);
if (pField) {
struct TAOS_FIELD *pItem = (struct TAOS_FIELD *)&pList->p[pList->nField++];
memcpy(pItem, pField, sizeof(TAOS_FIELD));
if (pVar == NULL) {
return pList;
}
/*
* Here we do not employ the assign function, since we need the pz attribute of structure
* , which is the point to char string, to free it!
*
* Otherwise, the original pointer may be lost, which causes memory leak.
*/
tVariantListItem item;
item.pVar = *pVar;
item.sortOrder = sortOrder;
taosArrayPush(pList, &item);
return pList;
}
void tFieldListDestroy(tFieldList *pList) {
if (pList == NULL) return;
SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) {
if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) {
return tVariantListAppend(NULL, pVar, sortOrder);
}
free(pList->p);
free(pList);
tVariantListItem item;
item.pVar = *pVar;
item.sortOrder = sortOrder;
taosArrayInsert(pList, index, &item);
return pList;
}
void setDBName(SStrToken *pCpxName, SStrToken *pDB) {
@ -464,8 +388,6 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType)
void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
pField->type = -1;
int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar.
for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) {
if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) &&
(type->n == tDataTypeDesc[i].nameLen)) {
@ -481,14 +403,14 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
if (type->type == 0) {
pField->bytes = 0;
} else {
pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
}
} else if (i == TSDB_DATA_TYPE_BINARY) {
/* for binary, the TOKENTYPE is the length of binary */
if (type->type == 0) {
pField->bytes = 0;
} else {
pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE);
}
}
break;
@ -499,9 +421,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
/*
* extract the select info out of sql string
*/
SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
assert(pSelection != NULL);
SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL));
@ -535,6 +457,11 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection,
return pQuery;
}
void freeVariant(void *pItem) {
tVariantListItem* p = (tVariantListItem*) pItem;
tVariantDestroy(&p->pVar);
}
void doDestroyQuerySql(SQuerySQL *pQuerySql) {
if (pQuerySql == NULL) {
return;
@ -547,17 +474,18 @@ void doDestroyQuerySql(SQuerySQL *pQuerySql) {
tSQLExprDestroy(pQuerySql->pWhere);
pQuerySql->pWhere = NULL;
tVariantListDestroy(pQuerySql->pSortOrder);
taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant);
pQuerySql->pSortOrder = NULL;
tVariantListDestroy(pQuerySql->pGroupby);
taosArrayDestroyEx(pQuerySql->pGroupby, freeVariant);
pQuerySql->pGroupby = NULL;
tVariantListDestroy(pQuerySql->from);
taosArrayDestroyEx(pQuerySql->from, freeVariant);
pQuerySql->from = NULL;
tVariantListDestroy(pQuerySql->fillType);
taosArrayDestroyEx(pQuerySql->fillType, freeVariant);
pQuerySql->fillType = NULL;
free(pQuerySql);
}
@ -574,8 +502,8 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) {
tfree(pClause->pClause);
}
SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName,
tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) {
SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pStableName,
SArray *pTagVals, SQuerySQL *pSelect, int32_t type) {
SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL));
switch (type) {
@ -607,7 +535,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrTo
return pCreate;
}
SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) {
SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type) {
SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL));
pAlterTable->name = *pMeterName;
@ -637,14 +565,14 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo;
doDestroyQuerySql(pCreateTableInfo->pSelect);
tFieldListDestroy(pCreateTableInfo->colInfo.pColumns);
tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns);
taosArrayDestroy(pCreateTableInfo->colInfo.pColumns);
taosArrayDestroy(pCreateTableInfo->colInfo.pTagColumns);
tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals);
taosArrayDestroyEx(pCreateTableInfo->usingInfo.pTagVals, freeVariant);
tfree(pInfo->pCreateTableInfo);
} else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
tVariantListDestroy(pInfo->pAlterInfo->varList);
tFieldListDestroy(pInfo->pAlterInfo->pAddColumns);
taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant);
taosArrayDestroy(pInfo->pAlterInfo->pAddColumns);
tfree(pInfo->pAlterInfo);
} else {
@ -653,7 +581,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
}
if (pInfo->pDCLInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) {
tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep);
taosArrayDestroyEx(pInfo->pDCLInfo->dbOpt.keep, freeVariant);
}
tfree(pInfo->pDCLInfo);

119
src/query/src/sql.c
View File

@ -107,6 +107,7 @@ typedef union {
tSQLExpr* yy64;
tVariant yy134;
SCreateAcctSQL yy149;
SArray* yy165;
int64_t yy207;
SLimitVal yy216;
TAOS_FIELD yy223;
@ -115,8 +116,6 @@ typedef union {
tSQLExprList* yy290;
SQuerySQL* yy414;
SCreateTableSQL* yy470;
tVariantList* yy498;
tFieldList* yy523;
SIntervalVal yy532;
} YYMINORTYPE;
#ifndef YYSTACKDEPTH
@ -1368,18 +1367,14 @@ static void yy_destructor(
/********* Begin destructor definitions ***************************************/
case 227: /* keep */
case 228: /* tagitemlist */
case 245: /* columnlist */
case 253: /* fill_opt */
case 255: /* groupby_opt */
case 256: /* orderby_opt */
case 266: /* sortlist */
case 270: /* grouplist */
{
tVariantListDestroy((yypminor->yy498));
}
break;
case 245: /* columnlist */
{
tFieldListDestroy((yypminor->yy523));
taosArrayDestroy((yypminor->yy165));
}
break;
case 246: /* select */
@ -2238,7 +2233,7 @@ static void yy_reduce(
yymsp[-8].minor.yy149 = yylhsminor.yy149;
break;
case 72: /* keep ::= KEEP tagitemlist */
{ yymsp[-1].minor.yy498 = yymsp[0].minor.yy498; }
{ yymsp[-1].minor.yy165 = yymsp[0].minor.yy165; }
break;
case 73: /* cache ::= CACHE INTEGER */
case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74);
@ -2314,7 +2309,7 @@ static void yy_reduce(
break;
case 99: /* db_optr ::= db_optr keep */
case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104);
{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy498; }
{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy165; }
yymsp[-1].minor.yy268 = yylhsminor.yy268;
break;
case 100: /* db_optr ::= db_optr update */
@ -2362,20 +2357,20 @@ static void yy_reduce(
break;
case 116: /* create_table_args ::= LP columnlist RP */
{
yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy165, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
{
yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE);
yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, NULL, NULL, TSQL_CREATE_STABLE);
setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
{
yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy165, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
}
break;
@ -2386,12 +2381,12 @@ static void yy_reduce(
}
break;
case 120: /* columnlist ::= columnlist COMMA column */
{yylhsminor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223); }
yymsp[-2].minor.yy523 = yylhsminor.yy523;
{taosArrayPush(yymsp[-2].minor.yy165, &yymsp[0].minor.yy223); yylhsminor.yy165 = yymsp[-2].minor.yy165; }
yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
case 121: /* columnlist ::= column */
{yylhsminor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);}
yymsp[0].minor.yy523 = yylhsminor.yy523;
{yylhsminor.yy165 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy165, &yymsp[0].minor.yy223);}
yymsp[0].minor.yy165 = yylhsminor.yy165;
break;
case 122: /* column ::= ids typename */
{
@ -2400,12 +2395,12 @@ static void yy_reduce(
yymsp[-1].minor.yy223 = yylhsminor.yy223;
break;
case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */
{ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); }
yymsp[-2].minor.yy498 = yylhsminor.yy498;
{ yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1); }
yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
case 124: /* tagitemlist ::= tagitem */
{ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
yymsp[0].minor.yy498 = yylhsminor.yy498;
{ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
yymsp[0].minor.yy165 = yylhsminor.yy165;
break;
case 125: /* tagitem ::= INTEGER */
case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126);
@ -2432,7 +2427,7 @@ static void yy_reduce(
break;
case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
{
yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy165, yymsp[-8].minor.yy64, yymsp[-4].minor.yy165, yymsp[-3].minor.yy165, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy165, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
}
yymsp[-11].minor.yy414 = yylhsminor.yy414;
break;
@ -2491,45 +2486,45 @@ static void yy_reduce(
{ yymsp[1].minor.yy0.n = 0; }
break;
case 148: /* from ::= FROM tablelist */
{yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;}
{yymsp[-1].minor.yy165 = yymsp[0].minor.yy165;}
break;
case 149: /* tablelist ::= ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name
yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1); // table alias name
}
yymsp[-1].minor.yy498 = yylhsminor.yy498;
yymsp[-1].minor.yy165 = yylhsminor.yy165;
break;
case 150: /* tablelist ::= ids cpxName ids */
{
toTSDBType(yymsp[-2].minor.yy0.type);
toTSDBType(yymsp[0].minor.yy0.type);
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1);
}
yymsp[-2].minor.yy498 = yylhsminor.yy498;
yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
case 151: /* tablelist ::= tablelist COMMA ids cpxName */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1);
yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1);
}
yymsp[-3].minor.yy498 = yylhsminor.yy498;
yymsp[-3].minor.yy165 = yylhsminor.yy165;
break;
case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */
{
toTSDBType(yymsp[-2].minor.yy0.type);
toTSDBType(yymsp[0].minor.yy0.type);
yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
yylhsminor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1);
yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(yymsp[-4].minor.yy165, &yymsp[-2].minor.yy0, -1);
yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1);
}
yymsp[-4].minor.yy498 = yylhsminor.yy498;
yymsp[-4].minor.yy165 = yylhsminor.yy165;
break;
case 153: /* tmvar ::= VARIABLE */
{yylhsminor.yy0 = yymsp[0].minor.yy0;}
@ -2545,7 +2540,7 @@ static void yy_reduce(
{memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));}
break;
case 157: /* fill_opt ::= */
{yymsp[1].minor.yy498 = 0; }
{yymsp[1].minor.yy165 = 0; }
break;
case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
{
@ -2553,14 +2548,14 @@ static void yy_reduce(
toTSDBType(yymsp[-3].minor.yy0.type);
tVariantCreate(&A, &yymsp[-3].minor.yy0);
tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0);
yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498;
tVariantListInsert(yymsp[-1].minor.yy165, &A, -1, 0);
yymsp[-5].minor.yy165 = yymsp[-1].minor.yy165;
}
break;
case 159: /* fill_opt ::= FILL LP ID RP */
{
toTSDBType(yymsp[-1].minor.yy0.type);
yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
yymsp[-3].minor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
}
break;
case 160: /* sliding_opt ::= SLIDING LP tmvar RP */
@ -2571,23 +2566,23 @@ static void yy_reduce(
break;
case 162: /* orderby_opt ::= */
case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170);
{yymsp[1].minor.yy498 = 0;}
{yymsp[1].minor.yy165 = 0;}
break;
case 163: /* orderby_opt ::= ORDER BY sortlist */
case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171);
{yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;}
{yymsp[-2].minor.yy165 = yymsp[0].minor.yy165;}
break;
case 164: /* sortlist ::= sortlist COMMA item sortorder */
{
yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
yylhsminor.yy165 = tVariantListAppend(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
}
yymsp[-3].minor.yy498 = yylhsminor.yy498;
yymsp[-3].minor.yy165 = yylhsminor.yy165;
break;
case 165: /* sortlist ::= item sortorder */
{
yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
}
yymsp[-1].minor.yy498 = yylhsminor.yy498;
yymsp[-1].minor.yy165 = yylhsminor.yy165;
break;
case 166: /* item ::= ids cpxName */
{
@ -2609,15 +2604,15 @@ static void yy_reduce(
break;
case 172: /* grouplist ::= grouplist COMMA item */
{
yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);
yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1);
}
yymsp[-2].minor.yy498 = yylhsminor.yy498;
yymsp[-2].minor.yy165 = yylhsminor.yy165;
break;
case 173: /* grouplist ::= item */
{
yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
}
yymsp[0].minor.yy498 = yylhsminor.yy498;
yymsp[0].minor.yy165 = yylhsminor.yy165;
break;
case 174: /* having_opt ::= */
case 184: /* where_opt ::= */ yytestcase(yyruleno==184);
@ -2633,15 +2628,21 @@ static void yy_reduce(
{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
break;
case 177: /* limit_opt ::= LIMIT signed */
case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181);
{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;}
break;
case 178: /* limit_opt ::= LIMIT signed OFFSET signed */
case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==182);
{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
break;
case 179: /* limit_opt ::= LIMIT signed COMMA signed */
case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==183);
{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
break;
case 181: /* slimit_opt ::= SLIMIT signed */
{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;}
break;
case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */
{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
break;
case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */
{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
break;
case 186: /* expr ::= LP expr RP */
@ -2787,7 +2788,7 @@ static void yy_reduce(
case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
@ -2796,7 +2797,7 @@ static void yy_reduce(
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
toTSDBType(yymsp[0].minor.yy0.type);
tVariantList* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@ -2805,7 +2806,7 @@ static void yy_reduce(
case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
{
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
}
break;
@ -2814,7 +2815,7 @@ static void yy_reduce(
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
toTSDBType(yymsp[0].minor.yy0.type);
tVariantList* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN);
setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@ -2825,7 +2826,7 @@ static void yy_reduce(
yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
toTSDBType(yymsp[-1].minor.yy0.type);
tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
toTSDBType(yymsp[0].minor.yy0.type);
A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1);
@ -2839,7 +2840,7 @@ static void yy_reduce(
yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
toTSDBType(yymsp[-2].minor.yy0.type);
tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1);
SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);
@ -3146,4 +3147,4 @@ void Parse(
}
#endif
return;
}
}

18
src/rpc/src/rpcMain.c
View File

@ -135,7 +135,7 @@ int tsRpcOverhead;
static int tsRpcRefId = -1;
static int32_t tsRpcNum = 0;
static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
// server:0 client:1 tcp:2 udp:0
#define RPC_CONN_UDPS 0
@ -221,13 +221,15 @@ static void rpcFree(void *p) {
free(p);
}
void rpcInit(void) {
int32_t rpcInit(void) {
tsProgressTimer = tsRpcTimer/2;
tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
tsRpcHeadSize = RPC_MSG_OVERHEAD;
tsRpcOverhead = sizeof(SRpcReqContext);
tsRpcRefId = taosOpenRef(200, rpcFree);
return 0;
}
void rpcCleanup(void) {
@ -238,7 +240,7 @@ void rpcCleanup(void) {
void *rpcOpen(const SRpcInit *pInit) {
SRpcInfo *pRpc;
pthread_once(&tsRpcInit, rpcInit);
//pthread_once(&tsRpcInit, rpcInit);
pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) return NULL;
@ -379,7 +381,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
}
int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {
SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext;
@ -405,14 +407,10 @@ int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
|| type == TSDB_MSG_TYPE_CM_SHOW )
pContext->connType = RPC_CONN_TCPC;
// set the handle to pContext, so app can cancel the request
if (pMsg->handle) *((void **)pMsg->handle) = pContext;
pContext->rid = taosAddRef(tsRpcRefId, pContext);
if (pRid) *pRid = pContext->rid;
rpcSendReqToServer(pRpc, pContext);
return pContext->rid;
}
void rpcSendResponse(const SRpcMsg *pRsp) {
@ -528,7 +526,7 @@ void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
pContext->pRsp = pRsp;
pContext->pSet = pEpSet;
rpcSendRequest(shandle, pEpSet, pMsg);
rpcSendRequest(shandle, pEpSet, pMsg, NULL);
tsem_wait(&sem);
tsem_destroy(&sem);

2
src/rpc/src/rpcTcp.c
View File

@ -373,8 +373,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d",
pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
} else {
taosCloseSocket(fd);
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
taosCloseSocket(fd);
}
return pFdObj;

2
src/rpc/test/rclient.c
View File

@ -57,7 +57,7 @@ static void *sendRequest(void *param) {
rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1;
tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if ( pInfo->num % 20000 == 0 )
tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
tsem_wait(&pInfo->rspSem);

32
src/sync/src/syncMain.c
View File

@ -498,7 +498,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs;
if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs);
taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
}
@ -575,6 +575,18 @@ static void syncChooseMaster(SSyncNode *pNode) {
if (index == pNode->selfIndex) {
sInfo("vgId:%d, start to work as master", pNode->vgId);
nodeRole = TAOS_SYNC_ROLE_MASTER;
#if 0
for (int32_t i = 0; i < pNode->replica; ++i) {
if (i == index) continue;
pPeer = pNode->peerInfo[i];
if (pPeer->version == nodeVersion) {
pPeer->role = TAOS_SYNC_ROLE_SLAVE;
pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
sInfo("%s, it shall work as slave", pPeer->id);
}
}
#endif
syncResetFlowCtrl(pNode);
(*pNode->notifyRole)(pNode->ahandle, nodeRole);
} else {
@ -1097,7 +1109,7 @@ static void syncProcessBrokenLink(void *param) {
SSyncPeer *pPeer = param;
SSyncNode *pNode = pPeer->pSyncNode;
if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return;
if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return;
pthread_mutex_lock(&(pNode->mutex));
sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@ -1209,13 +1221,17 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
int32_t fwdLen;
int32_t code = 0;
if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
pWalHead->version, nodeVersion);
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
syncRestartConnection(pPeer);
if (pWalHead->version > nodeVersion + 1) {
sError("vgId:%d, hver:%" PRIu64 ", inconsistent with ver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion);
if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
sInfo("vgId:%d, restart connection", pNode->vgId);
for (int32_t i = 0; i < pNode->replica; ++i) {
pPeer = pNode->peerInfo[i];
syncRestartConnection(pPeer);
}
}
return TSDB_CODE_SYN_INVALID_VERSION;
}

2
src/sync/src/syncRetrieve.c
View File

@ -182,6 +182,8 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEve
return 0;
}
assert(pHead->len <= TSDB_MAX_WAL_SIZE);
ret = read(sfd, pHead->cont, pHead->len);
if (ret < 0) return -1;

2
src/sync/src/taosTcpPool.c
View File

@ -327,5 +327,5 @@ static void taosStopPoolThread(SThreadObj *pThread) {
}
pthread_join(thread, NULL);
if (fd >= 0) taosClose(fd);
taosClose(fd);
}

2
src/sync/test/syncClient.c
View File

@ -57,7 +57,7 @@ void *sendRequest(void *param) {
rpcMsg.ahandle = pInfo;
rpcMsg.msgType = 1;
uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if (pInfo->num % 20000 == 0) {
uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
}

13
src/tsdb/inc/tsdbMain.h
View File

@ -208,6 +208,18 @@ typedef struct {
} SFileGroupIter;
// ------------------ tsdbMain.c
typedef struct {
int32_t totalLen;
int32_t len;
SDataRow row;
} SSubmitBlkIter;
typedef struct {
int32_t totalLen;
int32_t len;
void * pMsg;
} SSubmitMsgIter;
typedef struct {
int8_t state;
@ -430,7 +442,6 @@ void tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
// ------------------ tsdbMemTable.c
int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);

337
src/tsdb/src/tsdbCommit.c Normal file
View File

@ -0,0 +1,337 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdbMain.h"
static int tsdbCommitTSData(STsdbRepo *pRepo);
static int tsdbCommitMeta(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
void *tsdbCommitData(STsdbRepo *pRepo) {
SMemTable * pMem = pRepo->imem;
tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d",
REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList));
// Commit to update meta file
if (tsdbCommitMeta(pRepo) < 0) {
tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
// Create the iterator to read from cache
if (tsdbCommitTSData(pRepo) < 0) {
tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
tsdbFitRetention(pRepo);
tsdbInfo("vgId:%d commit over, succeed", REPO_ID(pRepo));
tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS);
return NULL;
_err:
ASSERT(terrno != TSDB_CODE_SUCCESS);
tsdbInfo("vgId:%d commit over, failed", REPO_ID(pRepo));
tsdbEndCommit(pRepo, terrno);
return NULL;
}
static int tsdbCommitTSData(STsdbRepo *pRepo) {
SMemTable * pMem = pRepo->imem;
SDataCols * pDataCols = NULL;
STsdbMeta * pMeta = pRepo->tsdbMeta;
SCommitIter *iters = NULL;
SRWHelper whelper = {0};
STsdbCfg * pCfg = &(pRepo->config);
if (pMem->numOfRows <= 0) return 0;
iters = tsdbCreateCommitIters(pRepo);
if (iters == NULL) {
tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
goto _err;
}
int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
// Loop to commit to each file
for (int fid = sfid; fid <= efid; fid++) {
if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _err;
}
}
tdFreeDataCols(pDataCols);
tsdbDestroyCommitIters(iters, pMem->maxTables);
tsdbDestroyHelper(&whelper);
return 0;
_err:
tdFreeDataCols(pDataCols);
tsdbDestroyCommitIters(iters, pMem->maxTables);
tsdbDestroyHelper(&whelper);
return -1;
}
static int tsdbCommitMeta(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
STsdbMeta *pMeta = pRepo->tsdbMeta;
SActObj * pAct = NULL;
SActCont * pCont = NULL;
if (listNEles(pMem->actList) <= 0) return 0;
if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
SListNode *pNode = NULL;
while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
pAct = (SActObj *)pNode->data;
if (pAct->act == TSDB_UPDATE_META) {
pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno));
tdKVStoreEndCommit(pMeta->pStore);
goto _err;
}
} else if (pAct->act == TSDB_DROP_META) {
if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno));
tdKVStoreEndCommit(pMeta->pStore);
goto _err;
}
} else {
ASSERT(false);
}
}
if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
return 0;
_err:
return -1;
}
static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
sem_post(&(pRepo->readyToCommit));
}
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
return 0;
}
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
char * dataDir = NULL;
STsdbCfg * pCfg = &pRepo->config;
STsdbFileH *pFileH = pRepo->tsdbFileH;
SFileGroup *pGroup = NULL;
SMemTable * pMem = pRepo->imem;
bool newLast = false;
TSKEY minKey = 0, maxKey = 0;
tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
// Check if there are data to commit to this file
int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
if (!hasDataToCommit) {
tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
return 0;
}
// Create and open files for commit
dataDir = tsdbGetDataDirName(pRepo->rootDir);
if (dataDir == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) {
tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _err;
}
// Open files for write/read
if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
newLast = TSDB_NLAST_FILE_OPENED(pHelper);
if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
// Loop to commit data in each table
for (int tid = 1; tid < pMem->maxTables; tid++) {
SCommitIter *pIter = iters + tid;
if (pIter->pTable == NULL) continue;
taosRLockLatch(&(pIter->pTable->latch));
if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
if (pIter->pIter != NULL) {
if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
taosRUnLockLatch(&(pIter->pTable->latch));
tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
tstrerror(terrno));
goto _err;
}
}
taosRUnLockLatch(&(pIter->pTable->latch));
// Move the last block to the new .l file if neccessary
if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
// Write the SCompBlock part
if (tsdbWriteCompInfo(pHelper) < 0) {
tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
}
if (tsdbWriteCompIdx(pHelper) < 0) {
tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _err;
}
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 0, pGroup);
pthread_rwlock_wrlock(&(pFileH->fhlock));
(void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
if (newLast) {
(void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
} else {
pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
}
pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
pthread_rwlock_unlock(&(pFileH->fhlock));
return 0;
_err:
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 1, NULL);
return -1;
}
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
STsdbMeta *pMeta = pRepo->tsdbMeta;
SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
if (iters == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
// reference all tables
for (int i = 0; i < pMem->maxTables; i++) {
if (pMeta->tables[i] != NULL) {
tsdbRefTable(pMeta->tables[i]);
iters[i].pTable = pMeta->tables[i];
}
}
if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
for (int i = 0; i < pMem->maxTables; i++) {
if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
tSkipListIterNext(iters[i].pIter);
}
}
return iters;
_err:
tsdbDestroyCommitIters(iters, pMem->maxTables);
return NULL;
}
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
if (iters == NULL) return;
for (int i = 1; i < maxTables; i++) {
if (iters[i].pTable != NULL) {
tsdbUnRefTable(iters[i].pTable);
tSkipListDestroyIter(iters[i].pIter);
}
}
free(iters);
}

5
src/tsdb/src/tsdbFile.c
View File

@ -256,7 +256,8 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
pFileH->pFGroup[pFileH->nFGroups++] = fGroup;
qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup);
pthread_rwlock_unlock(&pFileH->fhlock);
return tsdbSearchFGroup(pFileH, fid, TD_EQ);
pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
ASSERT(pGroup != NULL);
}
return pGroup;
@ -516,7 +517,7 @@ void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) {
SFile file;
SFile * pFile = &file;
strncpy(pFile->fname, fname, TSDB_FILENAME_LEN);
strncpy(pFile->fname, fname, TSDB_FILENAME_LEN - 1);
pFile->fd = -1;
if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;

282
src/tsdb/src/tsdbMain.c
View File

@ -32,18 +32,6 @@
#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
typedef struct {
int32_t totalLen;
int32_t len;
SDataRow row;
} SSubmitBlkIter;
typedef struct {
int32_t totalLen;
int32_t len;
void * pMsg;
} SSubmitMsgIter;
static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg);
static int32_t tsdbUnsetRepoEnv(char *rootDir);
@ -52,20 +40,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg);
static char * tsdbGetCfgFname(char *rootDir);
static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg);
static void tsdbFreeRepo(STsdbRepo *pRepo);
static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows);
static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
static int tsdbRestoreInfo(STsdbRepo *pRepo);
static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
static int keyFGroupCompFunc(const void *key, const void *fgroup);
static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg);
static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg);
static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
static void tsdbStartStream(STsdbRepo *pRepo);
static void tsdbStopStream(STsdbRepo *pRepo);
@ -177,40 +158,6 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
tsdbDebug("vgId:%d repository is closed", vgId);
}
int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
STsdbRepo * pRepo = (STsdbRepo *)repo;
SSubmitMsgIter msgIter = {0};
if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
}
return -1;
}
if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) {
tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
SSubmitBlk *pBlock = NULL;
int32_t affectedrows = 0;
TSKEY now = taosGetTimestamp(pRepo->config.precision);
while (true) {
tsdbGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
if (tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows) < 0) {
return -1;
}
}
if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
if (tsdbCheckCommit(pRepo) < 0) return -1;
return 0;
}
uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) {
STsdbRepo *pRepo = (STsdbRepo *)repo;
// STsdbMeta *pMeta = pRepo->tsdbMeta;
@ -294,6 +241,7 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) {
int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
// TODO: think about multithread cases
STsdbRepo *pRepo = (STsdbRepo *)repo;
STsdbCfg config = pRepo->config;
STsdbCfg * pRCfg = &pRepo->config;
if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1;
@ -308,22 +256,25 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
bool configChanged = false;
if (pRCfg->compression != pCfg->compression) {
tsdbAlterCompression(pRepo, pCfg->compression);
config.compression = pCfg->compression;
configChanged = true;
}
if (pRCfg->keep != pCfg->keep) {
if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) {
tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno));
config.keep = pCfg->keep;
return -1;
}
configChanged = true;
}
if (pRCfg->totalBlocks != pCfg->totalBlocks) {
tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
config.totalBlocks = pCfg->totalBlocks;
configChanged = true;
}
if (configChanged) {
if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) {
if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) {
tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
}
@ -731,93 +682,6 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) {
}
}
static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
if (pMsg == NULL) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1;
}
pIter->totalLen = pMsg->length;
pIter->len = 0;
pIter->pMsg = pMsg;
if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1;
}
return 0;
}
static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
int64_t points = 0;
ASSERT(pBlock->tid < pMeta->maxTables);
STable *pTable = pMeta->tables[pBlock->tid];
ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
SSubmitBlkIter blkIter = {0};
SDataRow row = NULL;
TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
tsdbInitSubmitBlkIter(pBlock, &blkIter);
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
" maxKey %" PRId64,
REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey);
terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
return -1;
}
if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
(*affectedrows)++;
points++;
}
STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
pRepo->stat.totalStorage += points * schemaVLen(pSchema);
return 0;
}
static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
if (pIter->len == 0) {
pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE;
} else {
SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
}
if (pIter->len > pIter->totalLen) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
*pPBlock = NULL;
return -1;
}
*pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
return 0;
}
static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
SDataRow row = pIter->row;
if (row == NULL) return NULL;
pIter->len += dataRowLen(row);
if (pIter->len >= pIter->totalLen) {
pIter->row = NULL;
} else {
pIter->row = (char *)row + dataRowLen(row);
}
return row;
}
static int tsdbRestoreInfo(STsdbRepo *pRepo) {
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbFileH *pFileH = pRepo->tsdbFileH;
@ -851,14 +715,6 @@ _err:
return -1;
}
static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
if (pBlock->dataLen <= 0) return -1;
pIter->totalLen = pBlock->dataLen;
pIter->len = 0;
pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
return 0;
}
static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
int8_t ocompression = pRepo->config.compression;
pRepo->config.compression = compression;
@ -955,134 +811,6 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
return buf;
}
static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
ASSERT(pTable != NULL);
STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
int sversion = schemaVersion(pSchema);
if (pBlock->sversion == sversion) {
return 0;
} else {
if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema
terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
return -1;
}
}
if (pBlock->sversion > sversion) { // may need to update table schema
if (pBlock->schemaLen > 0) {
tsdbDebug(
"vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
int numOfCols = pBlock->schemaLen / sizeof(STColumn);
STColumn *pTCol = (STColumn *)pBlock->data;
STSchemaBuilder schemaBuilder = {0};
if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
tstrerror(terrno));
return -1;
}
for (int i = 0; i < numOfCols; i++) {
if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
tstrerror(terrno));
tdDestroyTSchemaBuilder(&schemaBuilder);
return -1;
}
}
STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
if (pNSchema == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tdDestroyTSchemaBuilder(&schemaBuilder);
return -1;
}
tdDestroyTSchemaBuilder(&schemaBuilder);
tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
} else {
tsdbDebug(
"vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
return -1;
}
} else {
ASSERT(pBlock->sversion >= 0);
if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
}
terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
return -1;
}
return 0;
}
static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
ASSERT(pMsg != NULL);
STsdbMeta * pMeta = pRepo->tsdbMeta;
SSubmitMsgIter msgIter = {0};
SSubmitBlk * pBlock = NULL;
terrno = TSDB_CODE_SUCCESS;
pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
while (true) {
if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
if (pBlock == NULL) break;
pBlock->uid = htobe64(pBlock->uid);
pBlock->tid = htonl(pBlock->tid);
pBlock->sversion = htonl(pBlock->sversion);
pBlock->dataLen = htonl(pBlock->dataLen);
pBlock->schemaLen = htonl(pBlock->schemaLen);
pBlock->numOfRows = htons(pBlock->numOfRows);
if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
STable *pTable = pMeta->tables[pBlock->tid];
if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
// Check schema version and update schema if needed
if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
continue;
} else {
return -1;
}
}
}
if (terrno != TSDB_CODE_SUCCESS) return -1;
return 0;
}
static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
// TODO
// STsdbCache *pCache = pRepo->tsdbCache;

853
src/tsdb/src/tsdbMemTable.c
View File

@ -18,117 +18,56 @@
#define TSDB_DATA_SKIPLIST_LEVEL 5
static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes);
static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
static void tsdbFreeMemTable(SMemTable *pMemTable);
static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
static void tsdbFreeTableData(STableData *pTableData);
static char * tsdbGetTsTupleKey(const void *data);
static int tsdbCommitMeta(STsdbRepo *pRepo);
static void tsdbEndCommit(STsdbRepo *pRepo);
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow);
static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter);
static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter);
// ---------------- INTERNAL FUNCTIONS ----------------
int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
STsdbCfg * pCfg = &pRepo->config;
STsdbMeta * pMeta = pRepo->tsdbMeta;
TKEY tkey = dataRowTKey(row);
TSKEY key = dataRowKey(row);
SMemTable * pMemTable = pRepo->mem;
STableData *pTableData = NULL;
bool isRowDelete = TKEY_IS_DELETED(tkey);
static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
TSKEY now);
if (isRowDelete) {
if (!pCfg->update) {
tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
STsdbRepo * pRepo = (STsdbRepo *)repo;
SSubmitMsgIter msgIter = {0};
SSubmitBlk * pBlock = NULL;
int32_t affectedrows = 0;
if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
}
if (key > TABLE_LASTKEY(pTable)) {
tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
return 0;
}
}
void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
if (pRow == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
return -1;
}
dataRowCpy(pRow, row);
// Operations above may change pRepo->mem, retake those values
ASSERT(pRepo->mem != NULL);
pMemTable = pRepo->mem;
if (TABLE_TID(pTable) >= pMemTable->maxTables) {
if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
tsdbInitSubmitMsgIter(pMsg, &msgIter);
while (true) {
tsdbGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
if (tsdbInsertDataToTable(pRepo, pBlock, &affectedrows) < 0) {
return -1;
}
}
pTableData = pMemTable->tData[TABLE_TID(pTable)];
if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
if (pTableData != NULL) {
taosWLockLatch(&(pMemTable->latch));
pMemTable->tData[TABLE_TID(pTable)] = NULL;
tsdbFreeTableData(pTableData);
taosWUnLockLatch(&(pMemTable->latch));
}
pTableData = tsdbNewTableData(pCfg, pTable);
if (pTableData == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64
" to table %s while create new table data object since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
return -1;
}
pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
}
ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
int64_t oldSize = SL_SIZE(pTableData->pData);
if (tSkipListPut(pTableData->pData, pRow) == NULL) {
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
} else {
int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
if (isRowDelete) {
if (TABLE_LASTKEY(pTable) == key) {
// TODO: need to update table last key here (may from file)
}
} else {
if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
}
if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
if (pMemTable->keyLast < key) pMemTable->keyLast = key;
pMemTable->numOfRows += deltaSize;
if (pTableData->keyFirst > key) pTableData->keyFirst = key;
if (pTableData->keyLast < key) pTableData->keyLast = key;
pTableData->numOfRows += deltaSize;
}
tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
key);
if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
if (tsdbCheckCommit(pRepo) < 0) return -1;
return 0;
}
// ---------------- INTERNAL FUNCTIONS ----------------
int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
if (pMemTable == NULL) return 0;
int ref = T_REF_INC(pMemTable);
@ -152,7 +91,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
}
int code = pthread_cond_signal(&pBufPool->poolNotEmpty);
if (code != 0) {
tsdbUnlockRepo(pRepo);
if (tsdbUnlockRepo(pRepo) < 0) return -1;
tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
@ -189,6 +128,8 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) {
}
void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) {
tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
if (pMem != NULL) {
taosRUnLockLatch(&(pMem->latch));
tsdbUnRefMemTable(pRepo, pMem);
@ -197,8 +138,6 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem)
if (pIMem != NULL) {
tsdbUnRefMemTable(pRepo, pIMem);
}
tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
}
void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
@ -230,6 +169,10 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
ASSERT(pRepo->mem->extraBuffList != NULL);
SListNode *pNode = (SListNode *)malloc(sizeof(SListNode) + bytes);
if (pNode == NULL) {
if (listNEles(pRepo->mem->extraBuffList) == 0) {
tdListFree(pRepo->mem->extraBuffList);
pRepo->mem->extraBuffList = NULL;
}
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
@ -260,18 +203,18 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
}
int tsdbAsyncCommit(STsdbRepo *pRepo) {
if (pRepo->mem == NULL) return 0;
SMemTable *pIMem = pRepo->imem;
if (pRepo->mem != NULL) {
sem_wait(&(pRepo->readyToCommit));
sem_wait(&(pRepo->readyToCommit));
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
if (tsdbLockRepo(pRepo) < 0) return -1;
pRepo->imem = pRepo->mem;
pRepo->mem = NULL;
tsdbScheduleCommit(pRepo);
if (tsdbUnlockRepo(pRepo) < 0) return -1;
}
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START, TSDB_CODE_SUCCESS);
if (tsdbLockRepo(pRepo) < 0) return -1;
pRepo->imem = pRepo->mem;
pRepo->mem = NULL;
tsdbScheduleCommit(pRepo);
if (tsdbUnlockRepo(pRepo) < 0) return -1;
if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
@ -406,88 +349,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
return 0;
}
void *tsdbCommitData(STsdbRepo *pRepo) {
SMemTable * pMem = pRepo->imem;
STsdbCfg * pCfg = &pRepo->config;
SDataCols * pDataCols = NULL;
STsdbMeta * pMeta = pRepo->tsdbMeta;
SCommitIter *iters = NULL;
SRWHelper whelper = {0};
ASSERT(pMem != NULL);
tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
// Create the iterator to read from cache
if (pMem->numOfRows > 0) {
iters = tsdbCreateCommitIters(pRepo);
if (iters == NULL) {
tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
goto _exit;
}
int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
// Loop to commit to each file
for (int fid = sfid; fid <= efid; fid++) {
if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _exit;
}
}
}
// Commit to update meta file
if (tsdbCommitMeta(pRepo) < 0) {
tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _exit;
}
tsdbFitRetention(pRepo);
_exit:
tdFreeDataCols(pDataCols);
tsdbDestroyCommitIters(iters, pMem->maxTables);
tsdbDestroyHelper(&whelper);
tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
tsdbEndCommit(pRepo);
return NULL;
}
// ---------------- LOCAL FUNCTIONS ----------------
static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) {
ASSERT(pRepo->mem != NULL);
if (pRepo->mem->extraBuffList == NULL) {
STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
ASSERT(pBufBlock != NULL);
pBufBlock->offset -= bytes;
pBufBlock->remain += bytes;
ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
} else {
SListNode *pNode = (SListNode *)POINTER_SHIFT(ptr, -(int)(sizeof(SListNode)));
ASSERT(listTail(pRepo->mem->extraBuffList) == pNode);
tdListPopNode(pRepo->mem->extraBuffList, pNode);
free(pNode);
tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes);
}
}
static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) {
STsdbMeta *pMeta = pRepo->tsdbMeta;
@ -578,240 +440,11 @@ static void tsdbFreeTableData(STableData *pTableData) {
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
static int tsdbCommitMeta(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
STsdbMeta *pMeta = pRepo->tsdbMeta;
SActObj * pAct = NULL;
SActCont * pCont = NULL;
if (listNEles(pMem->actList) > 0) {
if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
SListNode *pNode = NULL;
while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
pAct = (SActObj *)pNode->data;
if (pAct->act == TSDB_UPDATE_META) {
pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno));
tdKVStoreEndCommit(pMeta->pStore);
goto _err;
}
} else if (pAct->act == TSDB_DROP_META) {
if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
tstrerror(terrno));
tdKVStoreEndCommit(pMeta->pStore);
goto _err;
}
} else {
ASSERT(false);
}
}
if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
}
return 0;
_err:
return -1;
}
static void tsdbEndCommit(STsdbRepo *pRepo) {
if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
sem_post(&(pRepo->readyToCommit));
}
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
return 0;
}
void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
*minKey = fileId * daysPerFile * tsMsPerDay[precision];
*maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
}
static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
char * dataDir = NULL;
STsdbCfg * pCfg = &pRepo->config;
STsdbFileH *pFileH = pRepo->tsdbFileH;
SFileGroup *pGroup = NULL;
SMemTable * pMem = pRepo->imem;
bool newLast = false;
TSKEY minKey = 0, maxKey = 0;
tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
// Check if there are data to commit to this file
int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
if (!hasDataToCommit) {
tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
return 0;
}
// Create and open files for commit
dataDir = tsdbGetDataDirName(pRepo->rootDir);
if (dataDir == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) {
tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _err;
}
// Open files for write/read
if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
newLast = TSDB_NLAST_FILE_OPENED(pHelper);
if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
// Loop to commit data in each table
for (int tid = 1; tid < pMem->maxTables; tid++) {
SCommitIter *pIter = iters + tid;
if (pIter->pTable == NULL) continue;
taosRLockLatch(&(pIter->pTable->latch));
if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
if (pIter->pIter != NULL) {
if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
taosRUnLockLatch(&(pIter->pTable->latch));
tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
tstrerror(terrno));
goto _err;
}
}
taosRUnLockLatch(&(pIter->pTable->latch));
// Move the last block to the new .l file if neccessary
if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
// Write the SCompBlock part
if (tsdbWriteCompInfo(pHelper) < 0) {
tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
goto _err;
}
}
if (tsdbWriteCompIdx(pHelper) < 0) {
tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
goto _err;
}
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 0, pGroup);
pthread_rwlock_wrlock(&(pFileH->fhlock));
(void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
if (newLast) {
(void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
} else {
pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
}
pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
pthread_rwlock_unlock(&(pFileH->fhlock));
return 0;
_err:
tfree(dataDir);
tsdbCloseHelperFile(pHelper, 1, NULL);
return -1;
}
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
SMemTable *pMem = pRepo->imem;
STsdbMeta *pMeta = pRepo->tsdbMeta;
SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
if (iters == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
// reference all tables
for (int i = 0; i < pMem->maxTables; i++) {
if (pMeta->tables[i] != NULL) {
tsdbRefTable(pMeta->tables[i]);
iters[i].pTable = pMeta->tables[i];
}
}
if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
for (int i = 0; i < pMem->maxTables; i++) {
if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
tSkipListIterNext(iters[i].pIter);
}
}
return iters;
_err:
tsdbDestroyCommitIters(iters, pMem->maxTables);
return NULL;
}
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
if (iters == NULL) return;
for (int i = 1; i < maxTables; i++) {
if (iters[i].pTable != NULL) {
tsdbUnRefTable(iters[i].pTable);
tSkipListDestroyIter(iters[i].pIter);
}
}
free(iters);
}
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
ASSERT(pMemTable->maxTables < maxTables);
@ -848,4 +481,400 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema *
}
return 0;
}
static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
if (pBlock->dataLen <= 0) return -1;
pIter->totalLen = pBlock->dataLen;
pIter->len = 0;
pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
return 0;
}
static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
SDataRow row = pIter->row;
if (row == NULL) return NULL;
pIter->len += dataRowLen(row);
if (pIter->len >= pIter->totalLen) {
pIter->row = NULL;
} else {
pIter->row = (char *)row + dataRowLen(row);
}
return row;
}
static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
TSKEY now) {
if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
" maxKey %" PRId64 " row key %" PRId64,
REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey,
dataRowKey(row));
terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
return -1;
}
return 0;
}
static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
ASSERT(pMsg != NULL);
STsdbMeta * pMeta = pRepo->tsdbMeta;
SSubmitMsgIter msgIter = {0};
SSubmitBlk * pBlock = NULL;
SSubmitBlkIter blkIter = {0};
SDataRow row = NULL;
TSKEY now = taosGetTimestamp(pRepo->config.precision);
TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
terrno = TSDB_CODE_SUCCESS;
pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
while (true) {
if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
if (pBlock == NULL) break;
pBlock->uid = htobe64(pBlock->uid);
pBlock->tid = htonl(pBlock->tid);
pBlock->sversion = htonl(pBlock->sversion);
pBlock->dataLen = htonl(pBlock->dataLen);
pBlock->schemaLen = htonl(pBlock->schemaLen);
pBlock->numOfRows = htons(pBlock->numOfRows);
if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
STable *pTable = pMeta->tables[pBlock->tid];
if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
// Check schema version and update schema if needed
if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
continue;
} else {
return -1;
}
}
tsdbInitSubmitBlkIter(pBlock, &blkIter);
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
if (tsdbCheckRowRange(pRepo, pTable, row, minKey, maxKey, now) < 0) {
return -1;
}
}
}
if (terrno != TSDB_CODE_SUCCESS) return -1;
return 0;
}
static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows) {
STsdbMeta * pMeta = pRepo->tsdbMeta;
int64_t points = 0;
STable * pTable = NULL;
SSubmitBlkIter blkIter = {0};
SDataRow row = NULL;
void ** rows = NULL;
int rowCounter = 0;
ASSERT(pBlock->tid < pMeta->maxTables);
pTable = pMeta->tables[pBlock->tid];
ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
rows = (void **)calloc(pBlock->numOfRows, sizeof(void *));
if (rows == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
tsdbInitSubmitBlkIter(pBlock, &blkIter);
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) {
tsdbFreeRows(pRepo, rows, rowCounter);
goto _err;
}
(*affectedrows)++;
points++;
if (rows[rowCounter] != NULL) {
rowCounter++;
}
}
if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
goto _err;
}
STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
pRepo->stat.totalStorage += points * schemaVLen(pSchema);
free(rows);
return 0;
_err:
free(rows);
return -1;
}
static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow) {
STsdbCfg * pCfg = &pRepo->config;
TKEY tkey = dataRowTKey(row);
TSKEY key = dataRowKey(row);
bool isRowDelete = TKEY_IS_DELETED(tkey);
if (isRowDelete) {
if (!pCfg->update) {
tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
if (key > TABLE_LASTKEY(pTable)) {
tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
return 0;
}
}
void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
if (pRow == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
return -1;
}
dataRowCpy(pRow, row);
ppRow[0] = pRow;
tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
key);
return 0;
}
static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
if (pMsg == NULL) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1;
}
pIter->totalLen = pMsg->length;
pIter->len = 0;
pIter->pMsg = pMsg;
if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1;
}
return 0;
}
static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
if (pIter->len == 0) {
pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE;
} else {
SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
}
if (pIter->len > pIter->totalLen) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
*pPBlock = NULL;
return -1;
}
*pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
return 0;
}
static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
ASSERT(pTable != NULL);
STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
int sversion = schemaVersion(pSchema);
if (pBlock->sversion == sversion) {
return 0;
} else {
if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema
terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
return -1;
}
}
if (pBlock->sversion > sversion) { // may need to update table schema
if (pBlock->schemaLen > 0) {
tsdbDebug(
"vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
int numOfCols = pBlock->schemaLen / sizeof(STColumn);
STColumn *pTCol = (STColumn *)pBlock->data;
STSchemaBuilder schemaBuilder = {0};
if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
tstrerror(terrno));
return -1;
}
for (int i = 0; i < numOfCols; i++) {
if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
tstrerror(terrno));
tdDestroyTSchemaBuilder(&schemaBuilder);
return -1;
}
}
STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
if (pNSchema == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tdDestroyTSchemaBuilder(&schemaBuilder);
return -1;
}
tdDestroyTSchemaBuilder(&schemaBuilder);
tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
} else {
tsdbDebug(
"vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
return -1;
}
} else {
ASSERT(pBlock->sversion >= 0);
if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
}
terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
return -1;
}
return 0;
}
static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter) {
if (rowCounter < 1) return 0;
SMemTable * pMemTable = NULL;
STableData *pTableData = NULL;
STsdbMeta * pMeta = pRepo->tsdbMeta;
STsdbCfg * pCfg = &(pRepo->config);
ASSERT(pRepo->mem != NULL);
pMemTable = pRepo->mem;
if (TABLE_TID(pTable) >= pMemTable->maxTables) {
if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
tsdbFreeRows(pRepo, rows, rowCounter);
return -1;
}
}
pTableData = pMemTable->tData[TABLE_TID(pTable)];
if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
if (pTableData != NULL) {
taosWLockLatch(&(pMemTable->latch));
pMemTable->tData[TABLE_TID(pTable)] = NULL;
tsdbFreeTableData(pTableData);
taosWUnLockLatch(&(pMemTable->latch));
}
pTableData = tsdbNewTableData(pCfg, pTable);
if (pTableData == NULL) {
tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo),
TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno));
tsdbFreeRows(pRepo, rows, rowCounter);
return -1;
}
pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
}
ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
int64_t osize = SL_SIZE(pTableData->pData);
tSkipListPutBatch(pTableData->pData, rows, rowCounter);
int64_t dsize = SL_SIZE(pTableData->pData) - osize;
if (pMemTable->keyFirst > dataRowKey(rows[0])) pMemTable->keyFirst = dataRowKey(rows[0]);
if (pMemTable->keyLast < dataRowKey(rows[rowCounter - 1])) pMemTable->keyLast = dataRowKey(rows[rowCounter - 1]);
pMemTable->numOfRows += dsize;
if (pTableData->keyFirst > dataRowKey(rows[0])) pTableData->keyFirst = dataRowKey(rows[0]);
if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]);
pTableData->numOfRows += dsize;
// TODO: impl delete row thing
if (TABLE_LASTKEY(pTable) < dataRowKey(rows[rowCounter-1])) TABLE_LASTKEY(pTable) = dataRowKey(rows[rowCounter-1]);
return 0;
}
static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
ASSERT(pRepo->mem != NULL);
STsdbBufPool *pBufPool = pRepo->pPool;
for (int i = rowCounter - 1; i >= 0; --i) {
SDataRow row = (SDataRow)rows[i];
int bytes = (int)dataRowLen(row);
if (pRepo->mem->extraBuffList == NULL) {
STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
ASSERT(pBufBlock != NULL && pBufBlock->offset >= bytes);
pBufBlock->offset -= bytes;
pBufBlock->remain += bytes;
ASSERT(row == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
if (pBufBlock->offset == 0) { // return the block to buffer pool
tsdbLockRepo(pRepo);
SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList);
tdListPrependNode(pBufPool->bufBlockList, pNode);
tsdbUnlockRepo(pRepo);
}
} else {
ASSERT(listNEles(pRepo->mem->extraBuffList) > 0);
SListNode *pNode = tdListPopTail(pRepo->mem->extraBuffList);
ASSERT(row == pNode->data);
free(pNode);
tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes);
if (listNEles(pRepo->mem->extraBuffList) == 0) {
tdListFree(pRepo->mem->extraBuffList);
pRepo->mem->extraBuffList = NULL;
}
}
}
}

2
src/tsdb/src/tsdbRWHelper.c
View File

@ -1595,7 +1595,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
tblkIdx++;
} else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
// Delete the block and do some stuff
ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
// ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
*pCommitIter->pIter = slIter;
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;

224
src/tsdb/src/tsdbRead.c
View File

@ -120,8 +120,6 @@ typedef struct STsdbQueryHandle {
SDataCols *pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size
SMemRef *pMemRef;
// SMemTable *mem; // mem-table
// SMemTable *imem; // imem-table, acquired from snapshot
SArray *defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
@ -194,9 +192,12 @@ static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
}
static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
assert(pQueryHandle != NULL);
SMemRef* pMemRef = pQueryHandle->pMemRef;
if (pMemRef == NULL) { // it has been freed
return;
}
if (--pMemRef->ref == 0) {
tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
pMemRef->mem = NULL;
@ -205,6 +206,7 @@ static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
pQueryHandle->pMemRef = NULL;
}
static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
assert(sizeOfGroup >= 1 && pMeta != NULL);
@ -1849,6 +1851,8 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
pQueryHandle->activeIndex += 1;
}
// no data in memtable or imemtable, decrease the memory reference.
tsdbMayUnTakeMemSnapshot(pQueryHandle);
return false;
}
@ -1976,6 +1980,111 @@ static void destroyHelper(void* param) {
free(param);
}
static bool getNeighborRows(STsdbQueryHandle* pQueryHandle) {
assert (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL);
SDataBlockInfo blockInfo = {{0}, 0};
pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
pQueryHandle->order = TSDB_ORDER_DESC;
if (!tsdbNextDataBlock((void*) pQueryHandle)) {
return false;
}
tsdbRetrieveDataBlockInfo((void*) pQueryHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pQueryHandle, pQueryHandle->defaultLoadColumn);
if (terrno != TSDB_CODE_SUCCESS) {
return false;
}
if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
// data already retrieve, discard other data rows and return
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
}
pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.rows = 1;
pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
return true;
} else {
STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX};
STsdbQueryCond cond = {
.order = TSDB_ORDER_ASC,
.numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle))
};
cond.twindow = win;
cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
if (cond.colList == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return false;
}
for(int32_t i = 0; i < cond.numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i);
memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
}
STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
tfree(cond.colList);
pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
if (pSecQueryHandle->pTableCheckInfo == NULL) {
tsdbCleanupQueryHandle(pSecQueryHandle);
return false;
}
if (!tsdbNextDataBlock((void*) pSecQueryHandle)) {
tsdbCleanupQueryHandle(pSecQueryHandle);
return false;
}
tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle));
size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
assert(pCol->info.colId == pCol1->info.colId);
memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
}
SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
// it is ascending order
pQueryHandle->order = TSDB_ORDER_DESC;
pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
pQueryHandle->cur.rows = 2;
pQueryHandle->cur.mixBlock = true;
int32_t step = -1;// one step for ascending order traverse
for (int32_t j = 0; j < si; ++j) {
STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step;
}
tsdbCleanupQueryHandle(pSecQueryHandle);
}
//disable it after retrieve data
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
pQueryHandle->checkFiles = false;
return true;
}
// handle data in cache situation
bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
@ -1986,106 +2095,15 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
assert(numOfTables > 0);
SDataBlockInfo blockInfo = {{0}, 0};
if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
pQueryHandle->order = TSDB_ORDER_DESC;
SMemRef* pMemRef = pQueryHandle->pMemRef;
tsdbMayTakeMemSnapshot(pQueryHandle);
bool ret = getNeighborRows(pQueryHandle);
tsdbMayUnTakeMemSnapshot(pQueryHandle);
if (!tsdbNextDataBlock(pHandle)) {
return false;
}
tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
if (terrno != TSDB_CODE_SUCCESS) {
return false;
}
if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
// data already retrieve, discard other data rows and return
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
}
pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.rows = 1;
pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
return true;
} else {
STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX};
STsdbQueryCond cond = {
.order = TSDB_ORDER_ASC,
.numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle))
};
cond.twindow = win;
cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
if (cond.colList == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return false;
}
for(int32_t i = 0; i < cond.numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i);
memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
}
STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
tfree(cond.colList);
pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey);
if (pSecQueryHandle->pTableCheckInfo == NULL) {
tsdbCleanupQueryHandle(pSecQueryHandle);
return false;
}
if (!tsdbNextDataBlock((void*) pSecQueryHandle)) {
tsdbCleanupQueryHandle(pSecQueryHandle);
return false;
}
tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle));
size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes);
SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
assert(pCol->info.colId == pCol1->info.colId);
memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
}
SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
// it is ascending order
pQueryHandle->order = TSDB_ORDER_DESC;
pQueryHandle->window = pQueryHandle->cur.win;
pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]};
pQueryHandle->cur.rows = 2;
pQueryHandle->cur.mixBlock = true;
int32_t step = -1;// one step for ascending order traverse
for (int32_t j = 0; j < si; ++j) {
STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step;
}
tsdbCleanupQueryHandle(pSecQueryHandle);
}
//disable it after retrieve data
pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
pQueryHandle->checkFiles = false;
return true;
// restore the pMemRef
pQueryHandle->pMemRef = pMemRef;
return ret;
}
if (pQueryHandle->checkFiles) {
@ -2343,7 +2361,7 @@ void filterPrepare(void* expr, void* param) {
if (pInfo->optr == TSDB_RELATION_IN) {
pInfo->q = (char*) pCond->arr;
} else {
} else if (pCond != NULL) {
uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE;
if (size < (uint32_t)pSchema->bytes) {
size = pSchema->bytes;

14
src/util/inc/tarray.h
View File

@ -70,6 +70,13 @@ void* taosArrayGet(const SArray* pArray, size_t index);
*/
void* taosArrayGetP(const SArray* pArray, size_t index);
/**
* get the last element in the array list
* @param pArray
* @return
*/
void* taosArrayGetLast(const SArray* pArray);
/**
* return the size of array
* @param pArray
@ -117,6 +124,13 @@ void taosArrayClear(SArray* pArray);
*/
void taosArrayDestroy(SArray* pArray);
/**
*
* @param pArray
* @param fp
*/
void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
/**
* sort the array
* @param pArray

1
src/util/inc/tskiplist.h
View File

@ -131,6 +131,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _
__sl_key_fn_t fn);
void tSkipListDestroy(SSkipList *pSkipList);
SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData);
void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata);
SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);

21
src/util/src/tarray.c
View File

@ -99,6 +99,10 @@ void* taosArrayGetP(const SArray* pArray, size_t index) {
return *(void**)d;
}
void* taosArrayGetLast(const SArray* pArray) {
return TARRAY_GET_ELEM(pArray, pArray->size - 1);
}
size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; }
void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
@ -189,6 +193,23 @@ void taosArrayDestroy(SArray* pArray) {
free(pArray);
}
void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
if (pArray == NULL) {
return;
}
if (fp == NULL) {
taosArrayDestroy(pArray);
return;
}
for(int32_t i = 0; i < pArray->size; ++i) {
fp(TARRAY_GET_ELEM(pArray, i));
}
taosArrayDestroy(pArray);
}
void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
assert(pArray != NULL);
assert(compar != NULL);

5
src/util/src/tcache.c
View File

@ -658,7 +658,11 @@ void* taosCacheTimedRefresh(void *handle) {
int64_t count = 0;
while(1) {
#if defined LINUX
usleep(500*1000);
#else
taosMsleep(500);
#endif
// check if current cache object will be deleted every 500ms.
if (pCacheObj->deleting) {
@ -677,6 +681,7 @@ void* taosCacheTimedRefresh(void *handle) {
continue;
}
uDebug("%s refresh thread timed scan", pCacheObj->name);
pCacheObj->statistics.refreshCount++;
// refresh data in hash table

1
src/util/src/tcompare.c
View File

@ -1,7 +1,6 @@
#include "taosdef.h"
#include "tcompare.h"
#include "tarray.h"
#include "tutil.h"
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);

2
src/util/src/tkvstore.c
View File

@ -236,6 +236,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR);
if (rInfo.offset < 0) {
uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
@ -254,6 +255,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
if (taosWrite(pStore->fd, cont, contLen) < contLen) {
uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}

4
src/util/src/tlog.c
View File

@ -287,17 +287,17 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
tsLogObj.fileNum = maxFileNum;
taosGetLogFileName(fn);
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
strcat(name, ".0");
}
bool log0Exist = stat(name, &logstat0) >= 0;
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn);
strcat(name, ".1");
}
bool log0Exist = stat(name, &logstat0) >= 0;
bool log1Exist = stat(name, &logstat1) >= 0;
// if none of the log files exist, open 0, if both exists, open the old one

4
src/util/src/tref.c
View File

@ -426,11 +426,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
(*pSet->fp)(pNode->p);
uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
free(pNode);
released = 1;
} else {
uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count);
uTrace("rsetId:%d p:%p rid:%" PRId64 " is released, count:%d", rsetId, pNode->p, rid, pNode->count);
}
} else {
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);

168
src/util/src/tskiplist.c
View File

@ -24,10 +24,12 @@ static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, in
static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode);
static void tSkipListCorrectLevel(SSkipList *pSkipList);
static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode);
static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData);
static SSkipListNode * tSkipListNewNode(uint8_t level);
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward);
static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData);
static SSkipListNode *tSkipListNewNode(uint8_t level);
#define tSkipListFreeNode(n) tfree((n))
static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward,
bool hasDup);
static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList);
static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList);
@ -109,32 +111,87 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) {
if (pSkipList == NULL || pData == NULL) return NULL;
SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
uint8_t dupMode = SL_DUP_MODE(pSkipList);
SSkipListNode *pNode = NULL;
tSkipListWLock(pSkipList);
bool hasDup = tSkipListGetPosToPut(pSkipList, backward, pData);
if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) {
if (dupMode == SL_UPDATE_DUP_KEY) {
pNode = SL_NODE_GET_BACKWARD_POINTER(backward[0], 0);
atomic_store_ptr(&(pNode->pData), pData);
}
} else {
pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
if (pNode != NULL) {
pNode->pData = pData;
tSkipListDoInsert(pSkipList, backward, pNode);
}
}
pNode = tSkipListPutImpl(pSkipList, pData, backward, false, hasDup);
tSkipListUnlock(pSkipList);
return pNode;
}
// Put a batch of data into skiplist. The batch of data must be in ascending order
void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata) {
SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
bool hasDup = false;
char * pKey = NULL;
char * pDataKey = NULL;
int compare = 0;
tSkipListWLock(pSkipList);
// backward to put the first data
hasDup = tSkipListGetPosToPut(pSkipList, backward, ppData[0]);
tSkipListPutImpl(pSkipList, ppData[0], backward, false, hasDup);
for (int level = 0; level < pSkipList->maxLevel; level++) {
forward[level] = SL_NODE_GET_BACKWARD_POINTER(backward[level], level);
}
// forward to put the rest of data
for (int idata = 1; idata < ndata; idata++) {
pDataKey = pSkipList->keyFn(ppData[idata]);
// Compare max key
pKey = SL_GET_MAX_KEY(pSkipList);
compare = pSkipList->comparFn(pDataKey, pKey);
if (compare > 0) {
for (int i = 0; i < pSkipList->maxLevel; i++) {
forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i);
}
hasDup = false;
} else {
SSkipListNode *px = pSkipList->pHead;
for (int i = pSkipList->maxLevel - 1; i >= 0; --i) {
if (i < pSkipList->level) {
// set new px
if (forward[i] != pSkipList->pHead) {
if (px == pSkipList->pHead ||
pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, forward[i]), SL_GET_NODE_KEY(pSkipList, px)) > 0) {
px = forward[i];
}
}
SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i);
while (p != pSkipList->pTail) {
pKey = SL_GET_NODE_KEY(pSkipList, p);
compare = pSkipList->comparFn(pKey, pDataKey);
if (compare >= 0) {
if (compare == 0) hasDup = true;
break;
} else {
px = p;
p = SL_NODE_GET_FORWARD_POINTER(px, i);
}
}
}
forward[i] = px;
}
}
tSkipListPutImpl(pSkipList, ppData[idata], forward, true, hasDup);
}
tSkipListUnlock(pSkipList);
}
uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) {
uint32_t count = 0;
@ -310,22 +367,25 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
}
}
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode) {
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) {
for (int32_t i = 0; i < pNode->level; ++i) {
if (i >= pSkipList->level) {
SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail;
SL_NODE_GET_BACKWARD_POINTER(pNode, i) = pSkipList->pHead;
SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode;
SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
SSkipListNode *x = direction[i];
if (isForward) {
SL_NODE_GET_BACKWARD_POINTER(pNode, i) = x;
SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(x, i);
SL_NODE_GET_BACKWARD_POINTER(next, i) = pNode;
SL_NODE_GET_FORWARD_POINTER(pNode, i) = next;
SL_NODE_GET_FORWARD_POINTER(x, i) = pNode;
} else {
SSkipListNode *x = backward[i];
SL_NODE_GET_FORWARD_POINTER(pNode, i) = x;
SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(x, i);
SL_NODE_GET_FORWARD_POINTER(prev, i) = pNode;
SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode;
SL_NODE_GET_BACKWARD_POINTER(pNode, i) = prev;
SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode;
}
}
@ -377,7 +437,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
char * pDataKey = pSkipList->keyFn(pData);
if (pSkipList->size == 0) {
for (int i = 0; i < pSkipList->level; i++) {
for (int i = 0; i < pSkipList->maxLevel; i++) {
backward[i] = pSkipList->pTail;
}
} else {
@ -387,7 +447,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
pKey = SL_GET_MAX_KEY(pSkipList);
compare = pSkipList->comparFn(pDataKey, pKey);
if (compare >= 0) {
for (int i = 0; i < pSkipList->level; i++) {
for (int i = 0; i < pSkipList->maxLevel; i++) {
backward[i] = pSkipList->pTail;
}
@ -398,7 +458,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
pKey = SL_GET_MIN_KEY(pSkipList);
compare = pSkipList->comparFn(pDataKey, pKey);
if (compare < 0) {
for (int i = 0; i < pSkipList->level; i++) {
for (int i = 0; i < pSkipList->maxLevel; i++) {
backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i);
}
@ -406,18 +466,20 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
}
SSkipListNode *px = pSkipList->pTail;
for (int i = pSkipList->level - 1; i >= 0; --i) {
SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i);
while (p != pSkipList->pHead) {
pKey = SL_GET_NODE_KEY(pSkipList, p);
for (int i = pSkipList->maxLevel - 1; i >= 0; --i) {
if (i < pSkipList->level) {
SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i);
while (p != pSkipList->pHead) {
pKey = SL_GET_NODE_KEY(pSkipList, p);
compare = pSkipList->comparFn(pKey, pDataKey);
if (compare <= 0) {
if (compare == 0 && !hasDupKey) hasDupKey = true;
break;
} else {
px = p;
p = SL_NODE_GET_BACKWARD_POINTER(px, i);
compare = pSkipList->comparFn(pKey, pDataKey);
if (compare <= 0) {
if (compare == 0 && !hasDupKey) hasDupKey = true;
break;
} else {
px = p;
p = SL_NODE_GET_BACKWARD_POINTER(px, i);
}
}
}
@ -579,6 +641,32 @@ static SSkipListNode *tSkipListNewNode(uint8_t level) {
return pNode;
}
static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward,
bool hasDup) {
uint8_t dupMode = SL_DUP_MODE(pSkipList);
SSkipListNode *pNode = NULL;
if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) {
if (dupMode == SL_UPDATE_DUP_KEY) {
if (isForward) {
pNode = SL_NODE_GET_FORWARD_POINTER(direction[0], 0);
} else {
pNode = SL_NODE_GET_BACKWARD_POINTER(direction[0], 0);
}
atomic_store_ptr(&(pNode->pData), pData);
}
} else {
pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
if (pNode != NULL) {
pNode->pData = pData;
tSkipListDoInsert(pSkipList, direction, pNode, isForward);
}
}
return pNode;
}
// static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey,
// int32_t cond, SSkipListNode ***pRes) {
// pthread_rwlock_rdlock(&pSkipList->lock);

4
src/util/src/tsocket.c
View File

@ -107,7 +107,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
while (nleft > 0) {
nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
if (nwritten <= 0) {
if (errno == EINTR)
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
continue;
else
return -1;
@ -133,7 +133,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
if (nread == 0) {
break;
} else if (nread < 0) {
if (errno == EINTR) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue;
} else {
return -1;

4
src/util/tests/trefTest.c
View File

@ -77,8 +77,8 @@ void *acquireRelease(void *param) {
printf("a");
id = random() % pSpace->refNum;
code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
if (code >= 0) {
void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
if (p) {
usleep(id % 5 + 1);
taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
}

2
src/vnode/inc/vnodeInt.h
View File

@ -58,7 +58,7 @@ typedef struct {
char *rootDir;
tsem_t sem;
int8_t dropped;
char db[TSDB_DB_NAME_LEN];
char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
} SVnodeObj;
void vnodeInitWriteFp(void);

4
src/vnode/src/vnodeCfg.c
View File

@ -25,7 +25,7 @@
#include "vnodeCfg.h"
static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
strcpy(pVnode->db, vnodeMsg->db);
tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db));
pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion;
pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize;
pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks;
@ -97,7 +97,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
vError("vgId:%d, failed to read %s, db not found", pVnode->vgId, file);
goto PARSE_VCFG_ERROR;
}
strcpy(vnodeMsg.db, db->valuestring);
tstrncpy(vnodeMsg.db, db->valuestring, sizeof(vnodeMsg.db));
cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
if (!cfgVersion || cfgVersion->type != cJSON_Number) {

23
src/vnode/src/vnodeMain.c
View File

@ -30,7 +30,7 @@
static SHashObj*tsVnodesHash;
static void vnodeCleanUp(SVnodeObj *pVnode);
static int vnodeProcessTsdbStatus(void *arg, int status);
static int vnodeProcessTsdbStatus(void *arg, int status, int eno);
static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId);
static void vnodeNotifyRole(void *ahandle, int8_t role);
@ -315,7 +315,13 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
pVnode->version = walGetVersion(pVnode->wal);
}
tsdbSyncCommit(pVnode->tsdb);
code = tsdbSyncCommit(pVnode->tsdb);
if (code != 0) {
vError("vgId:%d, failed to commit after restore from wal since %s", pVnode->vgId, tstrerror(code));
vnodeCleanUp(pVnode);
return code;
}
walRemoveAllOldFiles(pVnode->wal);
walRenew(pVnode->wal);
@ -412,6 +418,7 @@ void vnodeRelease(void *pVnodeRaw) {
}
if (pVnode->wal) {
walRemoveAllOldFiles(pVnode->wal);
walClose(pVnode->wal);
pVnode->wal = NULL;
}
@ -583,13 +590,21 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
}
// TODO: this is a simple implement
static int vnodeProcessTsdbStatus(void *arg, int status) {
static int vnodeProcessTsdbStatus(void *arg, int status, int eno) {
SVnodeObj *pVnode = arg;
if (eno != TSDB_CODE_SUCCESS) {
// TODO: deal with the error here
}
if (status == TSDB_STATUS_COMMIT_START) {
pVnode->fversion = pVnode->version;
vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
return walRenew(pVnode->wal);
if (pVnode->status == TAOS_VN_STATUS_INIT) {
return 0;
} else {
return walRenew(pVnode->wal);
}
}
if (status == TSDB_STATUS_COMMIT_OVER) {

2
src/wal/inc/walInt.h
View File

@ -37,7 +37,7 @@ extern int32_t wDebugFlag;
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE))
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (TSDB_FILENAME_LEN + 32)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_FILE_NUM 3
typedef struct {

8
src/wal/src/walMgmt.c
View File

@ -124,15 +124,7 @@ void walClose(void *handle) {
SWal *pWal = handle;
pthread_mutex_lock(&pWal->mutex);
taosClose(pWal->fd);
if (pWal->keep != TAOS_WAL_KEEP) {
walRemoveAllOldFiles(pWal);
} else {
wDebug("vgId:%d, wal:%p file:%s, it is closed and kept", pWal->vgId, pWal, pWal->name);
}
pthread_mutex_unlock(&pWal->mutex);
taosRemoveRef(tsWal.refId, pWal->rid);
}

4
src/wal/src/walWrite.c
View File

@ -67,6 +67,7 @@ void walRemoveOneOldFile(void *handle) {
SWal *pWal = handle;
if (pWal == NULL) return;
if (pWal->keep == TAOS_WAL_KEEP) return;
if (pWal->fd <= 0) return;
pthread_mutex_lock(&pWal->mutex);
@ -91,6 +92,8 @@ void walRemoveAllOldFiles(void *handle) {
SWal * pWal = handle;
int64_t fileId = -1;
pthread_mutex_lock(&pWal->mutex);
while (walGetNextFile(pWal, &fileId) >= 0) {
snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
@ -100,6 +103,7 @@ void walRemoveAllOldFiles(void *handle) {
wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name);
}
}
pthread_mutex_unlock(&pWal->mutex);
}
int32_t walWrite(void *handle, SWalHead *pHead) {

3
tests/pytest/fulltest.sh
View File

@ -155,6 +155,9 @@ python3 ./test.py -f query/queryCountCSVData.py
python3 ./test.py -f query/natualInterval.py
python3 ./test.py -f query/bug1471.py
#python3 ./test.py -f query/dataLossTest.py
python3 ./test.py -f query/bug1874.py
python3 ./test.py -f query/bug1875.py
python3 ./test.py -f query/bug1876.py
#stream
python3 ./test.py -f stream/metric_1.py

340
tests/pytest/update/allow_update-0.py
View File

@ -1,170 +1,170 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
print("==============step1")
tdSql.execute("create database udb update 0")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
print("==============step1")
tdSql.execute("create database udb update 0")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

532
tests/pytest/update/allow_update.py
View File

@ -1,266 +1,266 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
print("==============step1")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 2)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 2)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 4)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 4)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 2)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 4)
tdSql.checkData(7, 0, 2)
tdSql.checkData(8, 0, 4)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
print("==============step7")
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
tdSql.checkData(0, 1, None)
tdSql.checkData(1, 1, None)
tdSql.checkData(2, 1, None)
tdSql.checkData(0, 2, 'c-3')
tdSql.checkData(1, 2, 'c+0')
tdSql.checkData(2, 2, 'c+3')
tdSql.checkData(0, 3, None)
tdSql.checkData(1, 3, None)
tdSql.checkData(2, 3, None)
print("==============step8")
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
print("==============step9")
insertRows = 20000
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
tdSql.query("select a,b from ct limit 3")
tdSql.checkData(0, 0, insertRows+0)
tdSql.checkData(1, 0, insertRows+1)
tdSql.checkData(2, 0, insertRows+2)
tdSql.checkData(0, 1, insertRows+0)
tdSql.checkData(1, 1, insertRows+1)
tdSql.checkData(2, 1, insertRows+2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
print("==============step1")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 2)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 2)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 4)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 4)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 2)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 4)
tdSql.checkData(7, 0, 2)
tdSql.checkData(8, 0, 4)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
print("==============step7")
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
tdSql.checkData(0, 1, None)
tdSql.checkData(1, 1, None)
tdSql.checkData(2, 1, None)
tdSql.checkData(0, 2, 'c-3')
tdSql.checkData(1, 2, 'c+0')
tdSql.checkData(2, 2, 'c+3')
tdSql.checkData(0, 3, None)
tdSql.checkData(1, 3, None)
tdSql.checkData(2, 3, None)
print("==============step8")
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
print("==============step9")
insertRows = 20000
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
tdSql.query("select a,b from ct limit 3")
tdSql.checkData(0, 0, insertRows+0)
tdSql.checkData(1, 0, insertRows+1)
tdSql.checkData(2, 0, insertRows+2)
tdSql.checkData(0, 1, insertRows+0)
tdSql.checkData(1, 1, insertRows+1)
tdSql.checkData(2, 1, insertRows+2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

11
tests/script/general/parser/constCol.sim
View File

@ -347,6 +347,17 @@ if $rows != 3 then
return -1
endi
sql select 0.1 + 0.2 from t1
if $rows != 3 then
return -1
endi
print =============================> td-2036
if $data00 != 0.3000000 then
print expect: 0.3000000, actual:$data00
return -1
endi
print ======================udc with normal column group by
sql_error select from t1

131
tests/script/general/parser/sliding.sim
View File

@ -4,6 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c debugFlag -v 135
system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135
system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
system sh/exec.sh -n dnode1 -s start
sleep 1000
sql connect
@ -28,7 +29,7 @@ $mt = $mtPrefix . $i
sql drop database if exists $db -x step1
step1:
sql create database if not exists $db maxtables 4 keep 36500
sql create database if not exists $db keep 36500
sql use $db
sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
@ -169,19 +170,20 @@ if $data94 != 90 then
endi
sql select count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by ts asc;
if $row != 30 then
if $row != 32 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
if $data00 != @99-12-31 23:59:40.000@ then
print expect 12-31 23:59:40.000, actual: $data00
return -1
endi
if $data01 != 1000 then
if $data01 != 334 then
return -1
endi
if $data02 != 99 then
if $data02 != 33 then
return -1
endi
@ -304,11 +306,11 @@ if $data13 != 9.810708435 then
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 1;
if $row != 14 then
if $row != 15 then
return -1
endi
if $data00 != @00-01-01 00:00:20.000@ then
if $data00 != @00-01-01 00:00:00.000@ then
return -1
endi
@ -316,7 +318,7 @@ if $data01 != 1000 then
return -1
endi
if $data02 != 66 then
if $data02 != 99 then
return -1
endi
@ -324,7 +326,7 @@ if $data03 != 28.866070048 then
return -1
endi
if $data90 != @00-01-01 00:03:20.000@ then
if $data90 != @00-01-01 00:03:00.000@ then
return -1
endi
@ -332,15 +334,70 @@ if $data91 != 1000 then
return -1
endi
if $data92 != 66 then
if $data92 != 99 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 5;
if $row != 11 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 6;
if $row != 10 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 7;
if $row != 9 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 8;
if $row != 8 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 9;
if $row != 7 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 10;
if $row != 6 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 11;
if $row != 5 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 12;
if $row != 4 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 13;
if $row != 3 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 14;
if $row != 2 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 15;
if $row != 1 then
return -1
endi
sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 16;
if $row != 0 then
return -1
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by ts desc;
if $row != 10 then
return -1
@ -364,16 +421,26 @@ if $data03 != 0.000000000 then
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 15;
if $row != 1 then
return -1
endi
sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 16;
if $row != 0 then
return -1
endi
sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10a) order by ts asc limit 1000;
if $row != 100 then
sql select count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by ts desc limit 10 offset 2;
if $data00 != @00-01-01 00:04:59.910@ then
return -1
endi
sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts asc limit 1000;
if $row != 3 then
return -1
endi
if $data00 != @00-01-01 00:00:00.000@ then
if $data00 != @99-12-31 23:59:40.000@ then
return -1
endi
@ -385,7 +452,7 @@ if $data05 != 33 then
return -1
endi
if $data10 != @00-01-01 00:00:00.010@ then
if $data10 != @99-12-31 23:59:50.000@ then
return -1
endi
@ -397,48 +464,28 @@ if $data15 != 33 then
return -1
endi
if $data95 != 33 then
if $data25 != 33 then
return -1
endi
sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10a) order by ts desc limit 1000;
if $row != 100 then
sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts desc limit 1000;
if $row != 1 then
return -1
endi
if $data00 != @00-01-01 00:00:00.990@ then
if $data00 != @99-12-31 23:59:40.000@ then
return -1
endi
if $data01 != 1 then
if $data01 != 33 then
return -1
endi
if $data02 != 0.000000000 then
if $data02 != 9.521904571 then
return -1
endi
if $data03 != 1 then
return -1
endi
if $data90 != @00-01-01 00:00:00.900@ then
return -1
endi
if $data91 != 4 then
return -1
endi
if $data92 != 1.118033989 then
return -1
endi
if $data93 != 4 then
return -1
endi
if $data94 != 30.00000 then
if $data03 != 33 then
return -1
endi
@ -457,5 +504,7 @@ sql_error select sum(c1) from sliding_tb0 interval(0) sliding(0);
sql_error select sum(c1) from sliding_tb0 interval(0m) sliding(0m);
sql_error select sum(c1) from sliding_tb0 interval(m) sliding(m);
sql_error select sum(c1) from sliding_tb0 sliding(4m);
sql_error select count(*) from sliding_tb0 interval(1s) sliding(10s);
sql_error select count(*) from sliding_tb0 interval(10s) sliding(10a);
system sh/exec.sh -n dnode1 -s stop -x SIGINT