xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:taosdata/TDengine into develop

This commit is contained in:
liuyq-617 2020-11-06 14:27:20 +08:00
parent a98a25e45b c0d6c718c6
commit 4638582eb8
72 changed files with 5979 additions and 2614 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Jenkinsfile vendored
View File

@ -27,6 +27,7 @@ pipeline {
cd debug
cmake .. > /dev/null
make > /dev/null
make install > /dev/null
cd ${WKC}/tests
#./test-all.sh smoke
./test-all.sh pytest

1
packaging/deb/makedeb.sh
View File

@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver

2
packaging/rpm/tdengine.spec
View File

@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri
cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver
cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include
cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include
@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${cfg_link_dir}/* || :
${csudo} rm -f ${inc_link_dir}/taos.h || :
${csudo} rm -f ${inc_link_dir}/taoserror.h || :

2
packaging/tools/makeclient.sh
View File

@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
else

2
packaging/tools/makepkg.sh
View File

@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
else
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
fi
lib_files="${build_dir}/lib/libtaos.so.${version}"

16
src/client/src/tscAsync.c
View File

@ -428,6 +428,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
} else {
assert(code == TSDB_CODE_SUCCESS);
}
// param already freed by other routine and pSql in tscCache when ctrl + c
if (atomic_load_ptr(&pSql->param) == NULL) {
return;
@ -441,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
// tscProcessSql can add error into async res
tscProcessSql(pSql);
return;
} else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
// tscProcessSql can add error into async res
tscProcessSql(pSql);
return;
@ -465,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscResetSqlCmdObj(pCmd, false);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else if (code != TSDB_CODE_SUCCESS) {

3
src/client/src/tscLocalMerge.c
View File

@ -97,6 +97,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
pCtx->param[2].i64Key = pQueryInfo->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
} else if (functionId == TSDB_FUNC_APERCT) {
pCtx->param[0].i64Key = pExpr->param[0].i64Key;
pCtx->param[0].nType = pExpr->param[0].nType;
}
pCtx->interBufBytes = pExpr->interBytes;

1
src/client/src/tscSQLParser.c
View File

@ -5311,6 +5311,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists;
pMsg->update = pCreateDb->update;
}
int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {

8
src/client/src/tscSubquery.c
View File

@ -523,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
assert(pSqlObj->subState.numOfRemain > 0);
if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code);
tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
freeJoinSubqueryObj(pSqlObj);
}
}
@ -565,7 +565,7 @@ int32_t tagValCompar(const void* p1, const void* p2) {
return (tag1->len > tag2->len)? 1: -1;
}
return strncmp(tag1->data, tag2->data, tag1->len);
return memcmp(tag1->data, tag2->data, tag1->len);
}
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@ -853,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
}
if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return.
assert(pParentSql->fp != tscJoinQueryCallback);
tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql);
// set no result command
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
assert(pParentSql->fp != tscJoinQueryCallback);
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
// proceed to for ts_comp query

84
src/common/inc/tdataformat.h
View File

@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
// ----------------- Semantic timestamp key definition
typedef uint64_t TKEY;
#define TKEY_INVALID UINT64_MAX
#define TKEY_NULL TKEY_INVALID
#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
if (key1 < key2) {
return -1;
} else if (key1 > key2) {
return 1;
} else {
return 0;
}
}
// ----------------- Data row structure
/* A data row, the format is like below:
@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
*
* NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
typedef void *SDataRow;
@ -137,11 +166,13 @@ typedef void *SDataRow;
#define dataRowLen(r) (*(uint16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
#define dataRowKey(r) tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (IS_VAR_DATA_TYPE(type)) {
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value);
break;
default:
} else {
if (offset == 0) {
ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
} else {
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
break;
}
}
return 0;
@ -171,11 +204,9 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (IS_VAR_DATA_TYPE(type)) {
return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
default:
} else {
return POINTER_SHIFT(row, offset);
}
}
@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle);
@ -204,27 +234,19 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (IS_VAR_DATA_TYPE(pCol->type)) {
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
break;
default:
} else {
return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
break;
}
}
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
ASSERT(rows > 0);
switch (pDataCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
break;
default:
} else {
return TYPE_BYTES[pDataCol->type] * rows;
}
}
@ -243,9 +265,14 @@ typedef struct {
} SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
#define dataColsTKeyLast(pCols) \
(((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
#define dataColsKeyLast(pCols) \
(((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
@ -253,10 +280,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!!
int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows);
// ----------------- K-V data row structure
/*

1
src/common/inc/tglobal.h
View File

@ -88,6 +88,7 @@ extern int16_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int32_t tsQuorum;
extern int32_t tsUpdate;
// balance
extern int32_t tsEnableBalance;

104
src/common/src/tdataformat.c
View File

@ -18,6 +18,9 @@
#include "tcoding.h"
#include "wchar.h"
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows);
/**
* Duplicate the schema and return a new object
*/
@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
pDataCol->len = 0;
if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
}
}
// value from timestamp should be TKEY here instead of TSKEY
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (IS_VAR_DATA_TYPE(pCol->type)) {
// set offset
pCol->dataOff[numOfRows] = pCol->len;
// Copy data
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
// Update the length
pCol->len += varDataTLen(value);
break;
default:
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes;
break;
}
}
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
int pointsLeft = numOfRows - pointsToPop;
ASSERT(pointsLeft > 0);
if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
ASSERT(pCol->len > 0);
VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
pCol->len = pCol->len - toffset;
ASSERT(pCol->len > 0);
memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
dataColSetOffset(pCol, pointsLeft);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
}
}
bool isNEleNull(SDataCol *pCol, int nEle) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
default:
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
}
}
void dataColSetNullAt(SDataCol *pCol, int index) {
@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
ASSERT(pDataCols->cols[i].dataOff != NULL);
pRet->cols[i].dataOff =
(int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].len = pDataCols->cols[i].len;
if (pDataCols->cols[i].len > 0) {
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
}
}
@ -420,11 +392,21 @@ void tdResetDataCols(SDataCols *pCols) {
}
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
int rcol = 0;
int dcol = 0;
if (dataRowDeleted(row)) {
for (; dcol < pCols->numOfCols; dcol++) {
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (dcol == 0) {
dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
} else {
dataColSetNullAt(pDataCol, pCols->numOfRows);
}
}
} else {
while (dcol < pCols->numOfCols) {
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (rcol >= schemaNCols(pSchema)) {
@ -435,7 +417,7 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
STColumn *pRowCol = schemaColAt(pSchema, rcol);
if (pRowCol->colId == pDataCol->colId) {
void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
dcol++;
rcol++;
@ -446,32 +428,18 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
dcol++;
}
}
}
pCols->numOfRows++;
}
// Pop pointsToPop points from the SDataCols
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
int pointsLeft = pCols->numOfRows - pointsToPop;
if (pointsLeft <= 0) {
tdResetDataCols(pCols);
return;
}
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SDataCol *pCol = pCols->cols + iCol;
dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
}
pCols->numOfRows = pointsLeft;
}
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
ASSERT(target->numOfCols == source->numOfCols);
SDataCols *pTarget = NULL;
if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0) {
@ -499,17 +467,23 @@ _err:
return -1;
}
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
// src2 data has more priority than src1
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows) {
tdResetDataCols(target);
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
while (target->numOfRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break;
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
if (key1 <= key2) {
ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
if (src1->cols[i].len > 0) {
@ -520,8 +494,8 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->numOfRows++;
(*iter1)++;
if (key1 == key2) (*iter2)++;
} else {
} else if (key1 >= key2) {
if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0) {
@ -529,10 +503,14 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->maxPoints);
}
}
target->numOfRows++;
(*iter2)++;
}
(*iter2)++;
if (key1 == key2) (*iter1)++;
}
ASSERT(target->numOfRows <= target->maxPoints);
}
}

11
src/common/src/tglobal.c
View File

@ -120,6 +120,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@ -786,6 +787,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "update";
cfg.ptr = &tsUpdate;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_UPDATE;
cfg.maxValue = TSDB_MAX_DB_UPDATE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mqttHostName";
cfg.ptr = tsMqttHostName;
cfg.valType = TAOS_CFG_VTYPE_STRING;

48
src/dnode/src/dnodeVRead.c
View File

@ -92,33 +92,23 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
pHead->vgId = htonl(pHead->vgId);
pHead->contLen = htonl(pHead->contLen);
taos_queue queue = vnodeAcquireRqueue(pHead->vgId);
if (queue == NULL) {
leftLen -= pHead->contLen;
pCont -= pHead->contLen;
continue;
void *pVnode = vnodeAcquire(pHead->vgId);
if (pVnode != NULL) {
int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
vnodeRelease(pVnode);
}
// put message into queue
SVReadMsg *pRead = taosAllocateQitem(sizeof(SVReadMsg));
pRead->rpcMsg = *pMsg;
pRead->pCont = pCont;
pRead->contLen = pHead->contLen;
// next vnode
leftLen -= pHead->contLen;
pCont -= pHead->contLen;
queuedMsgNum++;
taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
}
if (queuedMsgNum == 0) {
SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
rpcSendResponse(&rpcRsp);
rpcFreeCont(pMsg->pCont);
}
rpcFreeCont(pMsg->pCont);
}
void *dnodeAllocVReadQueue(void *pVnode) {
@ -162,50 +152,48 @@ void dnodeFreeVReadQueue(void *rqueue) {
void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) {
SRpcMsg rpcRsp = {
.handle = pRead->rpcMsg.handle,
.handle = pRead->rpcHandle,
.pCont = pRead->rspRet.rsp,
.contLen = pRead->rspRet.len,
.code = code,
};
rpcSendResponse(&rpcRsp);
rpcFreeCont(pRead->rpcMsg.pCont);
vnodeRelease(pVnode);
}
void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) {
rpcFreeCont(pRead->rpcMsg.pCont);
vnodeRelease(pVnode);
}
static void *dnodeProcessReadQueue(void *param) {
SVReadMsg *pReadMsg;
SVReadMsg *pRead;
int32_t qtype;
void * pVnode;
while (1) {
if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pReadMsg, &pVnode) == 0) {
if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) {
dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset);
break;
}
dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle,
taosMsg[pReadMsg->rpcMsg.msgType], qtype, pReadMsg);
dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead,
taosMsg[pRead->msgType], qtype);
int32_t code = vnodeProcessRead(pVnode, pReadMsg);
int32_t code = vnodeProcessRead(pVnode, pRead);
if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
dnodeSendRpcVReadRsp(pVnode, pReadMsg, code);
dnodeSendRpcVReadRsp(pVnode, pRead, code);
} else {
if (code == TSDB_CODE_QRY_HAS_RSP) {
dnodeSendRpcVReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code);
dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code);
} else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5));
dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
dnodeDispatchNonRspMsg(pVnode, pRead, code);
}
}
taosFreeQitem(pReadMsg);
taosFreeQitem(pRead);
}
return NULL;

5
src/inc/taosdef.h
View File

@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_DATA_SMALLINT_NULL 0x8000
#define TSDB_DATA_INT_NULL 0x80000000L
#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L
#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN
@ -363,6 +364,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_DB_UPDATE 0
#define TSDB_MAX_DB_UPDATE 1
#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second

5
src/inc/taosmsg.h
View File

@ -544,6 +544,8 @@ typedef struct {
int8_t replications;
int8_t quorum;
int8_t ignoreExist;
int8_t update;
int8_t reserve[9];
} SCreateDbMsg, SAlterDbMsg;
typedef struct {
@ -654,7 +656,8 @@ typedef struct {
int8_t replications;
int8_t wals;
int8_t quorum;
int8_t reserved[16];
int8_t update;
int8_t reserved[15];
} SVnodeCfg;
typedef struct {

1
src/inc/tsdb.h
View File

@ -65,6 +65,7 @@ typedef struct {
int32_t maxRowsPerFileBlock; // maximum rows per file block
int8_t precision;
int8_t compression;
int8_t update;
} STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS

217
src/inc/ttokendef.h
View File

@ -114,114 +114,115 @@
#define TK_FSYNC 95
#define TK_COMP 96
#define TK_PRECISION 97
#define TK_LP 98
#define TK_RP 99
#define TK_TAGS 100
#define TK_USING 101
#define TK_AS 102
#define TK_COMMA 103
#define TK_NULL 104
#define TK_SELECT 105
#define TK_UNION 106
#define TK_ALL 107
#define TK_FROM 108
#define TK_VARIABLE 109
#define TK_INTERVAL 110
#define TK_FILL 111
#define TK_SLIDING 112
#define TK_ORDER 113
#define TK_BY 114
#define TK_ASC 115
#define TK_DESC 116
#define TK_GROUP 117
#define TK_HAVING 118
#define TK_LIMIT 119
#define TK_OFFSET 120
#define TK_SLIMIT 121
#define TK_SOFFSET 122
#define TK_WHERE 123
#define TK_NOW 124
#define TK_RESET 125
#define TK_QUERY 126
#define TK_ADD 127
#define TK_COLUMN 128
#define TK_TAG 129
#define TK_CHANGE 130
#define TK_SET 131
#define TK_KILL 132
#define TK_CONNECTION 133
#define TK_STREAM 134
#define TK_COLON 135
#define TK_ABORT 136
#define TK_AFTER 137
#define TK_ATTACH 138
#define TK_BEFORE 139
#define TK_BEGIN 140
#define TK_CASCADE 141
#define TK_CLUSTER 142
#define TK_CONFLICT 143
#define TK_COPY 144
#define TK_DEFERRED 145
#define TK_DELIMITERS 146
#define TK_DETACH 147
#define TK_EACH 148
#define TK_END 149
#define TK_EXPLAIN 150
#define TK_FAIL 151
#define TK_FOR 152
#define TK_IGNORE 153
#define TK_IMMEDIATE 154
#define TK_INITIALLY 155
#define TK_INSTEAD 156
#define TK_MATCH 157
#define TK_KEY 158
#define TK_OF 159
#define TK_RAISE 160
#define TK_REPLACE 161
#define TK_RESTRICT 162
#define TK_ROW 163
#define TK_STATEMENT 164
#define TK_TRIGGER 165
#define TK_VIEW 166
#define TK_COUNT 167
#define TK_SUM 168
#define TK_AVG 169
#define TK_MIN 170
#define TK_MAX 171
#define TK_FIRST 172
#define TK_LAST 173
#define TK_TOP 174
#define TK_BOTTOM 175
#define TK_STDDEV 176
#define TK_PERCENTILE 177
#define TK_APERCENTILE 178
#define TK_LEASTSQUARES 179
#define TK_HISTOGRAM 180
#define TK_DIFF 181
#define TK_SPREAD 182
#define TK_TWA 183
#define TK_INTERP 184
#define TK_LAST_ROW 185
#define TK_RATE 186
#define TK_IRATE 187
#define TK_SUM_RATE 188
#define TK_SUM_IRATE 189
#define TK_AVG_RATE 190
#define TK_AVG_IRATE 191
#define TK_TBID 192
#define TK_SEMI 193
#define TK_NONE 194
#define TK_PREV 195
#define TK_LINEAR 196
#define TK_IMPORT 197
#define TK_METRIC 198
#define TK_TBNAME 199
#define TK_JOIN 200
#define TK_METRICS 201
#define TK_STABLE 202
#define TK_INSERT 203
#define TK_INTO 204
#define TK_VALUES 205
#define TK_UPDATE 98
#define TK_LP 99
#define TK_RP 100
#define TK_TAGS 101
#define TK_USING 102
#define TK_AS 103
#define TK_COMMA 104
#define TK_NULL 105
#define TK_SELECT 106
#define TK_UNION 107
#define TK_ALL 108
#define TK_FROM 109
#define TK_VARIABLE 110
#define TK_INTERVAL 111
#define TK_FILL 112
#define TK_SLIDING 113
#define TK_ORDER 114
#define TK_BY 115
#define TK_ASC 116
#define TK_DESC 117
#define TK_GROUP 118
#define TK_HAVING 119
#define TK_LIMIT 120
#define TK_OFFSET 121
#define TK_SLIMIT 122
#define TK_SOFFSET 123
#define TK_WHERE 124
#define TK_NOW 125
#define TK_RESET 126
#define TK_QUERY 127
#define TK_ADD 128
#define TK_COLUMN 129
#define TK_TAG 130
#define TK_CHANGE 131
#define TK_SET 132
#define TK_KILL 133
#define TK_CONNECTION 134
#define TK_STREAM 135
#define TK_COLON 136
#define TK_ABORT 137
#define TK_AFTER 138
#define TK_ATTACH 139
#define TK_BEFORE 140
#define TK_BEGIN 141
#define TK_CASCADE 142
#define TK_CLUSTER 143
#define TK_CONFLICT 144
#define TK_COPY 145
#define TK_DEFERRED 146
#define TK_DELIMITERS 147
#define TK_DETACH 148
#define TK_EACH 149
#define TK_END 150
#define TK_EXPLAIN 151
#define TK_FAIL 152
#define TK_FOR 153
#define TK_IGNORE 154
#define TK_IMMEDIATE 155
#define TK_INITIALLY 156
#define TK_INSTEAD 157
#define TK_MATCH 158
#define TK_KEY 159
#define TK_OF 160
#define TK_RAISE 161
#define TK_REPLACE 162
#define TK_RESTRICT 163
#define TK_ROW 164
#define TK_STATEMENT 165
#define TK_TRIGGER 166
#define TK_VIEW 167
#define TK_COUNT 168
#define TK_SUM 169
#define TK_AVG 170
#define TK_MIN 171
#define TK_MAX 172
#define TK_FIRST 173
#define TK_LAST 174
#define TK_TOP 175
#define TK_BOTTOM 176
#define TK_STDDEV 177
#define TK_PERCENTILE 178
#define TK_APERCENTILE 179
#define TK_LEASTSQUARES 180
#define TK_HISTOGRAM 181
#define TK_DIFF 182
#define TK_SPREAD 183
#define TK_TWA 184
#define TK_INTERP 185
#define TK_LAST_ROW 186
#define TK_RATE 187
#define TK_IRATE 188
#define TK_SUM_RATE 189
#define TK_SUM_IRATE 190
#define TK_AVG_RATE 191
#define TK_AVG_IRATE 192
#define TK_TBID 193
#define TK_SEMI 194
#define TK_NONE 195
#define TK_PREV 196
#define TK_LINEAR 197
#define TK_IMPORT 198
#define TK_METRIC 199
#define TK_TBNAME 200
#define TK_JOIN 201
#define TK_METRICS 202
#define TK_STABLE 203
#define TK_INSERT 204
#define TK_INTO 205
#define TK_VALUES 206
#define TK_SPACE 300
#define TK_COMMENT 301

17
src/inc/vnode.h
View File

@ -37,10 +37,15 @@ typedef struct {
} SRspRet;
typedef struct {
SRspRet rspRet;
void * pCont;
int32_t code;
int32_t contLen;
SRpcMsg rpcMsg;
void * rpcHandle;
void * rpcAhandle;
void * qhandle;
int8_t qtype;
int8_t msgType;
SRspRet rspRet;
char pCont[];
} SVReadMsg;
typedef struct {
@ -62,13 +67,11 @@ int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
int32_t vnodeClose(int32_t vgId);
void* vnodeAcquire(int32_t vgId); // add refcount
void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue
void vnodeRelease(void *pVnode); // dec refCount
void* vnodeGetWal(void *pVnode);
int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam);
int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam);
int32_t vnodeCheckWrite(void *pVnode);
int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
void vnodeBuildStatusMsg(void *param);
void vnodeConfirmForward(void *param, uint64_t version, int32_t code);
@ -77,8 +80,8 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
int32_t vnodeInitResources();
void vnodeCleanupResources();
int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pReadMsg);
int32_t vnodeCheckRead(void *pVnode);
int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
#ifdef __cplusplus
}

3
src/mnode/inc/mnodeDef.h
View File

@ -172,7 +172,8 @@ typedef struct {
int8_t walLevel;
int8_t replications;
int8_t quorum;
int8_t reserved[12];
int8_t update;
int8_t reserved[11];
} SDbCfg;
typedef struct SDbObj {

30
src/mnode/src/mnodeDb.c
View File

@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
}
#endif
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
return TSDB_CODE_MND_INVALID_DB_OPTION;
}
return TSDB_CODE_SUCCESS;
}
@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
if (pCfg->replications < 0) pCfg->replications = tsReplications;
if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
if (pCfg->update < 0) pCfg->update = tsUpdate;
}
static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@ -391,7 +397,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
.compression = pCreate->compression,
.walLevel = pCreate->walLevel,
.replications = pCreate->replications,
.quorum = pCreate->quorum
.quorum = pCreate->quorum,
.update = pCreate->update
};
mnodeSetDefaultDbCfg(&pDb->cfg);
@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "update");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.update;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
if (pDb->status == TSDB_DB_STATUS_READY) {
const char *src = "ready";
@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
int8_t replications = pAlter->replications;
int8_t quorum = pAlter->quorum;
int8_t precision = pAlter->precision;
int8_t update = pAlter->update;
terrno = TSDB_CODE_SUCCESS;
@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
newCfg.quorum = quorum;
}
if (update >= 0 && update != pDb->cfg.update) {
#if 0
mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update);
newCfg.update = update;
#else
mError("db:%s, can't alter update option", pDb->name);
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
#endif
}
return newCfg;
}

1
src/mnode/src/mnodeVgroup.c
View File

@ -850,6 +850,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
pCfg->replications = (int8_t) pVgroup->numOfVnodes;
pCfg->wals = 3;
pCfg->quorum = pDb->cfg.quorum;
pCfg->update = pDb->cfg.update;
SVnodeDesc *pNodes = pVnode->nodes;
for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {

1
src/query/inc/qSqlparser.h
View File

@ -129,6 +129,7 @@ typedef struct SCreateDBInfo {
int32_t compressionLevel;
SStrToken precision;
bool ignoreExists;
int8_t update;
tVariantList *keep;
} SCreateDBInfo;

3
src/query/inc/sql.y
View File

@ -239,6 +239,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; }
update(Y) ::= UPDATE INTEGER(X). { Y = X; }
%type db_optr {SCreateDBInfo}
db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type alter_db_optr {SCreateDBInfo}
alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
typename(A) ::= ids(X). {

2
src/query/inc/tsqlfunction.h
View File

@ -149,7 +149,7 @@ typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value
bool initialized; // output buffer has been initialized
bool complete; // query has completed
uint16_t numOfRes; // num of output result in current buffer
uint32_t numOfRes; // num of output result in current buffer
} SResultRowCellInfo;
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))

25
src/query/src/qAst.c
View File

@ -32,7 +32,6 @@
#include "tstoken.h"
#include "ttokendef.h"
#include "tulog.h"
#include "tutil.h"
/*
*
@ -415,9 +414,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
}
if (cond.start != NULL) {
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC);
} else {
iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC);
iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC);
}
if (cond.start != NULL) {
@ -432,7 +431,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
break;
}
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
} else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
@ -450,7 +449,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_GREATER) {
continue;
} else {
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false;
}
@ -465,14 +464,14 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue;
}
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
tSkipListDestroyIter(iter);
comp = true;
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC);
while(tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
@ -480,7 +479,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue;
}
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
@ -504,7 +503,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_LESS) {
continue;
} else {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false; // no need to compare anymore
}
@ -518,7 +517,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
(pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
}
@ -682,7 +681,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
if (filterItem(pExpr, pNode, param)) {
taosArrayPush(pResult, SL_GET_NODE_DATA(pNode));
taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode)));
}
}
tSkipListDestroyIter(iter);
@ -697,7 +696,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
SSkipListNode *pNode = tSkipListIterGet(iter);
char * pData = SL_GET_NODE_DATA(pNode);
tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
tstr *name = (tstr*) tsdbGetTableName((void*) pData);
// todo speed up by using hash
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
@ -711,7 +710,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
}
if (addToResult) {
STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(res, &info);
}
}

3
src/query/src/qExecutor.c
View File

@ -1609,7 +1609,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t));
pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);//calloc(1, sizeof(SResultRow));
pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);
if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) {
goto _clean;
@ -1745,6 +1745,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
taosTFree(pRuntimeEnv->keyBuf);
taosTFree(pRuntimeEnv->rowCellInfoOffset);
taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
pRuntimeEnv->pResultRowHashTable = NULL;

2
src/query/src/qFill.c
View File

@ -170,7 +170,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
if (pFillInfo->order != TSDB_ORDER_ASC) {
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}

2
src/query/src/qHistogram.c
View File

@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
(*pHisto)->numOfEntries += 1;
}
} else { /* insert a new slot */
if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) {
if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
if (idx > 0) {
assert((*pHisto)->elems[idx - 1].val <= val);
}

1
src/query/src/qParserImpl.c
View File

@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
pDBInfo->quorum = -1;
pDBInfo->keep = NULL;
pDBInfo->update = -1;
memset(&pDBInfo->precision, 0, sizeof(SStrToken));
}

1
src/query/src/qTokenizer.c
View File

@ -155,6 +155,7 @@ static SKeyword keywordTable[] = {
{"INSERT", TK_INSERT},
{"INTO", TK_INTO},
{"VALUES", TK_VALUES},
{"UPDATE", TK_UPDATE},
{"RESET", TK_RESET},
{"QUERY", TK_QUERY},
{"ADD", TK_ADD},

2328
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

2
src/rpc/src/rpcMain.c
View File

@ -1622,7 +1622,7 @@ static void rpcDecRef(SRpcInfo *pRpc)
int count = atomic_sub_fetch_32(&tsRpcNum, 1);
if (count == 0) {
taosCloseRef(tsRpcRefId);
// taosCloseRef(tsRpcRefId);
// tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error
}
}

24
src/tsdb/inc/tsdbMain.h
View File

@ -320,6 +320,15 @@ typedef struct {
void* compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper;
typedef struct {
int rowsInserted;
int rowsUpdated;
int rowsDeleteSucceed;
int rowsDeleteFailed;
int nOperations;
TSKEY keyFirst;
TSKEY keyLast;
} SMergeInfo;
// ------------------ tsdbScan.c
typedef struct {
SFileGroup fGroup;
@ -422,7 +431,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
// ------------------ tsdbMemTable.c
int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
@ -430,7 +439,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem)
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
TSKEY* filterKeys, int nFilterKeys);
TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL;
@ -438,16 +447,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
SSkipListNode* node = tSkipListIterGet(pIter);
if (node == NULL) return NULL;
return *(SDataRow *)SL_GET_NODE_DATA(node);
return (SDataRow)SL_GET_NODE_DATA(node);
}
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return -1;
if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
return dataRowKey(row);
}
static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return TKEY_NULL;
return dataRowTKey(row);
}
static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
ASSERT(pRepo != NULL);
if (pRepo->mem == NULL) return NULL;

7
src/tsdb/src/tsdbMain.c
View File

@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
}
}
// update check
if (pCfg->update != 0) pCfg->update = 1;
return 0;
_err:
@ -762,7 +765,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY
return -1;
}
if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1;
if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
(*affectedrows)++;
points++;
@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
tlen += taosEncodeFixedI8(buf, pCfg->precision);
tlen += taosEncodeFixedI8(buf, pCfg->compression);
tlen += taosEncodeFixedI8(buf, pCfg->update);
return tlen;
}
@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
buf = taosDecodeFixedI8(buf, &(pCfg->precision));
buf = taosDecodeFixedI8(buf, &(pCfg->compression));
buf = taosDecodeFixedI8(buf, &(pCfg->update));
return buf;
}

232
src/tsdb/src/tsdbMemTable.c
View File

@ -32,43 +32,40 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
// ---------------- INTERNAL FUNCTIONS ----------------
int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
STsdbCfg * pCfg = &pRepo->config;
STsdbMeta * pMeta = pRepo->tsdbMeta;
int32_t level = 0;
int32_t headSize = 0;
TKEY tkey = dataRowTKey(row);
TSKEY key = dataRowKey(row);
SMemTable * pMemTable = pRepo->mem;
STableData *pTableData = NULL;
SSkipList * pSList = NULL;
bool isRowDelete = TKEY_IS_DELETED(tkey);
if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL &&
pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) {
pTableData = pMemTable->tData[TABLE_TID(pTable)];
pSList = pTableData->pData;
if (isRowDelete) {
if (!pCfg->update) {
tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
tSkipListNewNodeInfo(pSList, &level, &headSize);
SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *));
if (pNode == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
if (key > TABLE_LASTKEY(pTable)) {
tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
return 0;
}
}
void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
if (pRow == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
free(pNode);
return -1;
}
pNode->level = level;
dataRowCpy(pRow, row);
*(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow;
// Operations above may change pRepo->mem, retake those values
ASSERT(pRepo->mem != NULL);
@ -77,7 +74,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
if (TABLE_TID(pTable) >= pMemTable->maxTables) {
if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
free(pNode);
return -1;
}
}
@ -97,7 +93,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
" to table %s while create new table data object since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
free(pNode);
return -1;
}
@ -106,24 +101,31 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
if (tSkipListPut(pTableData->pData, pNode) == NULL) {
int64_t oldSize = SL_SIZE(pTableData->pData);
if (tSkipListPut(pTableData->pData, pRow) == NULL) {
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
free(pNode);
} else {
int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
if (isRowDelete) {
if (TABLE_LASTKEY(pTable) == key) {
// TODO: need to update table last key here (may from file)
}
} else {
if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
}
if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
if (pMemTable->keyLast < key) pMemTable->keyLast = key;
pMemTable->numOfRows++;
pMemTable->numOfRows += deltaSize;
if (pTableData->keyFirst > key) pTableData->keyFirst = key;
if (pTableData->keyLast < key) pTableData->keyLast = key;
pTableData->numOfRows++;
ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
pTableData->numOfRows += deltaSize;
}
tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key);
tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
key);
return 0;
}
@ -295,63 +297,124 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
return 0;
}
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
* This function load memory data until:
* 1. iterator ends
* 2. data key exceeds maxKey
* 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
* 4. operations in pCols not exceeds its max capacity if pCols is given
*
* The function tries to procceed AS MUSH AS POSSIBLE.
*/
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
TSKEY *filterKeys, int nFilterKeys) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
int numOfRows = 0;
TSKEY keyNext = 0;
TSKEY rowKey = 0;
TSKEY fKey = 0;
bool isRowDel = false;
int filterIter = 0;
SDataRow row = NULL;
if (nFilterKeys != 0) { // for filter purpose
ASSERT(filterKeys != NULL);
keyNext = tsdbNextIterKey(pIter);
if (keyNext < 0 || keyNext > maxKey) return numOfRows;
void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)));
}
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
pMergeInfo->keyFirst = INT64_MAX;
pMergeInfo->keyLast = INT64_MIN;
if (pCols) tdResetDataCols(pCols);
do {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) break;
keyNext = dataRowKey(row);
if (keyNext > maxKey) break;
bool keyFiltered = false;
if (nFilterKeys != 0) {
while (true) {
if (filterIter >= nFilterKeys) break;
if (keyNext == filterKeys[filterIter]) {
keyFiltered = true;
filterIter++;
break;
} else if (keyNext < filterKeys[filterIter]) {
break;
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
} else if (fKey > rowKey) {
if (isRowDel) {
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
}
tSkipListIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
} else {
if (isRowDel) {
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
}
}
tSkipListIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
}
}
if (!keyFiltered) {
if (numOfRows >= maxRowsToRead) break;
if (pCols) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (pSchema == NULL) {
ASSERT(0);
}
}
tdAppendDataRowToDataCol(row, pSchema, pCols);
}
numOfRows++;
}
} while (tSkipListIterNext(pIter));
return numOfRows;
return 0;
}
// ---------------- LOCAL FUNCTIONS ----------------
@ -440,8 +503,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
pTableData->keyLast = 0;
pTableData->numOfRows = 0;
pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP,
TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey);
pTableData->pData =
tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey);
if (pTableData->pData == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@ -461,7 +525,7 @@ static void tsdbFreeTableData(STableData *pTableData) {
}
}
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); }
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
static void *tsdbCommitData(void *arg) {
STsdbRepo * pRepo = (STsdbRepo *)arg;
@ -583,7 +647,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo) {
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
return 0;
}
@ -781,3 +845,19 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
return 0;
}
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendDataRowToDataCol(row, *ppSchema, pCols);
}
return 0;
}

36
src/tsdb/src/tsdbMeta.c
View File

@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable));
return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
goto _err;
}
if (pCfg->type == TSDB_CHILD_TABLE) {
@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) {
}
static char *getTagIndexKey(const void *pData) {
STable *pTable = *(STable **)pData;
STable *pTable = (STable *)pData;
STSchema *pSchema = tsdbGetTableTagSchema(pTable);
STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
}
pTable->tagVal = NULL;
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
pTable->pSuper = pSTable;
int32_t level = 0;
int32_t headSize = 0;
tSkipListPut(pSTable->pIndex, (void *)pTable);
tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
// NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
// actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
if (pNode == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
pNode->level = level;
memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
tSkipListPut(pSTable->pIndex, pNode);
if (refSuper) T_REF_INC(pSTable);
return 0;
}
@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
SSkipListNode *pNode = taosArrayGetP(res, i);
// STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
tSkipListRemoveNode(pSTable->pIndex, pNode);
}
}
@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
buf = tdDecodeSchema(buf, &(pTable->tagSchema));
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
pTable->pIndex =
tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL,
SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbFreeTable(pTable);
@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM);
} else {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1));
tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1));
} else {
tlen = sizeof(SListNode) + sizeof(SActObj);
}
@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) {
}
while (tSkipListIterNext(pIter)) {
STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable);
}
@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) {
tsdbWLockRepoMeta(pRepo);
while (tSkipListIterNext(pIter)) {
STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
tsdbRemoveTableFromMeta(pRepo, tTable, false, false);
}

302
src/tsdb/src/tsdbRWHelper.c
View File

@ -25,6 +25,7 @@
#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
#define TSDB_KEY_COL_OFFSET 0
#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
@ -32,7 +33,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD
static int compareKeyBlock(const void *arg1, const void *arg2);
static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
static int tsdbInitHelperFile(SRWHelper *pHelper);
@ -59,8 +60,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx);
static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows);
static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows, int8_t update);
static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
// ---------------------- INTERNAL FUNCTIONS ----------------------
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
@ -277,7 +280,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols
while (true) {
ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
if (keyFirst < 0 || keyFirst > maxKey) break; // iter over
if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over
if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
@ -923,7 +926,7 @@ _err:
return -1;
}
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) {
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
ASSERT(pCompBlock->numOfSubBlocks == 0);
SCompIdx *pIdx = &(pHelper->curCompIdx);
@ -956,9 +959,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
pSCompBlock->numOfSubBlocks++;
ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
pSCompBlock->len += sizeof(SCompBlock);
pSCompBlock->numOfRows += rowsAdded;
pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->keyFirst = pMergeInfo->keyFirst;
pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += sizeof(SCompBlock);
} else { // Need to create two sub-blocks
void *ptr = NULL;
@ -987,11 +990,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
((SCompBlock *)ptr)[1] = *pCompBlock;
pSCompBlock->numOfSubBlocks = 2;
pSCompBlock->numOfRows += rowsAdded;
pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
pSCompBlock->len = sizeof(SCompBlock) * 2;
pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast);
pSCompBlock->keyFirst = pMergeInfo->keyFirst;
pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += (sizeof(SCompBlock) * 2);
}
@ -1045,6 +1048,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
return 0;
}
static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
SCompIdx *pCompIdx = &(pHelper->curCompIdx);
ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
SCompBlock compBlock = *pCompBlock;
ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
if (pCompIdx->numOfBlocks == 1) {
memset(pCompIdx, 0, sizeof(*pCompIdx));
} else {
int tsize = 0;
if (compBlock.numOfSubBlocks > 1) {
tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
ASSERT(tsize > 0);
memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
tsize);
pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
}
tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
ASSERT(tsize > 0);
memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
pCompIdx->len -= sizeof(SCompBlock);
pCompIdx->numOfBlocks--;
pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
}
return 0;
}
static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
pHelper->idxH.numOfIdx = 0;
pHelper->idxH.curIdx = 0;
@ -1443,20 +1485,28 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SCompBlock compBlock = {0};
SMergeInfo mergeInfo = {0};
SMergeInfo *pMergeInfo = &mergeInfo;
ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
if (pIdx->hasLast) { // append to with last block
ASSERT(pIdx->len > 0);
SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows,
pDataCols, NULL, 0);
ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
NULL, 0, pCfg->update, pMergeInfo);
ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (pDataCols->numOfRows > 0) {
ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
@ -1469,19 +1519,22 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
}
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
}
} else {
ASSERT(!pHelper->hasOldLastBlock);
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0);
ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (pDataCols->numOfRows > 0) {
ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
}
}
#ifndef NDEBUG
TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
ASSERT(keyNext < 0 || keyNext > pIdx->maxKey);
ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
#endif
return 0;
@ -1495,7 +1548,10 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
SCompBlock compBlock = {0};
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SDataCols *pDataCols0 = pHelper->pDataCols[0];
SDataCols * pDataCols0 = pHelper->pDataCols[0];
SMergeInfo mergeInfo = {0};
SMergeInfo *pMergeInfo = &mergeInfo;
SCompBlock oBlock = {0};
SSkipListIterator slIter = {0};
@ -1505,112 +1561,75 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
ASSERT(pCompBlock != NULL);
int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
oBlock = *pCompBlock;
if (pCompBlock->last) {
ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1);
int16_t colId = 0;
slIter = *(pCommitIter->pIter);
if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows);
ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
int rows1 = defaultRowsInBlock - pCompBlock->numOfRows;
int rows2 =
tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows);
if (rows2 == 0) { // all data filtered out
*(pCommitIter->pIter) = slIter;
} else {
if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols,
pDataCols0->cols[0].pData, pDataCols0->numOfRows);
ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
tblkIdx++;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
int round = 0;
int dIter = 0;
if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
while (true) {
tdResetDataCols(pDataCols);
int rowsRead =
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock);
if (rowsRead == 0) break;
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
pCfg->update, pMergeInfo);
ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (pDataCols->numOfRows == 0) break;
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (round == 0) {
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
tblkIdx++;
round++;
}
}
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
}
} else {
TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1);
TSKEY blkKeyFirst = pCompBlock->keyFirst;
TSKEY blkKeyLast = pCompBlock->keyLast;
if (keyFirst < blkKeyFirst) {
while (true) {
tdResetDataCols(pDataCols);
int rowsRead =
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0);
if (rowsRead == 0) break;
ASSERT(rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
tblkIdx++;
}
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else {
ASSERT(keyFirst <= blkKeyLast);
int16_t colId = 0;
if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
slIter = *(pCommitIter->pIter);
int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows);
int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData,
pDataCols0->numOfRows);
tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
pCfg->update, pMergeInfo);
if (rows2 == 0) { // all filtered out
if (pMergeInfo->nOperations == 0) {
// Do nothing
ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
*(pCommitIter->pIter) = slIter;
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else {
int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2;
if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) {
int rows = (rows1 >= rows3) ? rows3 : rows2;
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols,
pDataCols0->cols[0].pData, pDataCols0->numOfRows);
ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0)
return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
tblkIdx++;
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
// Delete the block and do some stuff
ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
*pCommitIter->pIter = slIter;
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
} else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
// Append as a sub-block of the searched block
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
pDataCols0->numOfRows, pCfg->update, pMergeInfo);
ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
&compBlock, oBlock.last, false) < 0) {
return -1;
}
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
return -1;
}
tblkIdx++;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
// load the block data, merge with the memory data
if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
int round = 0;
int dIter = 0;
while (true) {
int rowsRead =
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock);
if (rowsRead == 0) break;
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
pCfg->update);
if (pDataCols->numOfRows == 0) break;
if (tblkIdx == pIdx->numOfBlocks - 1) {
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
} else {
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
}
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0)
return -1;
if (round == 0) {
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
@ -1619,10 +1638,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
round++;
tblkIdx++;
}
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
}
}
}
}
@ -1630,9 +1645,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
return 0;
}
static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows) {
int numOfRows = 0;
static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows, int8_t update) {
TSKEY key1 = INT64_MAX;
TSKEY key2 = INT64_MAX;
STSchema *pSchema = NULL;
@ -1642,35 +1656,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte
while (true) {
key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
bool isRowDel = false;
SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row);
if (row == NULL || dataRowKey(row) > maxKey) {
key2 = INT64_MAX;
} else {
key2 = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
if (key1 == INT64_MAX && key2 == INT64_MAX) break;
if (key1 <= key2) {
if (key1 < key2) {
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
pTarget->numOfRows++;
(*iter)++;
if (key1 == key2) tSkipListIterNext(pCommitIter->pIter);
} else {
} else if (key1 > key2) {
if (!isRowDel) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL);
}
tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
tSkipListIterNext(pCommitIter->pIter);
} else {
if (update) {
if (!isRowDel) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL);
}
tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
} else {
ASSERT(!isRowDel);
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
pTarget->numOfRows++;
}
(*iter)++;
tSkipListIterNext(pCommitIter->pIter);
}
numOfRows++;
if (numOfRows >= maxRows) break;
ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints);
if (pTarget->numOfRows >= maxRows) break;
}
return numOfRows;
}
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
@ -1693,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols,
return 0;
}
static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
STsdbCfg *pCfg = &(pHelper->pRepo->config);
int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
ASSERT(mergeRows > 0);
if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
if (pCompBlock->last) {
if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
} else {
if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
}
}
return false;
}

57
src/tsdb/src/tsdbRead.c
View File

@ -457,7 +457,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
assert(node != NULL);
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@ -479,7 +479,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
assert(node != NULL);
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@ -504,19 +504,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iiter);
}
static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) {
static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
SDataRow rmem = NULL, rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
rmem = *(SDataRow *)SL_GET_NODE_DATA(node);
rmem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = *(SDataRow *)SL_GET_NODE_DATA(node);
rimem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
@ -538,9 +538,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
TSKEY r2 = dataRowKey(rimem);
if (r1 == r2) { // data ts are duplicated, ignore the data in mem
if (!update) {
tSkipListIterNext(pCheckInfo->iter);
pCheckInfo->chosen = 1;
return rimem;
} else {
tSkipListIterNext(pCheckInfo->iiter);
pCheckInfo->chosen = 0;
return rmem;
}
} else {
if (ASCENDING_TRAVERSE(order)) {
if (r1 < r2) {
@ -594,6 +600,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
STsdbCfg *pCfg = &pHandle->pTsdb->config;
size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
pHandle->cur.fid = -1;
@ -607,7 +614,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
initTableMemIterator(pHandle, pCheckInfo);
}
SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
if (row == NULL) {
return false;
}
@ -827,11 +834,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur;
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
int32_t code = TSDB_CODE_SUCCESS;
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
assert(cur->pos >= 0 && cur->pos <= binfo.rows);
@ -1316,6 +1324,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
initTableMemIterator(pQueryHandle, pCheckInfo);
@ -1353,7 +1362,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL;
do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@ -1383,7 +1392,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
if (pCfg->update) {
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
}
cur->win.ekey = key;
cur->lastKey = key + step;
cur->mixBlock = true;
moveToNextRowInMem(pCheckInfo);
pos += step;
} else {
moveToNextRowInMem(pCheckInfo);
}
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) {
@ -1394,7 +1418,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
assert(end != -1);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
if (!pCfg->update) {
moveToNextRowInMem(pCheckInfo);
} else {
end -= step;
}
}
int32_t qstart = 0, qend = 0;
@ -1414,8 +1442,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
* copy them all to result buffer, since it may be overlapped with file data block.
*/
if (node == NULL ||
((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
// no data in cache or data in cache is greater than the ekey of time window, load data from file block
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos];
@ -1862,13 +1890,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj;
do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@ -1919,9 +1948,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode);
STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(list, &info);
}
@ -2464,7 +2493,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
static bool indexedNodeFilterFp(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*) param;
STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
char* val = NULL;

2
src/util/CMakeLists.txt
View File

@ -8,7 +8,7 @@ TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(tutil m rt)
ADD_SUBDIRECTORY(tests)
# ADD_SUBDIRECTORY(tests)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)

171
src/util/inc/tskiplist.h
View File

@ -27,33 +27,25 @@ extern "C" {
#define MAX_SKIP_LIST_LEVEL 15
#define SKIP_LIST_RECORD_PERFORMANCE 0
// For key property setting
#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage)
#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case)
#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case)
// For thread safety setting
#define SL_THREAD_SAFE (uint8_t)0x4
typedef char *SSkipListKey;
typedef char *(*__sl_key_fn_t)(const void *);
/**
* the skip list node is located in a consecutive memory area,
* the format of skip list node is as follows:
* +------------+-----------------------+------------------------+-----+------+
* | node level | forward pointer array | backward pointer array | key | data |
* +------------+-----------------------+------------------------+-----+------+
*/
typedef struct SSkipListNode {
uint8_t level;
void * pData;
struct SSkipListNode *forwards[];
} SSkipListNode;
#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
#define SL_GET_BACKWARD_POINTER(n, _l) \
((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
#define SL_GET_NODE_DATA(n) (n)->pData
#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)]
#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
/*
* @version 0.3
@ -103,21 +95,16 @@ typedef struct tSkipListState {
uint64_t nTotalElapsedTimeForInsert;
} tSkipListState;
typedef struct SSkipListKeyInfo {
uint8_t dupKey : 2; // if allow duplicated key in the skip list
uint8_t type : 4; // key type
uint8_t freeNode:2; // free node when destroy the skiplist
uint8_t len; // maximum key length, used in case of string key
} SSkipListKeyInfo;
typedef struct SSkipList {
__compar_fn_t comparFn;
__sl_key_fn_t keyFn;
uint32_t size;
uint8_t maxLevel;
uint8_t level;
SSkipListKeyInfo keyInfo;
pthread_rwlock_t *lock;
uint16_t len;
uint8_t maxLevel;
uint8_t flags;
uint8_t type; // static info above
uint8_t level;
uint32_t size;
SSkipListNode * pHead; // point to the first element
SSkipListNode * pTail; // point to the last element
#if SKIP_LIST_RECORD_PERFORMANCE
@ -125,12 +112,6 @@ typedef struct SSkipList {
#endif
} SSkipList;
/*
* iterate the skiplist
* this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
* continue iterating the skiplist, so add the reference count for skiplist
* TODO add the ref for skip list when one iterator is created
*/
typedef struct SSkipListIterator {
SSkipList * pSkipList;
SSkipListNode *cur;
@ -139,113 +120,25 @@ typedef struct SSkipListIterator {
SSkipListNode *next; // next points to the true qualified node in skip list
} SSkipListIterator;
/**
*
* @param nMaxLevel maximum skip list level
* @param keyType type of key
* @param dupKey allow the duplicated key in the skip list
* @return
*/
SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
uint8_t freeNode, __sl_key_fn_t fn);
#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE)
#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1))
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
#define SL_SIZE(s) (s)->size
/**
*
* @param pSkipList
* @return NULL will always be returned
*/
void *tSkipListDestroy(SSkipList *pSkipList);
/**
*
* @param pSkipList
* @param level
* @param headSize
*/
void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
/**
* put the skip list node into the skip list.
* If failed, NULL will be returned, otherwise, the pNode will be returned.
*
* @param pSkipList
* @param pNode
* @return
*/
SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
/**
* get *all* nodes which key are equivalent to pKey
*
* @param pSkipList
* @param pKey
* @return
*/
SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
/**
* get the size of skip list
* @param pSkipList
* @return
*/
size_t tSkipListGetSize(const SSkipList *pSkipList);
/**
* display skip list of the given level, for debug purpose only
* @param pSkipList
* @param nlevel
*/
SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
__sl_key_fn_t fn);
void tSkipListDestroy(SSkipList *pSkipList);
SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData);
SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
/**
* create skiplist iterator
* @param pSkipList
* @return
*/
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
/**
* create skip list iterator from the given node and specified the order
* @param pSkipList
* @param pNode start position, instead of the first node in skip list
* @param order traverse order of the iterator
* @return
*/
SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order);
/**
* forward the skip list iterator
* @param iter
* @return
*/
SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
bool tSkipListIterNext(SSkipListIterator *iter);
/**
* get the element of skip list node
* @param iter
* @return
*/
SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
/**
* destroy the skip list node
* @param iter
* @return
*/
void *tSkipListDestroyIter(SSkipListIterator *iter);
/*
* remove nodes of the pKey value.
* If more than one node has the same value, all will be removed
*
* @Return
* the count of removed nodes
*/
SSkipListNode * tSkipListIterGet(SSkipListIterator *iter);
void * tSkipListDestroyIter(SSkipListIterator *iter);
uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
/*
* remove the specified node in parameters
*/
void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
#ifdef __cplusplus

9
src/util/src/tcompare.c
View File

@ -367,7 +367,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE);
tstr* t1 = (tstr*) f1;
tstr* t2 = (tstr*) f2;
if (t1->len != t2->len) {
return t1->len > t2->len? 1:-1;
}
int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE);
if (ret == 0) {
return ret;
}

964
src/util/src/tskiplist.c
View File

File diff suppressed because it is too large Load Diff

4
src/util/tests/skiplistTest.cpp
View File

@ -247,7 +247,7 @@ void skiplistPerformanceTest() {
printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size);
printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
assert(tSkipListGetSize(pSkipList) == size);
assert(SL_GET_SIZE(pSkipList) == size);
// printf("the level of skiplist is:\n");
//
@ -273,7 +273,7 @@ void skiplistPerformanceTest() {
int64_t et = taosGetTimestampMs();
printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
assert(tSkipListGetSize(pSkipList) == size);
assert(SL_GET_SIZE(pSkipList) == size);
tSkipListDestroy(pSkipList);
taosTFree(total);

16
src/vnode/src/vnodeMain.c
View File

@ -140,6 +140,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression;
tsdbCfg.update = pVnodeCfg->cfg.update;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
@ -468,21 +469,6 @@ void *vnodeAcquire(int32_t vgId) {
return *ppVnode;
}
void *vnodeAcquireRqueue(int32_t vgId) {
SVnodeObj *pVnode = vnodeAcquire(vgId);
if (pVnode == NULL) return NULL;
int32_t code = vnodeCheckRead(pVnode);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]);
vnodeRelease(pVnode);
return NULL;
}
return pVnode->rqueue;
}
void *vnodeGetWal(void *pVnode) {
return ((SVnodeObj *)pVnode)->wal;
}

95
src/vnode/src/vnodeRead.c
View File

@ -46,7 +46,7 @@ void vnodeInitReadFp(void) {
//
int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) {
SVnodeObj *pVnode = (SVnodeObj *)param;
int msgType = pReadMsg->rpcMsg.msgType;
int32_t msgType = pReadMsg->msgType;
if (vnodeProcessReadMsgFp[msgType] == NULL) {
vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
@ -56,7 +56,7 @@ int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) {
return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
}
int32_t vnodeCheckRead(void *param) {
static int32_t vnodeCheckRead(void *param) {
SVnodeObj *pVnode = param;
if (pVnode->status != TAOS_VN_STATUS_READY) {
vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
@ -78,24 +78,58 @@ int32_t vnodeCheckRead(void *param) {
return TSDB_CODE_SUCCESS;
}
static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) {
SVnodeObj *pVnode = vparam;
if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) {
int32_t code = vnodeCheckRead(pVnode);
if (code != TSDB_CODE_SUCCESS) return code;
}
SVReadMsg *pRead = (SVReadMsg *)taosAllocateQitem(sizeof(SVReadMsg));
pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
pRead->pCont = qhandle;
pRead->contLen = 0;
pRead->rpcMsg.ahandle = ahandle;
int32_t size = sizeof(SVReadMsg) + contLen;
SVReadMsg *pRead = taosAllocateQitem(size);
if (pRead == NULL) {
return TSDB_CODE_VND_OUT_OF_MEMORY;
}
if (rparam != NULL) {
SRpcMsg *pRpcMsg = rparam;
pRead->rpcHandle = pRpcMsg->handle;
pRead->rpcAhandle = pRpcMsg->ahandle;
pRead->msgType = pRpcMsg->msgType;
pRead->code = pRpcMsg->code;
}
if (contLen != 0) {
pRead->contLen = contLen;
memcpy(pRead->pCont, pCont, contLen);
} else {
pRead->qhandle = pCont;
}
pRead->qtype = qtype;
atomic_add_fetch_32(&pVnode->refCount, 1);
vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead);
taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
taosWriteQitem(pVnode->rqueue, qtype, pRead);
return TSDB_CODE_SUCCESS;
}
static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
SRpcMsg rpcMsg = {0};
rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
rpcMsg.ahandle = ahandle;
int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
if (code == TSDB_CODE_SUCCESS) {
vDebug("QInfo:%p add to vread queue for exec query", *qhandle);
}
return code;
}
/**
*
* @param pRet response message object
@ -155,18 +189,18 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
memset(pRet, 0, sizeof(SRspRet));
// qHandle needs to be freed correctly
if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle);
assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1);
void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
if (qhandle == NULL || *qhandle == NULL) {
vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
pReadMsg->rpcMsg.handle);
pReadMsg->rpcHandle);
} else {
assert(*qhandle == (void *)killQueryMsg->qhandle);
@ -208,9 +242,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
if (handle != NULL &&
vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
pReadMsg->rpcMsg.handle);
pReadMsg->rpcHandle);
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
return pRsp->code;
@ -221,7 +255,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
if (handle != NULL) {
vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle);
code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
pRsp->code = code;
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@ -230,7 +264,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
} else {
assert(pCont != NULL);
void **qhandle = (void **)pCont;
void **qhandle = (void **)pReadMsg->qhandle;
vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
@ -242,14 +276,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
// build query rsp, the retrieve request has reached here already
if (buildRes) {
// update the connection info according to the retrieve connection
pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle);
assert(pReadMsg->rpcMsg.handle != NULL);
pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle);
assert(pReadMsg->rpcHandle != NULL);
vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
pReadMsg->rpcMsg.handle);
pReadMsg->rpcHandle);
// set the real rsp error code
pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle);
pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle);
// NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
code = TSDB_CODE_QRY_HAS_RSP;
@ -283,7 +317,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
pRetrieve->free, pReadMsg->rpcMsg.handle);
pRetrieve->free, pReadMsg->rpcHandle);
memset(pRet, 0, sizeof(SRspRet));
@ -298,7 +332,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
if (code != TSDB_CODE_SUCCESS) {
vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
vnodeBuildNoResultQueryRsp(pRet);
return code;
}
@ -314,9 +348,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
}
// register the qhandle to connect to quit query immediate if connection is broken
if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle,
pReadMsg->rpcMsg.handle);
if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle);
code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
qKillQuery(*handle);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@ -326,7 +359,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
bool freeHandle = true;
bool buildRes = false;
code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle);
if (code != TSDB_CODE_SUCCESS) {
// TODO handle malloc failure
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@ -337,7 +370,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
assert(buildRes == true);
#if _NON_BLOCKING_RETRIEVE
if (!buildRes) {
assert(pReadMsg->rpcMsg.handle != NULL);
assert(pReadMsg->rpcHandle != NULL);
qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
return TSDB_CODE_QRY_NOT_READY;
@ -345,7 +378,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
#endif
// ahandle is the sqlObj pointer
code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle);
code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle);
}
// If qhandle is not added into vread queue, the query should be completed already or paused with error.

2
src/vnode/src/vnodeWrite.c
View File

@ -97,7 +97,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
return syncCode;
}
int32_t vnodeCheckWrite(void *param) {
static int32_t vnodeCheckWrite(void *param) {
SVnodeObj *pVnode = param;
if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);

77
tests/examples/c/apitest.c
View File

@ -9,26 +9,40 @@
static void prepare_data(TAOS* taos) {
taos_query(taos, "drop database if exists test;");
TAOS_RES *result;
result = taos_query(taos, "drop database if exists test;");
taos_free_result(result);
usleep(100000);
taos_query(taos, "create database test;");
result = taos_query(taos, "create database test;");
taos_free_result(result);
usleep(100000);
taos_select_db(taos, "test");
taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
taos_free_result(result);
taos_query(taos, "create table t0 using meters tags(0);");
taos_query(taos, "create table t1 using meters tags(1);");
taos_query(taos, "create table t2 using meters tags(2);");
taos_query(taos, "create table t3 using meters tags(3);");
taos_query(taos, "create table t4 using meters tags(4);");
taos_query(taos, "create table t5 using meters tags(5);");
taos_query(taos, "create table t6 using meters tags(6);");
taos_query(taos, "create table t7 using meters tags(7);");
taos_query(taos, "create table t8 using meters tags(8);");
taos_query(taos, "create table t9 using meters tags(9);");
result = taos_query(taos, "create table t0 using meters tags(0);");
taos_free_result(result);
result = taos_query(taos, "create table t1 using meters tags(1);");
taos_free_result(result);
result = taos_query(taos, "create table t2 using meters tags(2);");
taos_free_result(result);
result = taos_query(taos, "create table t3 using meters tags(3);");
taos_free_result(result);
result = taos_query(taos, "create table t4 using meters tags(4);");
taos_free_result(result);
result = taos_query(taos, "create table t5 using meters tags(5);");
taos_free_result(result);
result = taos_query(taos, "create table t6 using meters tags(6);");
taos_free_result(result);
result = taos_query(taos, "create table t7 using meters tags(7);");
taos_free_result(result);
result = taos_query(taos, "create table t8 using meters tags(8);");
taos_free_result(result);
result = taos_query(taos, "create table t9 using meters tags(9);");
taos_free_result(result);
TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
" ('2020-01-01 00:01:00.000', 0)"
" ('2020-01-01 00:02:00.000', 0)"
" t1 values('2020-01-01 00:00:00.000', 0)"
@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) {
" t7 values('2020-01-01 00:01:02.000', 0)"
" t8 values('2020-01-01 00:01:02.000', 0)"
" t9 values('2020-01-01 00:01:02.000', 0)");
int affected = taos_affected_rows(res);
int affected = taos_affected_rows(result);
if (affected != 18) {
printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected);
}
taos_free_result(result);
// super tables subscription
usleep(1000000);
}
@ -135,6 +150,7 @@ static void verify_query(TAOS* taos) {
res = taos_query(taos, "select * from meters");
taos_stop_query(res);
taos_free_result(res);
}
@ -153,23 +169,30 @@ static void verify_subscribe(TAOS* taos) {
res = taos_consume(tsub);
check_row_count(__LINE__, res, 0);
taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
TAOS_RES *result;
result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
taos_free_result(result);
result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 2);
taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
taos_free_result(result);
result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 2);
taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 1);
// keep progress information and restart subscription
taos_unsubscribe(tsub, 1);
taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
taos_free_result(result);
tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 24);
@ -196,7 +219,8 @@ static void verify_subscribe(TAOS* taos) {
res = taos_consume(tsub);
check_row_count(__LINE__, res, 0);
taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
taos_free_result(result);
res = taos_consume(tsub);
check_row_count(__LINE__, res, 1);
@ -205,7 +229,8 @@ static void verify_subscribe(TAOS* taos) {
int blockFetch = 0;
tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000);
usleep(2000000);
taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
taos_free_result(result);
usleep(2000000);
taos_unsubscribe(tsub, 0);
}
@ -213,8 +238,9 @@ static void verify_subscribe(TAOS* taos) {
void verify_prepare(TAOS* taos) {
TAOS_RES* result = taos_query(taos, "drop database if exists test;");
taos_free_result(result);
usleep(100000);
taos_query(taos, "create database test;");
result = taos_query(taos, "create database test;");
int code = taos_errno(result);
if (code != 0) {
@ -429,7 +455,8 @@ void verify_stream(TAOS* taos) {
NULL);
printf("waiting for stream data\n");
usleep(100000);
taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
taos_free_result(result);
usleep(200000000);
taos_close_stream(strm);
}

41
tests/examples/c/asyncdemo.c
View File

@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
void taos_select_call_back(void *param, TAOS_RES *tres, int code);
void taos_error(TAOS *taos);
static void queryDB(TAOS *taos, char *command) {
int i;
TAOS_RES *pSql = NULL;
int32_t code = -1;
for (i = 0; i < 5; i++) {
if (NULL != pSql) {
taos_free_result(pSql);
pSql = NULL;
}
pSql = taos_query(taos, command);
code = taos_errno(pSql);
if (0 == code) {
break;
}
}
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(EXIT_FAILURE);
}
taos_free_result(pSql);
}
int main(int argc, char *argv[])
{
TAOS *taos;
@ -78,16 +106,14 @@ int main(int argc, char *argv[])
printf("success to connect to server\n");
sprintf(sql, "drop database %s", db);
taos_query(taos, sql);
sprintf(sql, "drop database if exists %s", db);
queryDB(taos, sql);
sprintf(sql, "create database %s", db);
if (taos_query(taos, sql) != 0)
taos_error(taos);
queryDB(taos, sql);
sprintf(sql, "use %s", db);
if (taos_query(taos, sql) != 0)
taos_error(taos);
queryDB(taos, sql);
strcpy(prefix, "asytbl_");
for (i = 0; i < numOfTables; ++i) {
@ -95,8 +121,7 @@ int main(int argc, char *argv[])
tableList[i].taos = taos;
sprintf(tableList[i].name, "%s%d", prefix, i);
sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i);
if (taos_query(taos, sql) != 0)
taos_error(taos);
queryDB(taos, sql);
}
gettimeofday(&systemTime, NULL);

57
tests/examples/c/demo.c
View File

@ -22,6 +22,34 @@
#include <inttypes.h>
#include <taos.h> // TAOS header file
static void queryDB(TAOS *taos, char *command) {
int i;
TAOS_RES *pSql = NULL;
int32_t code = -1;
for (i = 0; i < 5; i++) {
if (NULL != pSql) {
taos_free_result(pSql);
pSql = NULL;
}
pSql = taos_query(taos, command);
code = taos_errno(pSql);
if (0 == code) {
break;
}
}
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(EXIT_FAILURE);
}
taos_free_result(pSql);
}
int main(int argc, char *argv[]) {
TAOS * taos;
char qstr[1024];
@ -44,22 +72,26 @@ int main(int argc, char *argv[]) {
printf("success to connect to server\n");
taos_query(taos, "drop database demo");
//taos_query(taos, "drop database demo");
queryDB(taos, "drop database if exists demo");
result = taos_query(taos, "create database demo");
if (result == NULL) {
printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/);
exit(1);
}
//result = taos_query(taos, "create database demo");
//if (result == NULL) {
// printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/);
// exit(1);
//}
queryDB(taos, "create database demo");
printf("success to create database\n");
taos_query(taos, "use demo");
//taos_query(taos, "use demo");
queryDB(taos, "use demo");
// create table
if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
printf("failed to create table, reason:%s\n", taos_errstr(result));
exit(1);
}
//if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
// printf("failed to create table, reason:%s\n", taos_errstr(result));
// exit(1);
//}
queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))");
printf("success to create table\n");
// sleep for one second to make sure table is created on data node
@ -80,8 +112,10 @@ int main(int argc, char *argv[]) {
printf("insert row: %i\n", i);
} else {
printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/);
taos_free_result(result);
exit(1);
}
taos_free_result(result);
//sleep(1);
}
@ -92,6 +126,7 @@ int main(int argc, char *argv[]) {
result = taos_query(taos, qstr);
if (result == NULL || taos_errno(result) != 0) {
printf("failed to select, reason:%s\n", taos_errstr(result));
taos_free_result(result);
exit(1);
}

71
tests/pytest/alter/db_update_options.py Normal file
View File

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
import random
import string
import subprocess
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdLog.debug("check database")
tdSql.prepare()
# check default update value
sql = "create database if not exists db"
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
sql = "alter database db update 1"
# check update value
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,1)
sql = "alter database db update 0"
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
sql = "alter database db update -1"
tdSql.error(sql)
sql = "alter database db update 100"
tdSql.error(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
tdSql.execute('drop database db')
tdSql.error('create database db update 100')
tdSql.error('create database db update -1')
tdSql.execute('create database db update 1')
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,1)
tdSql.execute('drop database db')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

229
tests/pytest/crash_gen/crash_gen.py → tests/pytest/crash_gen/crash_gen_main.py
View File

@ -38,9 +38,9 @@ import resource
from guppy import hpy
import gc
from .service_manager import ServiceManager, TdeInstance
from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress
from .db import DbConn, MyTDSql, DbConnNative, DbManager
from crash_gen.service_manager import ServiceManager, TdeInstance
from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress
from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager
import taos
import requests
@ -243,7 +243,7 @@ class WorkerThread:
class ThreadCoordinator:
WORKER_THREAD_TIMEOUT = 180 # one minute
WORKER_THREAD_TIMEOUT = 120 # Normal: 120
def __init__(self, pool: ThreadPool, dbManager: DbManager):
self._curStep = -1 # first step is 0
@ -388,9 +388,9 @@ class ThreadCoordinator:
self._syncAtBarrier() # For now just cross the barrier
Progress.emit(Progress.END_THREAD_STEP)
except threading.BrokenBarrierError as err:
Logging.info("Main loop aborted, caused by worker thread time-out")
Logging.info("Main loop aborted, caused by worker thread(s) time-out")
self._execStats.registerFailure("Aborted due to worker thread timeout")
print("\n\nWorker Thread time-out detected, important thread info:")
print("\n\nWorker Thread time-out detected, TAOS related threads are:")
ts = ThreadStacks()
ts.print(filterInternal=True)
workerTimeout = True
@ -435,7 +435,7 @@ class ThreadCoordinator:
Logging.debug("\r\n\n--> Main thread ready to finish up...")
Logging.debug("Main thread joining all threads")
self._pool.joinAll() # Get all threads to finish
Logging.info("\nAll worker threads finished")
Logging.info(". . . All worker threads finished") # No CR/LF before
self._execStats.endExec()
def cleanup(self): # free resources
@ -1072,17 +1072,18 @@ class Database:
t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years
t4 = datetime.datetime.fromtimestamp(
t3.timestamp() + elSec2) # see explanation above
Logging.info("Setting up TICKS to start from: {}".format(t4))
Logging.debug("Setting up TICKS to start from: {}".format(t4))
return t4
@classmethod
def getNextTick(cls):
with cls._clsLock: # prevent duplicate tick
if cls._lastLaggingTick==0:
if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized
# 10k at 1/20 chance, should be enough to avoid overlaps
cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000)
if cls._lastTick==0: # should be quite a bit into the future
cls._lastTick = cls.setupLastTick()
tick = cls.setupLastTick()
cls._lastTick = tick
cls._lastLaggingTick = tick + datetime.timedelta(0, -10000)
# if : # should be quite a bit into the future
if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick
cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds
@ -1177,6 +1178,8 @@ class Task():
instead. But a task is always associated with a DB
'''
taskSn = 100
_lock = threading.Lock()
_tableLocks: Dict[str, threading.Lock] = {}
@classmethod
def allocTaskNum(cls):
@ -1198,6 +1201,8 @@ class Task():
self._execStats = execStats
self._db = db # A task is always associated/for a specific DB
def isSuccess(self):
return self._err is None
@ -1237,6 +1242,7 @@ class Task():
0x0B, # Unable to establish connection, more details in TD-1648
0x200, # invalid SQL， TODO: re-examine with TD-934
0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776
0x213, # "Disconnected from service", result of "kill connection ???"
0x217, # "db not selected", client side defined error code
# 0x218, # "Table does not exist" client side defined error code
0x360, # Table already exists
@ -1318,7 +1324,7 @@ class Task():
self._err = err
self._aborted = True
except Exception as e:
self.logInfo("Non-TAOS exception encountered")
Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__))
self._err = e
self._aborted = True
traceback.print_exc()
@ -1351,6 +1357,24 @@ class Task():
def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread
return wt.getQueryResult()
def lockTable(self, ftName): # full table name
# print(" <<" + ftName + '_', end="", flush=True)
with Task._lock:
if not ftName in Task._tableLocks:
Task._tableLocks[ftName] = threading.Lock()
Task._tableLocks[ftName].acquire()
def unlockTable(self, ftName):
# print('_' + ftName + ">> ", end="", flush=True)
with Task._lock:
if not ftName in self._tableLocks:
raise RuntimeError("Corrupt state, no such lock")
lock = Task._tableLocks[ftName]
if not lock.locked():
raise RuntimeError("Corrupte state, already unlocked")
lock.release()
class ExecutionStats:
def __init__(self):
@ -1461,7 +1485,7 @@ class StateTransitionTask(Task):
_baseTableNumber = None
_endState = None
_endState = None # TODO: no longter used?
@classmethod
def getInfo(cls): # each sub class should supply their own information
@ -1486,7 +1510,7 @@ class StateTransitionTask(Task):
@classmethod
def getRegTableName(cls, i):
if ( StateTransitionTask._baseTableNumber is None):
if ( StateTransitionTask._baseTableNumber is None): # Set it one time
StateTransitionTask._baseTableNumber = Dice.throw(
999) if gConfig.dynamic_db_table_names else 0
return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i)
@ -1544,8 +1568,11 @@ class TaskCreateSuperTable(StateTransitionTask):
sTable = self._db.getFixedSuperTable() # type: TdSuperTable
# wt.execSql("use db") # should always be in place
sTable.create(wt.getDbConn(), self._db.getName(),
{'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'})
{'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'},
dropIfExists = True
)
# self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
# No need to create the regular tables, INSERT will do that
# automatically
@ -1558,13 +1585,40 @@ class TdSuperTable:
def getName(self):
return self._stName
def drop(self, dbc, dbName, skipCheck = False):
if self.exists(dbc, dbName) : # if myself exists
fullTableName = dbName + '.' + self._stName
dbc.execute("DROP TABLE {}".format(fullTableName))
else:
if not skipCheck:
raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName))
def exists(self, dbc, dbName):
dbc.execute("USE " + dbName)
return dbc.existsSuperTable(self._stName)
# TODO: odd semantic, create() method is usually static?
def create(self, dbc, dbName, cols: dict, tags: dict):
def create(self, dbc, dbName, cols: dict, tags: dict,
dropIfExists = False
):
'''Creating a super table'''
sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format(
dbName,
self._stName,
",".join(['%s %s'%(k,v) for (k,v) in cols.items()]),
dbc.execute("USE " + dbName)
fullTableName = dbName + '.' + self._stName
if dbc.existsSuperTable(self._stName):
if dropIfExists:
dbc.execute("DROP TABLE {}".format(fullTableName))
else: # error
raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName))
# Now let's create
sql = "CREATE TABLE {} ({})".format(
fullTableName,
",".join(['%s %s'%(k,v) for (k,v) in cols.items()]))
if tags is None :
sql += " TAGS (dummy int) "
else:
sql += " TAGS ({})".format(
",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
)
dbc.execute(sql)
@ -1583,14 +1637,25 @@ class TdSuperTable:
def hasRegTables(self, dbc: DbConn, dbName: str):
return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0
def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str):
def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str):
sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName)
if dbc.query(sql) >= 1 : # reg table exists already
return
sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format(
dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
# acquire a lock first, so as to be able to *verify*. More details in TD-1471
fullTableName = dbName + '.' + regTableName
if task is not None: # optional lock
task.lockTable(fullTableName)
Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table
# print("(" + fullTableName[-3:] + ")", end="", flush=True)
try:
sql = "CREATE TABLE {} USING {}.{} tags ({})".format(
fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
)
dbc.execute(sql)
finally:
if task is not None:
task.unlockTable(fullTableName) # no matter what
def _getTagStrForSql(self, dbc, dbName: str) :
tags = self._getTags(dbc, dbName)
@ -1809,7 +1874,7 @@ class TaskRestartService(StateTransitionTask):
with self._classLock:
if self._isRunning:
print("Skipping restart task, another running already")
Logging.info("Skipping restart task, another running already")
return
self._isRunning = True
@ -1847,24 +1912,21 @@ class TaskAddData(StateTransitionTask):
def canBeginFrom(cls, state: AnyState):
return state.canAddData()
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
# ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
db = self._db
dbc = wt.getDbConn()
tblSeq = list(range(
self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))
random.shuffle(tblSeq)
for i in tblSeq:
if (i in self.activeTable): # wow already active
print("x", end="", flush=True) # concurrent insertion
else:
self.activeTable.add(i) # marking it active
def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor):
numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
fullTableName = db.getName() + '.' + regTableName
sTable = db.getFixedSuperTable()
regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i)
sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists
sql = "insert into {} values ".format(fullTableName)
for j in range(numRecords): # number of records per table
nextInt = db.getNextInt()
nextTick = db.getNextTick()
sql += "('{}', {});".format(nextTick, nextInt)
dbc.execute(sql)
for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table
def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches
numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
for j in range(numRecords): # number of records per table
nextInt = db.getNextInt()
nextTick = db.getNextTick()
if gConfig.record_ops:
@ -1872,27 +1934,29 @@ class TaskAddData(StateTransitionTask):
self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
self.fAddLogReady.flush()
os.fsync(self.fAddLogReady)
sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {})
db.getName(),
regTableName,
# TODO: too ugly trying to lock the table reliably, refactor...
fullTableName = db.getName() + '.' + regTableName
if gConfig.verify_data:
self.lockTable(fullTableName)
# print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written
try:
sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
fullTableName,
# ds.getFixedSuperTableName(),
# ds.getNextBinary(), ds.getNextFloat(),
nextTick, nextInt)
dbc.execute(sql)
# Successfully wrote the data into the DB, let's record it
# somehow
te.recordDataMark(nextInt)
if gConfig.record_ops:
self.fAddLogDone.write(
"Wrote {} to {}\n".format(
nextInt, regTableName))
self.fAddLogDone.flush()
os.fsync(self.fAddLogDone)
except: # Any exception at all
if gConfig.verify_data:
self.unlockTable(fullTableName)
raise
# Now read it back and verify, we might encounter an error if table is dropped
if gConfig.verify_data: # only if command line asks for it
try:
readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts= '{}'".
readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'".
format(db.getName(), regTableName, nextTick))
if readBack != nextInt :
raise taos.error.ProgrammingError(
@ -1905,16 +1969,52 @@ class TaskAddData(StateTransitionTask):
"Failed to read back same data for tick: {}, wrote: {}, read: {}"
.format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"),
errno)
# Re-throw no matter what
elif errno in [0x218, 0x362]: # table doesn't exist
# do nothing
dummy = 0
else:
# Re-throw otherwise
raise
finally:
self.unlockTable(fullTableName) # Unlock the table no matter what
# Successfully wrote the data into the DB, let's record it somehow
te.recordDataMark(nextInt)
if gConfig.record_ops:
self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName))
self.fAddLogDone.flush()
os.fsync(self.fAddLogDone)
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
# ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
db = self._db
dbc = wt.getDbConn()
numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES
numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
tblSeq = list(range(numTables ))
random.shuffle(tblSeq) # now we have random sequence
for i in tblSeq:
if (i in self.activeTable): # wow already active
print("x", end="", flush=True) # concurrent insertion
else:
self.activeTable.add(i) # marking it active
sTable = db.getFixedSuperTable()
regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i)
fullTableName = db.getName() + '.' + regTableName
# self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked"
sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists
# self._unlockTable(fullTableName)
if Dice.throw(1) == 0: # 1 in 2 chance
self._addData(db, dbc, regTableName, te)
else:
self._addDataInBatch(db, dbc, regTableName, te)
self.activeTable.discard(i) # not raising an error, unlike remove
class ThreadStacks: # stack info for all threads
def __init__(self):
self._allStacks = {}
@ -1936,17 +2036,18 @@ class ThreadStacks: # stack info for all threads
'__init__']: # the thread that extracted the stack
continue # ignore
# Now print
print("\n<----- Thread Info for ID: {}".format(thNid))
print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid))
stackFrame = 0
for frame in stack:
# print(frame)
print("File {filename}, line {lineno}, in {name}".format(
filename=frame.filename, lineno=frame.lineno, name=frame.name))
print("[{sf}] File {filename}, line {lineno}, in {name}".format(
sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
print(" {}".format(frame.line))
print("-----> End of Thread Info\n")
print("-----> End of Thread Info ----->\n")
class ClientManager:
def __init__(self):
print("Starting service manager")
Logging.info("Starting service manager")
# signal.signal(signal.SIGTERM, self.sigIntHandler)
# signal.signal(signal.SIGINT, self.sigIntHandler)
@ -2048,7 +2149,7 @@ class ClientManager:
thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps)
self.tc = ThreadCoordinator(thPool, dbManager)
print("Starting client instance to: {}".format(tInst))
Logging.info("Starting client instance: {}".format(tInst))
self.tc.run()
# print("exec stats: {}".format(self.tc.getExecStats()))
# print("TC failed = {}".format(self.tc.isFailed()))

6
tests/pytest/crash_gen/db.py
View File

@ -95,6 +95,11 @@ class DbConn:
# print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName)))
return dbName in dbs # TODO: super weird type mangling seen, once here
def existsSuperTable(self, stName):
self.query("show stables")
sts = [v[0] for v in self.getQueryResult()]
return stName in sts
def hasTables(self):
return self.query("show tables") > 0
@ -240,6 +245,7 @@ class MyTDSql:
def _execInternal(self, sql):
startTime = time.time()
# Logging.debug("Executing SQL: " + sql)
ret = self._cursor.execute(sql)
# print("\nSQL success: {}".format(sql))
queryTime = time.time() - startTime

9
tests/pytest/crash_gen/misc.py
View File

@ -27,7 +27,7 @@ class LoggingFilter(logging.Filter):
class MyLoggingAdapter(logging.LoggerAdapter):
def process(self, msg, kwargs):
return "[{}] {}".format(threading.get_ident() % 10000, msg), kwargs
return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs
# return '[%s] %s' % (self.extra['connid'], msg), kwargs
@ -51,7 +51,7 @@ class Logging:
_logger.addHandler(ch)
# Logging adapter, to be used as a logger
print("setting logger variable")
# print("setting logger variable")
# global logger
cls.logger = MyLoggingAdapter(_logger, [])
@ -166,6 +166,9 @@ class Progress:
SERVICE_RECONNECT_START = 4
SERVICE_RECONNECT_SUCCESS = 5
SERVICE_RECONNECT_FAILURE = 6
SERVICE_START_NAP = 7
CREATE_TABLE_ATTEMPT = 8
tokens = {
STEP_BOUNDARY: '.',
BEGIN_THREAD_STEP: '[',
@ -174,6 +177,8 @@ class Progress:
SERVICE_RECONNECT_START: '<r.',
SERVICE_RECONNECT_SUCCESS: '.r>',
SERVICE_RECONNECT_FAILURE: '.xr>',
SERVICE_START_NAP: '_zz',
CREATE_TABLE_ATTEMPT: '_c',
}
@classmethod

81
tests/pytest/crash_gen/service_manager.py
View File

@ -47,6 +47,17 @@ class TdeInstance():
.format(selfPath, projPath))
return buildPath
@classmethod
def prepareGcovEnv(cls, env):
# Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html
bPath = cls._getBuildPath() # build PATH
numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3
numSegments = numSegments - 1 # DEBUG only
env['GCOV_PREFIX'] = bPath + '/svc_gcov'
env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings
Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format(
numSegments, env['GCOV_PREFIX'] ))
def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030):
self._buildDir = self._getBuildPath()
self._subdir = '/' + subdir # TODO: tolerate "/"
@ -217,6 +228,11 @@ class TdeSubProcess:
# raise CrashGenError("Empty instance not allowed in TdeSubProcess")
# self._tInst = tInst # Default create at ServiceManagerThread
def __repr__(self):
if self.subProcess is None:
return '[TdeSubProc: Empty]'
return '[TdeSubProc: pid = {}]'.format(self.getPid())
def getStdOut(self):
return self.subProcess.stdout
@ -236,16 +252,29 @@ class TdeSubProcess:
if self.subProcess: # already there
raise RuntimeError("Corrupt process state")
# Prepare environment variables for coverage information
# Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment
myEnv = os.environ.copy()
TdeInstance.prepareGcovEnv(myEnv)
# print(myEnv)
# print(myEnv.items())
# print("Starting TDengine via Shell: {}".format(cmdLineStr))
useShell = True
self.subProcess = subprocess.Popen(
cmdLine,
shell=False,
' '.join(cmdLine) if useShell else cmdLine,
shell=useShell,
# svcCmdSingle, shell=True, # capture core dump?
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
# bufsize=1, # not supported in binary mode
close_fds=ON_POSIX
close_fds=ON_POSIX,
env=myEnv
) # had text=True, which interferred with reading EOF
STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process?
def stop(self):
"""
Stop a sub process, and try to return a meaningful return code.
@ -267,7 +296,7 @@ class TdeSubProcess:
SIGUSR2 12
"""
if not self.subProcess:
print("Sub process already stopped")
Logging.error("Sub process already stopped")
return # -1
retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N)
@ -278,20 +307,25 @@ class TdeSubProcess:
return retCode
# process still alive, let's interrupt it
print("Terminate running process, send SIG_INT and wait...")
Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL))
# sub process should end, then IPC queue should end, causing IO thread to end
# sig = signal.SIGINT
sig = signal.SIGKILL
self.subProcess.send_signal(sig) # SIGNINT or SIGKILL
topSubProc = psutil.Process(self.subProcess.pid)
for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False
child.send_signal(self.STOP_SIGNAL)
time.sleep(0.2) # 200 ms
# topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell)
self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell)
self.subProcess.wait(20)
retCode = self.subProcess.returncode # should always be there
# May throw subprocess.TimeoutExpired exception above, therefore
# The process is guranteed to have ended by now
self.subProcess = None
if retCode != 0: # != (- signal.SIGINT):
Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(sig, retCode))
Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(
self.STOP_SIGNAL, retCode))
else:
Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(sig))
Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL))
return - retCode
class ServiceManager:
@ -439,7 +473,7 @@ class ServiceManager:
time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round
# raise CrashGenError("dummy")
print("Service Manager Thread (with subprocess) ended, main thread exiting...")
Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...")
def _getFirstInstance(self):
return self._tInsts[0]
@ -452,7 +486,7 @@ class ServiceManager:
# Find if there's already a taosd service, and then kill it
for proc in psutil.process_iter():
if proc.name() == 'taosd':
print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt")
Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt")
time.sleep(2.0)
proc.kill()
# print("Process: {}".format(proc.name()))
@ -559,7 +593,8 @@ class ServiceManagerThread:
for i in range(0, 100):
time.sleep(1.0)
# self.procIpcBatch() # don't pump message during start up
print("_zz_", end="", flush=True)
Progress.emit(Progress.SERVICE_START_NAP)
# print("_zz_", end="", flush=True)
if self._status.isRunning():
Logging.info("[] TDengine service READY to process requests")
Logging.info("[] TAOS service started: {}".format(self))
@ -595,12 +630,12 @@ class ServiceManagerThread:
def stop(self):
# can be called from both main thread or signal handler
print("Terminating TDengine service running as the sub process...")
Logging.info("Terminating TDengine service running as the sub process...")
if self.getStatus().isStopped():
print("Service already stopped")
Logging.info("Service already stopped")
return
if self.getStatus().isStopping():
print("Service is already being stopped")
Logging.info("Service is already being stopped")
return
# Linux will send Control-C generated SIGINT to the TDengine process
# already, ref:
@ -616,10 +651,10 @@ class ServiceManagerThread:
if retCode == signal.SIGSEGV : # SGV
Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)")
except subprocess.TimeoutExpired as err:
print("Time out waiting for TDengine service process to exit")
Logging.info("Time out waiting for TDengine service process to exit")
else:
if self._tdeSubProcess.isRunning(): # still running, should now never happen
print("FAILED to stop sub process, it is still running... pid = {}".format(
Logging.error("FAILED to stop sub process, it is still running... pid = {}".format(
self._tdeSubProcess.getPid()))
else:
self._tdeSubProcess = None # not running any more
@ -683,9 +718,9 @@ class ServiceManagerThread:
return # we are done with THIS BATCH
else: # got line, printing out
if forceOutput:
Logging.info(line)
Logging.info('[TAOSD] ' + line)
else:
Logging.debug(line)
Logging.debug('[TAOSD] ' + line)
print(">", end="", flush=True)
_ProgressBars = ["--", "//", "||", "\\\\"]
@ -728,11 +763,11 @@ class ServiceManagerThread:
# queue.put(line)
# meaning sub process must have died
Logging.info("\nEnd of stream detected for TDengine STDOUT: {}".format(self))
Logging.info("EOF for TDengine STDOUT: {}".format(self))
out.close()
def svcErrorReader(self, err: IO, queue):
for line in iter(err.readline, b''):
print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line))
Logging.info("\nEnd of stream detected for TDengine STDERR: {}".format(self))
Logging.info("TDengine STDERR: {}".format(line))
Logging.info("EOF for TDengine STDERR: {}".format(self))
err.close()

2
tests/pytest/crash_gen_bootstrap.py
View File

@ -11,7 +11,7 @@
###################################################################
import sys
from crash_gen.crash_gen import MainExec
from crash_gen.crash_gen_main import MainExec
if __name__ == "__main__":

45
tests/pytest/query/queryLike.py Normal file
View File

@ -0,0 +1,45 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def run(self):
tdSql.prepare()
tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))")
tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');")
tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'")
tdSql.checkRows(0)
tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))")
tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');")
tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'")
tdSql.checkRows(1)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

170
tests/pytest/update/allow_update-0.py Normal file
View File

@ -0,0 +1,170 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
print("==============step1")
tdSql.execute("create database udb update 0")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

266
tests/pytest/update/allow_update.py Normal file
View File

@ -0,0 +1,266 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
print("==============step1")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 2)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 2)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 4)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 4)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 2)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 4)
tdSql.checkData(7, 0, 2)
tdSql.checkData(8, 0, 4)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
print("==============step7")
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
tdSql.checkData(0, 1, None)
tdSql.checkData(1, 1, None)
tdSql.checkData(2, 1, None)
tdSql.checkData(0, 2, 'c-3')
tdSql.checkData(1, 2, 'c+0')
tdSql.checkData(2, 2, 'c+3')
tdSql.checkData(0, 3, None)
tdSql.checkData(1, 3, None)
tdSql.checkData(2, 3, None)
print("==============step8")
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
print("==============step9")
insertRows = 20000
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
tdSql.query("select a,b from ct limit 3")
tdSql.checkData(0, 0, insertRows+0)
tdSql.checkData(1, 0, insertRows+1)
tdSql.checkData(2, 0, insertRows+2)
tdSql.checkData(0, 1, insertRows+0)
tdSql.checkData(1, 1, insertRows+1)
tdSql.checkData(2, 1, insertRows+2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

84
tests/pytest/update/append_commit_data-0.py Normal file
View File

@ -0,0 +1,84 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("create table && insert data")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1604298064000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0+i))
print("==========step2")
print("restart to commit ")
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows)
for k in range(0,100):
tdLog.info("insert %d rows" % (insertRows))
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into db.t1 values(%d,1)' %
(t0+k*200+i)
)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows+200*k)
print("==========step2")
print("insert into another table ")
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 20000
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d, 1)' %
(t0+i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

84
tests/pytest/update/append_commit_data.py Normal file
View File

@ -0,0 +1,84 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("create table && insert data")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db update 1'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1604298064000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0+i))
print("==========step2")
print("restart to commit ")
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows)
for k in range(0,100):
tdLog.info("insert %d rows" % (insertRows))
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into db.t1 values(%d,1)' %
(t0+k*200+i)
)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows+200*k)
print("==========step2")
print("insert into another table ")
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 20000
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d, 1)' %
(t0+i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

90
tests/pytest/update/append_commit_last-0.py Normal file
View File

@ -0,0 +1,90 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1604298064000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use db")
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(10):
tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(i + 1)
tdSql.query("select sum(a) from t1")
tdSql.checkData(0, 0, i + 1)
print("==============step2")
tdSql.execute("create table t2 (ts timestamp, a int)")
tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 1)
for i in range(1, 151):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(151)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 151)
print("==============step3")
tdSql.execute("create table t3 (ts timestamp, a int)")
tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 1)
for i in range(8):
for j in range(1, 11):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(81)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

85
tests/pytest/update/append_commit_last.py Normal file
View File

@ -0,0 +1,85 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1604298064000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(10):
tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(i + 1)
print("==============step2")
tdSql.execute("create table t2 (ts timestamp, a int)")
tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(1)
for i in range(1, 151):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(151)
print("==============step3")
tdSql.execute("create table t3 (ts timestamp, a int)")
tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(1)
for i in range(8):
for j in range(1, 11):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

351
tests/pytest/update/merge_commit_data-0.py Normal file
View File

@ -0,0 +1,351 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db days 30'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0 + i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
for k in range(0,10):
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into t1 values(%d,1)' %
(t0+i)
)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
print("==========step2")
print("PREPEND DATA ")
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
for i in range(-100,0):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
print("==========step3")
print("PREPEND MASSIVE DATA ")
ret = tdSql.execute('create table t3 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
for i in range(-6000,0):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
print("==========step4")
print("APPEND DATA")
ret = tdSql.execute('create table t4 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
print("==========step5")
print("APPEND DATA")
ret = tdSql.execute('create table t5 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
for i in range(0,6000):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
print("==========step6")
print("UPDATE BLOCK IN TWO STEP")
ret = tdSql.execute('create table t6 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t6 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
for i in range(0,200):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
print("==========step7")
print("UPDATE LAST HALF AND INSERT LITTLE DATA")
ret = tdSql.execute('create table t7 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t7 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
for i in range(100,300):
ret = tdSql.execute(
'insert into t7 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'400')
print("==========step8")
print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
ret = tdSql.execute('create table t8 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t8 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
for i in range(6000):
ret = tdSql.execute(
'insert into t8 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'11800')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'11800')
print("==========step9")
print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
ret = tdSql.execute('create table t9 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t9 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
for i in range(-100,100):
ret = tdSql.execute(
'insert into t9 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'400')
print("==========step10")
print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
ret = tdSql.execute('create table t10 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t10 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
for i in range(-6000,100):
ret = tdSql.execute(
'insert into t10 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12200')
print("==========step11")
print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
ret = tdSql.execute('create table t11 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t11 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
for i in range(100):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
for i in range(200,6000):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11800')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11800')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

351
tests/pytest/update/merge_commit_data.py Normal file
View File

@ -0,0 +1,351 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db update 1 days 30'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0 + i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
for k in range(0,10):
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into t1 values(%d,1)' %
(t0+i)
)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
print("==========step2")
print("PREPEND DATA ")
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
for i in range(-100,0):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
print("==========step3")
print("PREPEND MASSIVE DATA ")
ret = tdSql.execute('create table t3 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
for i in range(-6000,0):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
print("==========step4")
print("APPEND DATA")
ret = tdSql.execute('create table t4 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
print("==========step5")
print("APPEND DATA")
ret = tdSql.execute('create table t5 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
for i in range(0,6000):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
print("==========step6")
print("UPDATE BLOCK IN TWO STEP")
ret = tdSql.execute('create table t6 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t6 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'300')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'300')
for i in range(0,200):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'400')
print("==========step7")
print("UPDATE LAST HALF AND INSERT LITTLE DATA")
ret = tdSql.execute('create table t7 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t7 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
for i in range(100,300):
ret = tdSql.execute(
'insert into t7 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'500')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'500')
print("==========step8")
print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
ret = tdSql.execute('create table t8 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t8 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
for i in range(6000):
ret = tdSql.execute(
'insert into t8 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'12000')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'12000')
print("==========step9")
print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
ret = tdSql.execute('create table t9 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t9 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
for i in range(-100,100):
ret = tdSql.execute(
'insert into t9 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'500')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'500')
print("==========step10")
print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
ret = tdSql.execute('create table t10 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t10 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
for i in range(-6000,100):
ret = tdSql.execute(
'insert into t10 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12300')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12300')
print("==========step11")
print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
ret = tdSql.execute('create table t11 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t11 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
for i in range(100):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
for i in range(200,6000):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11900')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11900')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

352
tests/pytest/update/merge_commit_data2.py Normal file
View File

@ -0,0 +1,352 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import time
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def restart_taosd(self,db):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use %s;" % db)
def date_to_timestamp_microseconds(self, date):
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
return obj_stamp
def timestamp_microseconds_to_date(self, timestamp):
d = datetime.datetime.fromtimestamp(timestamp/1000)
str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
return str1
def run(self):
print("==========step1")
print("create table && insert data")
sql = 'reset query cache'
tdSql.execute(sql)
sql = 'drop database if exists db'
tdSql.execute(sql)
sql = 'create database db update 1 days 30;'
tdSql.execute(sql)
sql = 'use db;'
tdSql.execute(sql)
tdSql.execute('create table t1 (ts timestamp, a int)')
print("==================================1 start")
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
print("==========step2")
print("restart to commit ")
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
print("==========step3")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==========step4")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 2)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t1;')
# print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 2)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==================================2 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t2 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for k in range(10):
for i in range(10):
tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
print("==================================3 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t3 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(5200):
tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t3;')
for i in range(5200):
tdSql.checkData(i, 1, 2)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t3;')
for i in range(5200):
tdSql.checkData(i, 1, 2)
print("==================================4 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t4 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(100):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t4;')
for i in range(100):
tdSql.checkData(i, 1, 2)
for i in range(100, 200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t4;')
for i in range(100):
tdSql.checkData(i, 1, 2)
for i in range(100, 200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================5 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t5 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t5;')
for i in range(100):
tdSql.checkData(i, 1, 1)
for i in range(100, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5100):
tdSql.checkData(i, 1, 1)
for i in range(5100, 5200):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t5;')
for i in range(100):
tdSql.checkData(i, 1, 1)
for i in range(100, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5100):
tdSql.checkData(i, 1, 1)
for i in range(5100, 5200):
tdSql.checkData(i, 1, 2)
print("==================================6 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t6 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print("==================================7 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t7 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

384
tests/pytest/update/merge_commit_data2_update0.py Normal file
View File

@ -0,0 +1,384 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import time
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def restart_taosd(self,db):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use %s;" % db)
def date_to_timestamp_microseconds(self, date):
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
return obj_stamp
def timestamp_microseconds_to_date(self, timestamp):
d = datetime.datetime.fromtimestamp(timestamp/1000)
str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
return str1
def run(self):
print("==========step1")
print("create table && insert data")
sql = 'reset query cache'
tdSql.execute(sql)
sql = 'drop database if exists db'
tdSql.execute(sql)
sql = 'create database db update 0 days 30;'
tdSql.execute(sql)
sql = 'use db;'
tdSql.execute(sql)
tdSql.execute('create table t1 (ts timestamp, a int)')
print("==================================1 start")
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
print("==========step2")
print("restart to commit ")
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
print("==========step3")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==========step4")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t1;')
# print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==================================2 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t2 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for k in range(10):
for i in range(10):
tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
# print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
print("==================================3 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t3 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(5200):
tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t3;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t3;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================4 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t4 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(100):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t4;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t4;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
#
print("==================================5 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t5 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t5;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t5;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================6 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t6 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print("==================================7 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t7 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

309
tests/pytest/update/merge_commit_last-0.py Normal file
View File

@ -0,0 +1,309 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1603152000000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use db")
def run(self):
tdSql.prepare()
print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(5):
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 0)
print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
tdSql.execute("create table t2 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
for i in range(50):
tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
print("==============step 3: UPDATE PART OF THE LAST BLOCK")
tdSql.execute("create table t3 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
for i in range(25):
tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
tdSql.execute("create table t4 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 50)
for i in range(25):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
for i in range(50, 60):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t4")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 70)
print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
tdSql.execute("create table t5 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 50)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
for i in range(25):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
for i in range(25, 50):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
print("==============step 6: INSERT AHEAD A LOT OF DATA")
tdSql.execute("create table t6 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 0):
tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
tdSql.execute("create table t7 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 25):
tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2050)
print("==============step 8: INSERT AFTER A LOT AND UPDATE")
tdSql.execute("create table t8 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 50)
for i in range(25, 6000):
tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11950)
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11950)
print("==============step 9: UPDATE ONLY MIDDLE")
tdSql.execute("create table t9 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
for i in range(20, 30):
tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
tdSql.execute("create table t10 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 50)
for i in range(-4000, 4000):
tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 15950)
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 15950)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

321
tests/pytest/update/merge_commit_last.py Normal file
View File

@ -0,0 +1,321 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1603152000000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
tdSql.execute("create database udb update 1 days 30")
tdSql.execute("use udb")
print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(5):
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(1)
tdSql.checkData(0, 1, i)
print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
tdSql.execute("create table t2 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(50):
tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t2")
for i in range(50):
tdSql.checkData(i, 1, 2)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 2)
print("==============step 3: UPDATE PART OF THE LAST BLOCK")
tdSql.execute("create table t3 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(25):
tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t3")
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
tdSql.execute("create table t4 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(25):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
for i in range(50, 60):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t4")
tdSql.checkRows(60)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
for i in range(50, 60):
tdSql.checkData(i, 1, 2)
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(60)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
for i in range(50, 60):
tdSql.checkData(i, 1, 2)
print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
tdSql.execute("create table t5 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
for i in range(25):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 95)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 95)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
for i in range(25, 50):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 155)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 155)
print("==============step 6: INSERT AHEAD A LOT OF DATA")
tdSql.execute("create table t6 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 0):
tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
tdSql.execute("create table t7 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 25):
tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2075)
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2075)
print("==============step 8: INSERT AFTER A LOT AND UPDATE")
tdSql.execute("create table t8 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 50)
for i in range(25, 6000):
tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11975)
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11975)
print("==============step 9: UPDATE ONLY MIDDLE")
tdSql.execute("create table t9 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
for i in range(20, 30):
tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 60)
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 60)
print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
tdSql.execute("create table t10 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 50)
for i in range(-4000, 4000):
tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 16000)
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 16000)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

10
tests/pytest/updatetest.sh Normal file
View File

@ -0,0 +1,10 @@
# update
python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py
python3 ./test.py -f update/append_commit_data.py
python3 ./test.py -f update/append_commit_last-0.py
python3 ./test.py -f update/append_commit_last.py
python3 ./test.py -f update/merge_commit_data.py
python3 ./test.py -f update/merge_commit_data2.py
python3 ./test.py -f update/merge_commit_last-0.py
python3 ./test.py -f update/merge_commit_last.py

2
tests/script/general/http/restful_full.sim
View File

@ -81,7 +81,7 @@ print =============== step2 - no db
#11
system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql
print 11-> $system_content
if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then
return -1
endi