xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #4060 from taosdata/feature/update 

Feature/update
This commit is contained in:
Shengliang Guan 2020-11-06 11:30:33 +08:00 committed by GitHub
parent c3773983f9 11c4a2d0df
commit c0d6c718c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 5400 additions and 2355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/client/src/tscSQLParser.c
View File

@ -5311,6 +5311,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists;
pMsg->update = pCreateDb->update;
}
int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {

102
src/common/inc/tdataformat.h
View File

@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
// ----------------- Semantic timestamp key definition
typedef uint64_t TKEY;
#define TKEY_INVALID UINT64_MAX
#define TKEY_NULL TKEY_INVALID
#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
if (key1 < key2) {
return -1;
} else if (key1 > key2) {
return 1;
} else {
return 0;
}
}
// ----------------- Data row structure
/* A data row, the format is like below:
@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
*
* NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
typedef void *SDataRow;
@ -137,11 +166,13 @@ typedef void *SDataRow;
#define dataRowLen(r) (*(uint16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
#define dataRowKey(r) tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value);
break;
default:
if (IS_VAR_DATA_TYPE(type)) {
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value);
} else {
if (offset == 0) {
ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
} else {
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
break;
}
}
return 0;
@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
default:
return POINTER_SHIFT(row, offset);
if (IS_VAR_DATA_TYPE(type)) {
return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
} else {
return POINTER_SHIFT(row, offset);
}
}
@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle);
@ -204,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
break;
default:
return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
break;
if (IS_VAR_DATA_TYPE(pCol->type)) {
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
} else {
return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
}
}
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
ASSERT(rows > 0);
switch (pDataCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
break;
default:
return TYPE_BYTES[pDataCol->type] * rows;
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
} else {
return TYPE_BYTES[pDataCol->type] * rows;
}
}
@ -243,9 +265,14 @@ typedef struct {
} SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column
#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
#define dataColsTKeyLast(pCols) \
(((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
#define dataColsKeyLast(pCols) \
(((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols);
@ -253,10 +280,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!!
int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows);
// ----------------- K-V data row structure
/*

1
src/common/inc/tglobal.h
View File

@ -88,6 +88,7 @@ extern int16_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int32_t tsQuorum;
extern int32_t tsUpdate;
// balance
extern int32_t tsEnableBalance;

164
src/common/src/tdataformat.c
View File

@ -18,6 +18,9 @@
#include "tcoding.h"
#include "wchar.h"
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows);
/**
* Duplicate the schema and return a new object
*/
@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
pDataCol->len = 0;
if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
}
}
// value from timestamp should be TKEY here instead of TSKEY
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
// set offset
pCol->dataOff[numOfRows] = pCol->len;
// Copy data
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
// Update the length
pCol->len += varDataTLen(value);
break;
default:
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes;
break;
}
}
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
int pointsLeft = numOfRows - pointsToPop;
ASSERT(pointsLeft > 0);
if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
ASSERT(pCol->len > 0);
VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
pCol->len = pCol->len - toffset;
ASSERT(pCol->len > 0);
memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
dataColSetOffset(pCol, pointsLeft);
if (IS_VAR_DATA_TYPE(pCol->type)) {
// set offset
pCol->dataOff[numOfRows] = pCol->len;
// Copy data
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
// Update the length
pCol->len += varDataTLen(value);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes;
}
}
bool isNEleNull(SDataCol *pCol, int nEle) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
default:
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
}
void dataColSetNullAt(SDataCol *pCol, int index) {
@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
ASSERT(pDataCols->cols[i].dataOff != NULL);
pRet->cols[i].dataOff =
(int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].len = pDataCols->cols[i].len;
if (pDataCols->cols[i].len > 0) {
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
}
}
@ -420,58 +392,54 @@ void tdResetDataCols(SDataCols *pCols) {
}
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
int rcol = 0;
int dcol = 0;
while (dcol < pCols->numOfCols) {
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (rcol >= schemaNCols(pSchema)) {
dataColSetNullAt(pDataCol, pCols->numOfRows);
dcol++;
continue;
if (dataRowDeleted(row)) {
for (; dcol < pCols->numOfCols; dcol++) {
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (dcol == 0) {
dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
} else {
dataColSetNullAt(pDataCol, pCols->numOfRows);
}
}
} else {
while (dcol < pCols->numOfCols) {
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (rcol >= schemaNCols(pSchema)) {
dataColSetNullAt(pDataCol, pCols->numOfRows);
dcol++;
continue;
}
STColumn *pRowCol = schemaColAt(pSchema, rcol);
if (pRowCol->colId == pDataCol->colId) {
void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE);
dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
dcol++;
rcol++;
} else if (pRowCol->colId < pDataCol->colId) {
rcol++;
} else {
dataColSetNullAt(pDataCol, pCols->numOfRows);
dcol++;
STColumn *pRowCol = schemaColAt(pSchema, rcol);
if (pRowCol->colId == pDataCol->colId) {
void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
dcol++;
rcol++;
} else if (pRowCol->colId < pDataCol->colId) {
rcol++;
} else {
dataColSetNullAt(pDataCol, pCols->numOfRows);
dcol++;
}
}
}
pCols->numOfRows++;
}
// Pop pointsToPop points from the SDataCols
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
int pointsLeft = pCols->numOfRows - pointsToPop;
if (pointsLeft <= 0) {
tdResetDataCols(pCols);
return;
}
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SDataCol *pCol = pCols->cols + iCol;
dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
}
pCols->numOfRows = pointsLeft;
}
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
ASSERT(target->numOfCols == source->numOfCols);
SDataCols *pTarget = NULL;
if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0) {
@ -499,17 +467,23 @@ _err:
return -1;
}
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) {
// src2 data has more priority than src1
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows) {
tdResetDataCols(target);
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
while (target->numOfRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break;
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1];
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2];
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
if (key1 <= key2) {
ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
if (src1->cols[i].len > 0) {
@ -520,19 +494,23 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->numOfRows++;
(*iter1)++;
if (key1 == key2) (*iter2)++;
} else {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
target->maxPoints);
} else if (key1 >= key2) {
if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
target->maxPoints);
}
}
target->numOfRows++;
}
target->numOfRows++;
(*iter2)++;
if (key1 == key2) (*iter1)++;
}
ASSERT(target->numOfRows <= target->maxPoints);
}
}

11
src/common/src/tglobal.c
View File

@ -120,6 +120,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@ -786,6 +787,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "update";
cfg.ptr = &tsUpdate;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_UPDATE;
cfg.maxValue = TSDB_MAX_DB_UPDATE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mqttHostName";
cfg.ptr = tsMqttHostName;
cfg.valType = TAOS_CFG_VTYPE_STRING;

5
src/inc/taosdef.h
View File

@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_DATA_SMALLINT_NULL 0x8000
#define TSDB_DATA_INT_NULL 0x80000000L
#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L
#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN
@ -363,6 +364,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_DB_UPDATE 0
#define TSDB_MAX_DB_UPDATE 1
#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second

5
src/inc/taosmsg.h
View File

@ -544,6 +544,8 @@ typedef struct {
int8_t replications;
int8_t quorum;
int8_t ignoreExist;
int8_t update;
int8_t reserve[9];
} SCreateDbMsg, SAlterDbMsg;
typedef struct {
@ -654,7 +656,8 @@ typedef struct {
int8_t replications;
int8_t wals;
int8_t quorum;
int8_t reserved[16];
int8_t update;
int8_t reserved[15];
} SVnodeCfg;
typedef struct {

1
src/inc/tsdb.h
View File

@ -65,6 +65,7 @@ typedef struct {
int32_t maxRowsPerFileBlock; // maximum rows per file block
int8_t precision;
int8_t compression;
int8_t update;
} STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS

217
src/inc/ttokendef.h
View File

@ -114,114 +114,115 @@
#define TK_FSYNC 95
#define TK_COMP 96
#define TK_PRECISION 97
#define TK_LP 98
#define TK_RP 99
#define TK_TAGS 100
#define TK_USING 101
#define TK_AS 102
#define TK_COMMA 103
#define TK_NULL 104
#define TK_SELECT 105
#define TK_UNION 106
#define TK_ALL 107
#define TK_FROM 108
#define TK_VARIABLE 109
#define TK_INTERVAL 110
#define TK_FILL 111
#define TK_SLIDING 112
#define TK_ORDER 113
#define TK_BY 114
#define TK_ASC 115
#define TK_DESC 116
#define TK_GROUP 117
#define TK_HAVING 118
#define TK_LIMIT 119
#define TK_OFFSET 120
#define TK_SLIMIT 121
#define TK_SOFFSET 122
#define TK_WHERE 123
#define TK_NOW 124
#define TK_RESET 125
#define TK_QUERY 126
#define TK_ADD 127
#define TK_COLUMN 128
#define TK_TAG 129
#define TK_CHANGE 130
#define TK_SET 131
#define TK_KILL 132
#define TK_CONNECTION 133
#define TK_STREAM 134
#define TK_COLON 135
#define TK_ABORT 136
#define TK_AFTER 137
#define TK_ATTACH 138
#define TK_BEFORE 139
#define TK_BEGIN 140
#define TK_CASCADE 141
#define TK_CLUSTER 142
#define TK_CONFLICT 143
#define TK_COPY 144
#define TK_DEFERRED 145
#define TK_DELIMITERS 146
#define TK_DETACH 147
#define TK_EACH 148
#define TK_END 149
#define TK_EXPLAIN 150
#define TK_FAIL 151
#define TK_FOR 152
#define TK_IGNORE 153
#define TK_IMMEDIATE 154
#define TK_INITIALLY 155
#define TK_INSTEAD 156
#define TK_MATCH 157
#define TK_KEY 158
#define TK_OF 159
#define TK_RAISE 160
#define TK_REPLACE 161
#define TK_RESTRICT 162
#define TK_ROW 163
#define TK_STATEMENT 164
#define TK_TRIGGER 165
#define TK_VIEW 166
#define TK_COUNT 167
#define TK_SUM 168
#define TK_AVG 169
#define TK_MIN 170
#define TK_MAX 171
#define TK_FIRST 172
#define TK_LAST 173
#define TK_TOP 174
#define TK_BOTTOM 175
#define TK_STDDEV 176
#define TK_PERCENTILE 177
#define TK_APERCENTILE 178
#define TK_LEASTSQUARES 179
#define TK_HISTOGRAM 180
#define TK_DIFF 181
#define TK_SPREAD 182
#define TK_TWA 183
#define TK_INTERP 184
#define TK_LAST_ROW 185
#define TK_RATE 186
#define TK_IRATE 187
#define TK_SUM_RATE 188
#define TK_SUM_IRATE 189
#define TK_AVG_RATE 190
#define TK_AVG_IRATE 191
#define TK_TBID 192
#define TK_SEMI 193
#define TK_NONE 194
#define TK_PREV 195
#define TK_LINEAR 196
#define TK_IMPORT 197
#define TK_METRIC 198
#define TK_TBNAME 199
#define TK_JOIN 200
#define TK_METRICS 201
#define TK_STABLE 202
#define TK_INSERT 203
#define TK_INTO 204
#define TK_VALUES 205
#define TK_UPDATE 98
#define TK_LP 99
#define TK_RP 100
#define TK_TAGS 101
#define TK_USING 102
#define TK_AS 103
#define TK_COMMA 104
#define TK_NULL 105
#define TK_SELECT 106
#define TK_UNION 107
#define TK_ALL 108
#define TK_FROM 109
#define TK_VARIABLE 110
#define TK_INTERVAL 111
#define TK_FILL 112
#define TK_SLIDING 113
#define TK_ORDER 114
#define TK_BY 115
#define TK_ASC 116
#define TK_DESC 117
#define TK_GROUP 118
#define TK_HAVING 119
#define TK_LIMIT 120
#define TK_OFFSET 121
#define TK_SLIMIT 122
#define TK_SOFFSET 123
#define TK_WHERE 124
#define TK_NOW 125
#define TK_RESET 126
#define TK_QUERY 127
#define TK_ADD 128
#define TK_COLUMN 129
#define TK_TAG 130
#define TK_CHANGE 131
#define TK_SET 132
#define TK_KILL 133
#define TK_CONNECTION 134
#define TK_STREAM 135
#define TK_COLON 136
#define TK_ABORT 137
#define TK_AFTER 138
#define TK_ATTACH 139
#define TK_BEFORE 140
#define TK_BEGIN 141
#define TK_CASCADE 142
#define TK_CLUSTER 143
#define TK_CONFLICT 144
#define TK_COPY 145
#define TK_DEFERRED 146
#define TK_DELIMITERS 147
#define TK_DETACH 148
#define TK_EACH 149
#define TK_END 150
#define TK_EXPLAIN 151
#define TK_FAIL 152
#define TK_FOR 153
#define TK_IGNORE 154
#define TK_IMMEDIATE 155
#define TK_INITIALLY 156
#define TK_INSTEAD 157
#define TK_MATCH 158
#define TK_KEY 159
#define TK_OF 160
#define TK_RAISE 161
#define TK_REPLACE 162
#define TK_RESTRICT 163
#define TK_ROW 164
#define TK_STATEMENT 165
#define TK_TRIGGER 166
#define TK_VIEW 167
#define TK_COUNT 168
#define TK_SUM 169
#define TK_AVG 170
#define TK_MIN 171
#define TK_MAX 172
#define TK_FIRST 173
#define TK_LAST 174
#define TK_TOP 175
#define TK_BOTTOM 176
#define TK_STDDEV 177
#define TK_PERCENTILE 178
#define TK_APERCENTILE 179
#define TK_LEASTSQUARES 180
#define TK_HISTOGRAM 181
#define TK_DIFF 182
#define TK_SPREAD 183
#define TK_TWA 184
#define TK_INTERP 185
#define TK_LAST_ROW 186
#define TK_RATE 187
#define TK_IRATE 188
#define TK_SUM_RATE 189
#define TK_SUM_IRATE 190
#define TK_AVG_RATE 191
#define TK_AVG_IRATE 192
#define TK_TBID 193
#define TK_SEMI 194
#define TK_NONE 195
#define TK_PREV 196
#define TK_LINEAR 197
#define TK_IMPORT 198
#define TK_METRIC 199
#define TK_TBNAME 200
#define TK_JOIN 201
#define TK_METRICS 202
#define TK_STABLE 203
#define TK_INSERT 204
#define TK_INTO 205
#define TK_VALUES 206
#define TK_SPACE 300
#define TK_COMMENT 301

3
src/mnode/inc/mnodeDef.h
View File

@ -172,7 +172,8 @@ typedef struct {
int8_t walLevel;
int8_t replications;
int8_t quorum;
int8_t reserved[12];
int8_t update;
int8_t reserved[11];
} SDbCfg;
typedef struct SDbObj {

30
src/mnode/src/mnodeDb.c
View File

@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
}
#endif
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
return TSDB_CODE_MND_INVALID_DB_OPTION;
}
return TSDB_CODE_SUCCESS;
}
@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
if (pCfg->replications < 0) pCfg->replications = tsReplications;
if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
if (pCfg->update < 0) pCfg->update = tsUpdate;
}
static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@ -391,7 +397,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
.compression = pCreate->compression,
.walLevel = pCreate->walLevel,
.replications = pCreate->replications,
.quorum = pCreate->quorum
.quorum = pCreate->quorum,
.update = pCreate->update
};
mnodeSetDefaultDbCfg(&pDb->cfg);
@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "update");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.update;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
if (pDb->status == TSDB_DB_STATUS_READY) {
const char *src = "ready";
@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
int8_t replications = pAlter->replications;
int8_t quorum = pAlter->quorum;
int8_t precision = pAlter->precision;
int8_t update = pAlter->update;
terrno = TSDB_CODE_SUCCESS;
@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
newCfg.quorum = quorum;
}
if (update >= 0 && update != pDb->cfg.update) {
#if 0
mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update);
newCfg.update = update;
#else
mError("db:%s, can't alter update option", pDb->name);
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
#endif
}
return newCfg;
}

1
src/mnode/src/mnodeVgroup.c
View File

@ -850,6 +850,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
pCfg->replications = (int8_t) pVgroup->numOfVnodes;
pCfg->wals = 3;
pCfg->quorum = pDb->cfg.quorum;
pCfg->update = pDb->cfg.update;
SVnodeDesc *pNodes = pVnode->nodes;
for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {

1
src/query/inc/qSqlparser.h
View File

@ -129,6 +129,7 @@ typedef struct SCreateDBInfo {
int32_t compressionLevel;
SStrToken precision;
bool ignoreExists;
int8_t update;
tVariantList *keep;
} SCreateDBInfo;

3
src/query/inc/sql.y
View File

@ -239,6 +239,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; }
update(Y) ::= UPDATE INTEGER(X). { Y = X; }
%type db_optr {SCreateDBInfo}
db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type alter_db_optr {SCreateDBInfo}
alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
typename(A) ::= ids(X). {

26
src/query/src/qAst.c
View File

@ -414,9 +414,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
}
if (cond.start != NULL) {
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC);
} else {
iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC);
iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC);
}
if (cond.start != NULL) {
@ -431,7 +431,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
break;
}
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
} else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
@ -449,7 +449,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_GREATER) {
continue;
} else {
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false;
}
@ -464,22 +464,22 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue;
}
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
tSkipListDestroyIter(iter);
comp = true;
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC);
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC);
while(tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
if (comp) {
if (comp) {
continue;
}
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
@ -503,7 +503,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_LESS) {
continue;
} else {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false; // no need to compare anymore
}
@ -517,7 +517,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
(pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
}
@ -681,7 +681,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
if (filterItem(pExpr, pNode, param)) {
taosArrayPush(pResult, SL_GET_NODE_DATA(pNode));
taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode)));
}
}
tSkipListDestroyIter(iter);
@ -696,7 +696,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
SSkipListNode *pNode = tSkipListIterGet(iter);
char * pData = SL_GET_NODE_DATA(pNode);
tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
tstr *name = (tstr*) tsdbGetTableName((void*) pData);
// todo speed up by using hash
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
@ -710,7 +710,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
}
if (addToResult) {
STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(res, &info);
}
}

1
src/query/src/qParserImpl.c
View File

@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
pDBInfo->quorum = -1;
pDBInfo->keep = NULL;
pDBInfo->update = -1;
memset(&pDBInfo->precision, 0, sizeof(SStrToken));
}

3
src/query/src/qTokenizer.c
View File

@ -155,6 +155,7 @@ static SKeyword keywordTable[] = {
{"INSERT", TK_INSERT},
{"INTO", TK_INTO},
{"VALUES", TK_VALUES},
{"UPDATE", TK_UPDATE},
{"RESET", TK_RESET},
{"QUERY", TK_QUERY},
{"ADD", TK_ADD},
@ -664,4 +665,4 @@ void taosCleanupKeywordsTable() {
if (m != NULL && atomic_val_compare_exchange_ptr(&keywordHashTable, m, 0) == m) {
taosHashCleanup(m);
}
}
}

2330
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

24
src/tsdb/inc/tsdbMain.h
View File

@ -320,6 +320,15 @@ typedef struct {
void* compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper;
typedef struct {
int rowsInserted;
int rowsUpdated;
int rowsDeleteSucceed;
int rowsDeleteFailed;
int nOperations;
TSKEY keyFirst;
TSKEY keyLast;
} SMergeInfo;
// ------------------ tsdbScan.c
typedef struct {
SFileGroup fGroup;
@ -422,7 +431,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
// ------------------ tsdbMemTable.c
int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
@ -430,7 +439,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem)
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
TSKEY* filterKeys, int nFilterKeys);
TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL;
@ -438,16 +447,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
SSkipListNode* node = tSkipListIterGet(pIter);
if (node == NULL) return NULL;
return *(SDataRow *)SL_GET_NODE_DATA(node);
return (SDataRow)SL_GET_NODE_DATA(node);
}
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return -1;
if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
return dataRowKey(row);
}
static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return TKEY_NULL;
return dataRowTKey(row);
}
static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
ASSERT(pRepo != NULL);
if (pRepo->mem == NULL) return NULL;

7
src/tsdb/src/tsdbMain.c
View File

@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
}
}
// update check
if (pCfg->update != 0) pCfg->update = 1;
return 0;
_err:
@ -762,7 +765,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY
return -1;
}
if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1;
if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
(*affectedrows)++;
points++;
@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
tlen += taosEncodeFixedI8(buf, pCfg->precision);
tlen += taosEncodeFixedI8(buf, pCfg->compression);
tlen += taosEncodeFixedI8(buf, pCfg->update);
return tlen;
}
@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
buf = taosDecodeFixedI8(buf, &(pCfg->precision));
buf = taosDecodeFixedI8(buf, &(pCfg->compression));
buf = taosDecodeFixedI8(buf, &(pCfg->update));
return buf;
}

230
src/tsdb/src/tsdbMemTable.c
View File

@ -32,43 +32,40 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
// ---------------- INTERNAL FUNCTIONS ----------------
int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
STsdbCfg * pCfg = &pRepo->config;
STsdbMeta * pMeta = pRepo->tsdbMeta;
int32_t level = 0;
int32_t headSize = 0;
TKEY tkey = dataRowTKey(row);
TSKEY key = dataRowKey(row);
SMemTable * pMemTable = pRepo->mem;
STableData *pTableData = NULL;
SSkipList * pSList = NULL;
bool isRowDelete = TKEY_IS_DELETED(tkey);
if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL &&
pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) {
pTableData = pMemTable->tData[TABLE_TID(pTable)];
pSList = pTableData->pData;
}
if (isRowDelete) {
if (!pCfg->update) {
tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
tSkipListNewNodeInfo(pSList, &level, &headSize);
SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *));
if (pNode == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
if (key > TABLE_LASTKEY(pTable)) {
tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
return 0;
}
}
void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
if (pRow == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
free(pNode);
return -1;
}
pNode->level = level;
dataRowCpy(pRow, row);
*(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow;
// Operations above may change pRepo->mem, retake those values
ASSERT(pRepo->mem != NULL);
@ -77,7 +74,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
if (TABLE_TID(pTable) >= pMemTable->maxTables) {
if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
free(pNode);
return -1;
}
}
@ -97,7 +93,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
" to table %s while create new table data object since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
free(pNode);
return -1;
}
@ -106,24 +101,31 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
if (tSkipListPut(pTableData->pData, pNode) == NULL) {
int64_t oldSize = SL_SIZE(pTableData->pData);
if (tSkipListPut(pTableData->pData, pRow) == NULL) {
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
free(pNode);
} else {
if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
if (isRowDelete) {
if (TABLE_LASTKEY(pTable) == key) {
// TODO: need to update table last key here (may from file)
}
} else {
if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
}
if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
if (pMemTable->keyLast < key) pMemTable->keyLast = key;
pMemTable->numOfRows++;
pMemTable->numOfRows += deltaSize;
if (pTableData->keyFirst > key) pTableData->keyFirst = key;
if (pTableData->keyLast < key) pTableData->keyLast = key;
pTableData->numOfRows++;
ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
pTableData->numOfRows += deltaSize;
}
tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key);
tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
key);
return 0;
}
@ -295,63 +297,124 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
return 0;
}
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
* This function load memory data until:
* 1. iterator ends
* 2. data key exceeds maxKey
* 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
* 4. operations in pCols not exceeds its max capacity if pCols is given
*
* The function tries to procceed AS MUSH AS POSSIBLE.
*/
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
TSKEY *filterKeys, int nFilterKeys) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
if (pIter == NULL) return 0;
STSchema *pSchema = NULL;
int numOfRows = 0;
TSKEY keyNext = 0;
TSKEY rowKey = 0;
TSKEY fKey = 0;
bool isRowDel = false;
int filterIter = 0;
SDataRow row = NULL;
if (nFilterKeys != 0) { // for filter purpose
ASSERT(filterKeys != NULL);
keyNext = tsdbNextIterKey(pIter);
if (keyNext < 0 || keyNext > maxKey) return numOfRows;
void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)));
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
pMergeInfo->keyFirst = INT64_MAX;
pMergeInfo->keyLast = INT64_MIN;
if (pCols) tdResetDataCols(pCols);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
do {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) break;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
keyNext = dataRowKey(row);
if (keyNext > maxKey) break;
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
bool keyFiltered = false;
if (nFilterKeys != 0) {
while (true) {
if (filterIter >= nFilterKeys) break;
if (keyNext == filterKeys[filterIter]) {
keyFiltered = true;
filterIter++;
break;
} else if (keyNext < filterKeys[filterIter]) {
break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
} else if (fKey > rowKey) {
if (isRowDel) {
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
}
tSkipListIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
} else {
if (isRowDel) {
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
filterIter++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
}
}
}
if (!keyFiltered) {
if (numOfRows >= maxRowsToRead) break;
if (pCols) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (pSchema == NULL) {
ASSERT(0);
}
}
tdAppendDataRowToDataCol(row, pSchema, pCols);
tSkipListIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
numOfRows++;
}
} while (tSkipListIterNext(pIter));
return numOfRows;
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
}
}
return 0;
}
// ---------------- LOCAL FUNCTIONS ----------------
@ -440,8 +503,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
pTableData->keyLast = 0;
pTableData->numOfRows = 0;
pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP,
TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey);
pTableData->pData =
tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey);
if (pTableData->pData == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@ -461,7 +525,7 @@ static void tsdbFreeTableData(STableData *pTableData) {
}
}
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); }
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
static void *tsdbCommitData(void *arg) {
STsdbRepo * pRepo = (STsdbRepo *)arg;
@ -583,7 +647,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo) {
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1;
if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
}
return 0;
}
@ -779,5 +843,21 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
taosTFree(tData);
return 0;
}
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendDataRowToDataCol(row, *ppSchema, pCols);
}
return 0;
}

36
src/tsdb/src/tsdbMeta.c
View File

@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable));
return TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
goto _err;
}
if (pCfg->type == TSDB_CHILD_TABLE) {
@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) {
}
static char *getTagIndexKey(const void *pData) {
STable *pTable = *(STable **)pData;
STable *pTable = (STable *)pData;
STSchema *pSchema = tsdbGetTableTagSchema(pTable);
STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
}
pTable->tagVal = NULL;
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
pTable->pSuper = pSTable;
int32_t level = 0;
int32_t headSize = 0;
tSkipListPut(pSTable->pIndex, (void *)pTable);
tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
// NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
// actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
if (pNode == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
pNode->level = level;
memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
tSkipListPut(pSTable->pIndex, pNode);
if (refSuper) T_REF_INC(pSTable);
return 0;
}
@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
SSkipListNode *pNode = taosArrayGetP(res, i);
// STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
tSkipListRemoveNode(pSTable->pIndex, pNode);
}
}
@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
buf = tdDecodeSchema(buf, &(pTable->tagSchema));
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
pTable->pIndex =
tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL,
SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbFreeTable(pTable);
@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM);
} else {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1));
tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1));
} else {
tlen = sizeof(SListNode) + sizeof(SActObj);
}
@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) {
}
while (tSkipListIterNext(pIter)) {
STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable);
}
@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) {
tsdbWLockRepoMeta(pRepo);
while (tSkipListIterNext(pIter)) {
STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
tsdbRemoveTableFromMeta(pRepo, tTable, false, false);
}

394
src/tsdb/src/tsdbRWHelper.c
View File

@ -25,6 +25,7 @@
#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
#define TSDB_KEY_COL_OFFSET 0
#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
@ -32,7 +33,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD
static int compareKeyBlock(const void *arg1, const void *arg2);
static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded);
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
static int tsdbInitHelperFile(SRWHelper *pHelper);
@ -59,8 +60,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx);
static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows);
static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows, int8_t update);
static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
// ---------------------- INTERNAL FUNCTIONS ----------------------
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
@ -277,7 +280,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols
while (true) {
ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
if (keyFirst < 0 || keyFirst > maxKey) break; // iter over
if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over
if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
@ -923,7 +926,7 @@ _err:
return -1;
}
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) {
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
ASSERT(pCompBlock->numOfSubBlocks == 0);
SCompIdx *pIdx = &(pHelper->curCompIdx);
@ -956,9 +959,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
pSCompBlock->numOfSubBlocks++;
ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
pSCompBlock->len += sizeof(SCompBlock);
pSCompBlock->numOfRows += rowsAdded;
pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst);
pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast);
pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->keyFirst = pMergeInfo->keyFirst;
pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += sizeof(SCompBlock);
} else { // Need to create two sub-blocks
void *ptr = NULL;
@ -987,11 +990,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
((SCompBlock *)ptr)[1] = *pCompBlock;
pSCompBlock->numOfSubBlocks = 2;
pSCompBlock->numOfRows += rowsAdded;
pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
pSCompBlock->len = sizeof(SCompBlock) * 2;
pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst);
pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast);
pSCompBlock->keyFirst = pMergeInfo->keyFirst;
pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += (sizeof(SCompBlock) * 2);
}
@ -1045,6 +1048,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
return 0;
}
static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
SCompIdx *pCompIdx = &(pHelper->curCompIdx);
ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
SCompBlock compBlock = *pCompBlock;
ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
if (pCompIdx->numOfBlocks == 1) {
memset(pCompIdx, 0, sizeof(*pCompIdx));
} else {
int tsize = 0;
if (compBlock.numOfSubBlocks > 1) {
tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
ASSERT(tsize > 0);
memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
tsize);
pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
}
tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
ASSERT(tsize > 0);
memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
pCompIdx->len -= sizeof(SCompBlock);
pCompIdx->numOfBlocks--;
pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
}
return 0;
}
static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
pHelper->idxH.numOfIdx = 0;
pHelper->idxH.curIdx = 0;
@ -1437,51 +1479,62 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
}
static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) {
STsdbCfg * pCfg = &(pHelper->pRepo->config);
STable * pTable = pCommitIter->pTable;
SCompIdx * pIdx = &(pHelper->curCompIdx);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SCompBlock compBlock = {0};
STsdbCfg * pCfg = &(pHelper->pRepo->config);
STable * pTable = pCommitIter->pTable;
SCompIdx * pIdx = &(pHelper->curCompIdx);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SCompBlock compBlock = {0};
SMergeInfo mergeInfo = {0};
SMergeInfo *pMergeInfo = &mergeInfo;
ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
if (pIdx->hasLast) { // append to with last block
ASSERT(pIdx->len > 0);
SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows,
pDataCols, NULL, 0);
ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
NULL, 0, pCfg->update, pMergeInfo);
if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
if (pDataCols->numOfRows > 0) {
ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
}
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
}
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
} else {
ASSERT(!pHelper->hasOldLastBlock);
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0);
ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
if (pDataCols->numOfRows > 0) {
ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
}
}
#ifndef NDEBUG
TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
ASSERT(keyNext < 0 || keyNext > pIdx->maxKey);
ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
#endif
return 0;
@ -1489,13 +1542,16 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx) {
STsdbCfg * pCfg = &(pHelper->pRepo->config);
STable * pTable = pCommitIter->pTable;
SCompIdx * pIdx = &(pHelper->curCompIdx);
SCompBlock compBlock = {0};
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SDataCols *pDataCols0 = pHelper->pDataCols[0];
STsdbCfg * pCfg = &(pHelper->pRepo->config);
STable * pTable = pCommitIter->pTable;
SCompIdx * pIdx = &(pHelper->curCompIdx);
SCompBlock compBlock = {0};
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SDataCols * pDataCols0 = pHelper->pDataCols[0];
SMergeInfo mergeInfo = {0};
SMergeInfo *pMergeInfo = &mergeInfo;
SCompBlock oBlock = {0};
SSkipListIterator slIter = {0};
@ -1505,123 +1561,82 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
ASSERT(pCompBlock != NULL);
int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
oBlock = *pCompBlock;
if (pCompBlock->last) {
ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1);
int16_t colId = 0;
slIter = *(pCommitIter->pIter);
if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows);
ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
int rows1 = defaultRowsInBlock - pCompBlock->numOfRows;
int rows2 =
tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows);
if (rows2 == 0) { // all data filtered out
*(pCommitIter->pIter) = slIter;
} else {
if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols,
pDataCols0->cols[0].pData, pDataCols0->numOfRows);
ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
tblkIdx++;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
int round = 0;
int dIter = 0;
while (true) {
tdResetDataCols(pDataCols);
int rowsRead =
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock);
if (rowsRead == 0) break;
if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
while (true) {
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
pCfg->update, pMergeInfo);
ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (pDataCols->numOfRows == 0) break;
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (round == 0) {
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
tblkIdx++;
round++;
}
}
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
tblkIdx++;
}
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else {
TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1);
TSKEY blkKeyFirst = pCompBlock->keyFirst;
TSKEY blkKeyLast = pCompBlock->keyLast;
int16_t colId = 0;
if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
if (keyFirst < blkKeyFirst) {
while (true) {
tdResetDataCols(pDataCols);
int rowsRead =
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0);
if (rowsRead == 0) break;
TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
ASSERT(rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
tblkIdx++;
slIter = *(pCommitIter->pIter);
tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
pCfg->update, pMergeInfo);
if (pMergeInfo->nOperations == 0) {
// Do nothing
ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
*(pCommitIter->pIter) = slIter;
tblkIdx++;
} else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
// Delete the block and do some stuff
ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
*pCommitIter->pIter = slIter;
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
} else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
// Append as a sub-block of the searched block
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
pDataCols0->numOfRows, pCfg->update, pMergeInfo);
ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
&compBlock, oBlock.last, false) < 0) {
return -1;
}
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
return -1;
}
tblkIdx++;
} else {
ASSERT(keyFirst <= blkKeyLast);
int16_t colId = 0;
if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
// load the block data, merge with the memory data
if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
int round = 0;
int dIter = 0;
while (true) {
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
pCfg->update);
slIter = *(pCommitIter->pIter);
int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows);
int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData,
pDataCols0->numOfRows);
if (rows2 == 0) { // all filtered out
*(pCommitIter->pIter) = slIter;
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else {
int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2;
if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) {
int rows = (rows1 >= rows3) ? rows3 : rows2;
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols,
pDataCols0->cols[0].pData, pDataCols0->numOfRows);
ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0)
return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
tblkIdx++;
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
if (pDataCols->numOfRows == 0) break;
if (tblkIdx == pIdx->numOfBlocks - 1) {
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
int round = 0;
int dIter = 0;
while (true) {
int rowsRead =
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock);
if (rowsRead == 0) break;
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0)
return -1;
if (round == 0) {
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
round++;
tblkIdx++;
}
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
}
if (round == 0) {
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
round++;
tblkIdx++;
}
}
}
@ -1630,9 +1645,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
return 0;
}
static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows) {
int numOfRows = 0;
static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows, int8_t update) {
TSKEY key1 = INT64_MAX;
TSKEY key2 = INT64_MAX;
STSchema *pSchema = NULL;
@ -1642,35 +1656,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte
while (true) {
key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
bool isRowDel = false;
SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row);
if (row == NULL || dataRowKey(row) > maxKey) {
key2 = INT64_MAX;
} else {
key2 = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
if (key1 == INT64_MAX && key2 == INT64_MAX) break;
if (key1 <= key2) {
if (key1 < key2) {
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
pTarget->numOfRows++;
(*iter)++;
if (key1 == key2) tSkipListIterNext(pCommitIter->pIter);
} else {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL);
} else if (key1 > key2) {
if (!isRowDel) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL);
}
tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
tdAppendDataRowToDataCol(row, pSchema, pTarget);
tSkipListIterNext(pCommitIter->pIter);
} else {
if (update) {
if (!isRowDel) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL);
}
tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
} else {
ASSERT(!isRowDel);
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
pTarget->numOfRows++;
}
(*iter)++;
tSkipListIterNext(pCommitIter->pIter);
}
numOfRows++;
if (numOfRows >= maxRows) break;
ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints);
if (pTarget->numOfRows >= maxRows) break;
}
return numOfRows;
}
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
@ -1693,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols,
return 0;
}
static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
STsdbCfg *pCfg = &(pHelper->pRepo->config);
int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
ASSERT(mergeRows > 0);
if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
if (pCompBlock->last) {
if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
} else {
if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
}
}
return false;
}

67
src/tsdb/src/tsdbRead.c
View File

@ -457,7 +457,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
assert(node != NULL);
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@ -479,7 +479,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
assert(node != NULL);
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@ -504,19 +504,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iiter);
}
static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) {
static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
SDataRow rmem = NULL, rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
rmem = *(SDataRow *)SL_GET_NODE_DATA(node);
rmem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = *(SDataRow *)SL_GET_NODE_DATA(node);
rimem = (SDataRow)SL_GET_NODE_DATA(node);
}
}
@ -538,9 +538,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
TSKEY r2 = dataRowKey(rimem);
if (r1 == r2) { // data ts are duplicated, ignore the data in mem
tSkipListIterNext(pCheckInfo->iter);
pCheckInfo->chosen = 1;
return rimem;
if (!update) {
tSkipListIterNext(pCheckInfo->iter);
pCheckInfo->chosen = 1;
return rimem;
} else {
tSkipListIterNext(pCheckInfo->iiter);
pCheckInfo->chosen = 0;
return rmem;
}
} else {
if (ASCENDING_TRAVERSE(order)) {
if (r1 < r2) {
@ -594,6 +600,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
}
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
STsdbCfg *pCfg = &pHandle->pTsdb->config;
size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
pHandle->cur.fid = -1;
@ -607,7 +614,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
initTableMemIterator(pHandle, pCheckInfo);
}
SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
if (row == NULL) {
return false;
}
@ -827,11 +834,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur;
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
int32_t code = TSDB_CODE_SUCCESS;
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
assert(cur->pos >= 0 && cur->pos <= binfo.rows);
@ -1316,6 +1324,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
initTableMemIterator(pQueryHandle, pCheckInfo);
@ -1353,7 +1362,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL;
do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@ -1383,7 +1392,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
moveToNextRowInMem(pCheckInfo);
if (pCfg->update) {
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
}
cur->win.ekey = key;
cur->lastKey = key + step;
cur->mixBlock = true;
moveToNextRowInMem(pCheckInfo);
pos += step;
} else {
moveToNextRowInMem(pCheckInfo);
}
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) {
@ -1394,7 +1418,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
assert(end != -1);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
moveToNextRowInMem(pCheckInfo);
if (!pCfg->update) {
moveToNextRowInMem(pCheckInfo);
} else {
end -= step;
}
}
int32_t qstart = 0, qend = 0;
@ -1414,8 +1442,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
* copy them all to result buffer, since it may be overlapped with file data block.
*/
if (node == NULL ||
((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
// no data in cache or data in cache is greater than the ekey of time window, load data from file block
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos];
@ -1862,13 +1890,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj;
do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) {
break;
}
@ -1919,9 +1948,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode);
STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(list, &info);
}
@ -2464,7 +2493,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
static bool indexedNodeFilterFp(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*) param;
STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
char* val = NULL;

2
src/util/CMakeLists.txt
View File

@ -8,7 +8,7 @@ TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(tutil m rt)
ADD_SUBDIRECTORY(tests)
# ADD_SUBDIRECTORY(tests)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST)

185
src/util/inc/tskiplist.h
View File

@ -27,33 +27,25 @@ extern "C" {
#define MAX_SKIP_LIST_LEVEL 15
#define SKIP_LIST_RECORD_PERFORMANCE 0
// For key property setting
#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage)
#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case)
#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case)
// For thread safety setting
#define SL_THREAD_SAFE (uint8_t)0x4
typedef char *SSkipListKey;
typedef char *(*__sl_key_fn_t)(const void *);
/**
* the skip list node is located in a consecutive memory area,
* the format of skip list node is as follows:
* +------------+-----------------------+------------------------+-----+------+
* | node level | forward pointer array | backward pointer array | key | data |
* +------------+-----------------------+------------------------+-----+------+
*/
typedef struct SSkipListNode {
uint8_t level;
uint8_t level;
void * pData;
struct SSkipListNode *forwards[];
} SSkipListNode;
#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES)
#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)]
#define SL_GET_BACKWARD_POINTER(n, _l) \
((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
#define SL_GET_NODE_DATA(n) (n)->pData
#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)]
#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
/*
* @version 0.3
@ -103,34 +95,23 @@ typedef struct tSkipListState {
uint64_t nTotalElapsedTimeForInsert;
} tSkipListState;
typedef struct SSkipListKeyInfo {
uint8_t dupKey : 2; // if allow duplicated key in the skip list
uint8_t type : 4; // key type
uint8_t freeNode:2; // free node when destroy the skiplist
uint8_t len; // maximum key length, used in case of string key
} SSkipListKeyInfo;
typedef struct SSkipList {
__compar_fn_t comparFn;
__sl_key_fn_t keyFn;
uint32_t size;
uint8_t maxLevel;
uint8_t level;
SSkipListKeyInfo keyInfo;
pthread_rwlock_t *lock;
SSkipListNode * pHead; // point to the first element
SSkipListNode * pTail; // point to the last element
uint16_t len;
uint8_t maxLevel;
uint8_t flags;
uint8_t type; // static info above
uint8_t level;
uint32_t size;
SSkipListNode * pHead; // point to the first element
SSkipListNode * pTail; // point to the last element
#if SKIP_LIST_RECORD_PERFORMANCE
tSkipListState state; // skiplist state
#endif
} SSkipList;
/*
* iterate the skiplist
* this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
* continue iterating the skiplist, so add the reference count for skiplist
* TODO add the ref for skip list when one iterator is created
*/
typedef struct SSkipListIterator {
SSkipList * pSkipList;
SSkipListNode *cur;
@ -139,114 +120,26 @@ typedef struct SSkipListIterator {
SSkipListNode *next; // next points to the true qualified node in skip list
} SSkipListIterator;
/**
*
* @param nMaxLevel maximum skip list level
* @param keyType type of key
* @param dupKey allow the duplicated key in the skip list
* @return
*/
SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
uint8_t freeNode, __sl_key_fn_t fn);
#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE)
#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1))
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
#define SL_SIZE(s) (s)->size
/**
*
* @param pSkipList
* @return NULL will always be returned
*/
void *tSkipListDestroy(SSkipList *pSkipList);
/**
*
* @param pSkipList
* @param level
* @param headSize
*/
void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
/**
* put the skip list node into the skip list.
* If failed, NULL will be returned, otherwise, the pNode will be returned.
*
* @param pSkipList
* @param pNode
* @return
*/
SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
/**
* get *all* nodes which key are equivalent to pKey
*
* @param pSkipList
* @param pKey
* @return
*/
SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
/**
* get the size of skip list
* @param pSkipList
* @return
*/
size_t tSkipListGetSize(const SSkipList *pSkipList);
/**
* display skip list of the given level, for debug purpose only
* @param pSkipList
* @param nlevel
*/
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
/**
* create skiplist iterator
* @param pSkipList
* @return
*/
SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
__sl_key_fn_t fn);
void tSkipListDestroy(SSkipList *pSkipList);
SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData);
SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
/**
* create skip list iterator from the given node and specified the order
* @param pSkipList
* @param pNode start position, instead of the first node in skip list
* @param order traverse order of the iterator
* @return
*/
SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order);
/**
* forward the skip list iterator
* @param iter
* @return
*/
bool tSkipListIterNext(SSkipListIterator *iter);
/**
* get the element of skip list node
* @param iter
* @return
*/
SSkipListNode *tSkipListIterGet(SSkipListIterator *iter);
/**
* destroy the skip list node
* @param iter
* @return
*/
void *tSkipListDestroyIter(SSkipListIterator *iter);
/*
* remove nodes of the pKey value.
* If more than one node has the same value, all will be removed
*
* @Return
* the count of removed nodes
*/
uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
/*
* remove the specified node in parameters
*/
void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
bool tSkipListIterNext(SSkipListIterator *iter);
SSkipListNode * tSkipListIterGet(SSkipListIterator *iter);
void * tSkipListDestroyIter(SSkipListIterator *iter);
uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
#ifdef __cplusplus
}

970
src/util/src/tskiplist.c
View File

File diff suppressed because it is too large Load Diff

4
src/util/tests/skiplistTest.cpp
View File

@ -247,7 +247,7 @@ void skiplistPerformanceTest() {
printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size);
printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
assert(tSkipListGetSize(pSkipList) == size);
assert(SL_GET_SIZE(pSkipList) == size);
// printf("the level of skiplist is:\n");
//
@ -273,7 +273,7 @@ void skiplistPerformanceTest() {
int64_t et = taosGetTimestampMs();
printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
assert(tSkipListGetSize(pSkipList) == size);
assert(SL_GET_SIZE(pSkipList) == size);
tSkipListDestroy(pSkipList);
taosTFree(total);

1
src/vnode/src/vnodeMain.c
View File

@ -140,6 +140,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression;
tsdbCfg.update = pVnodeCfg->cfg.update;
char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);

71
tests/pytest/alter/db_update_options.py Normal file
View File

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
import random
import string
import subprocess
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdLog.debug("check database")
tdSql.prepare()
# check default update value
sql = "create database if not exists db"
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
sql = "alter database db update 1"
# check update value
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,1)
sql = "alter database db update 0"
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
sql = "alter database db update -1"
tdSql.error(sql)
sql = "alter database db update 100"
tdSql.error(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
tdSql.execute('drop database db')
tdSql.error('create database db update 100')
tdSql.error('create database db update -1')
tdSql.execute('create database db update 1')
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,1)
tdSql.execute('drop database db')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

170
tests/pytest/update/allow_update-0.py Normal file
View File

@ -0,0 +1,170 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
print("==============step1")
tdSql.execute("create database udb update 0")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

266
tests/pytest/update/allow_update.py Normal file
View File

@ -0,0 +1,266 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
print("==============step1")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 2)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 2)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 4)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 4)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 2)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 4)
tdSql.checkData(7, 0, 2)
tdSql.checkData(8, 0, 4)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
print("==============step7")
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
tdSql.checkData(0, 1, None)
tdSql.checkData(1, 1, None)
tdSql.checkData(2, 1, None)
tdSql.checkData(0, 2, 'c-3')
tdSql.checkData(1, 2, 'c+0')
tdSql.checkData(2, 2, 'c+3')
tdSql.checkData(0, 3, None)
tdSql.checkData(1, 3, None)
tdSql.checkData(2, 3, None)
print("==============step8")
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
print("==============step9")
insertRows = 20000
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
tdSql.query("select a,b from ct limit 3")
tdSql.checkData(0, 0, insertRows+0)
tdSql.checkData(1, 0, insertRows+1)
tdSql.checkData(2, 0, insertRows+2)
tdSql.checkData(0, 1, insertRows+0)
tdSql.checkData(1, 1, insertRows+1)
tdSql.checkData(2, 1, insertRows+2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

84
tests/pytest/update/append_commit_data-0.py Normal file
View File

@ -0,0 +1,84 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("create table && insert data")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1604298064000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0+i))
print("==========step2")
print("restart to commit ")
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows)
for k in range(0,100):
tdLog.info("insert %d rows" % (insertRows))
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into db.t1 values(%d,1)' %
(t0+k*200+i)
)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows+200*k)
print("==========step2")
print("insert into another table ")
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 20000
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d, 1)' %
(t0+i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

84
tests/pytest/update/append_commit_data.py Normal file
View File

@ -0,0 +1,84 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("create table && insert data")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db update 1'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1604298064000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0+i))
print("==========step2")
print("restart to commit ")
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows)
for k in range(0,100):
tdLog.info("insert %d rows" % (insertRows))
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into db.t1 values(%d,1)' %
(t0+k*200+i)
)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows+200*k)
print("==========step2")
print("insert into another table ")
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 20000
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d, 1)' %
(t0+i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

90
tests/pytest/update/append_commit_last-0.py Normal file
View File

@ -0,0 +1,90 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1604298064000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use db")
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(10):
tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(i + 1)
tdSql.query("select sum(a) from t1")
tdSql.checkData(0, 0, i + 1)
print("==============step2")
tdSql.execute("create table t2 (ts timestamp, a int)")
tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 1)
for i in range(1, 151):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(151)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 151)
print("==============step3")
tdSql.execute("create table t3 (ts timestamp, a int)")
tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 1)
for i in range(8):
for j in range(1, 11):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(81)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

85
tests/pytest/update/append_commit_last.py Normal file
View File

@ -0,0 +1,85 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1604298064000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(10):
tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(i + 1)
print("==============step2")
tdSql.execute("create table t2 (ts timestamp, a int)")
tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(1)
for i in range(1, 151):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(151)
print("==============step3")
tdSql.execute("create table t3 (ts timestamp, a int)")
tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(1)
for i in range(8):
for j in range(1, 11):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

351
tests/pytest/update/merge_commit_data-0.py Normal file
View File

@ -0,0 +1,351 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db days 30'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0 + i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
for k in range(0,10):
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into t1 values(%d,1)' %
(t0+i)
)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
print("==========step2")
print("PREPEND DATA ")
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
for i in range(-100,0):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
print("==========step3")
print("PREPEND MASSIVE DATA ")
ret = tdSql.execute('create table t3 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
for i in range(-6000,0):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
print("==========step4")
print("APPEND DATA")
ret = tdSql.execute('create table t4 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
print("==========step5")
print("APPEND DATA")
ret = tdSql.execute('create table t5 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
for i in range(0,6000):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
print("==========step6")
print("UPDATE BLOCK IN TWO STEP")
ret = tdSql.execute('create table t6 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t6 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
for i in range(0,200):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
print("==========step7")
print("UPDATE LAST HALF AND INSERT LITTLE DATA")
ret = tdSql.execute('create table t7 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t7 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
for i in range(100,300):
ret = tdSql.execute(
'insert into t7 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'400')
print("==========step8")
print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
ret = tdSql.execute('create table t8 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t8 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
for i in range(6000):
ret = tdSql.execute(
'insert into t8 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'11800')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'11800')
print("==========step9")
print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
ret = tdSql.execute('create table t9 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t9 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
for i in range(-100,100):
ret = tdSql.execute(
'insert into t9 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'400')
print("==========step10")
print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
ret = tdSql.execute('create table t10 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t10 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
for i in range(-6000,100):
ret = tdSql.execute(
'insert into t10 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12200')
print("==========step11")
print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
ret = tdSql.execute('create table t11 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t11 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
for i in range(100):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
for i in range(200,6000):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11800')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11800')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

351
tests/pytest/update/merge_commit_data.py Normal file
View File

@ -0,0 +1,351 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db update 1 days 30'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0 + i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
for k in range(0,10):
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into t1 values(%d,1)' %
(t0+i)
)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
print("==========step2")
print("PREPEND DATA ")
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
for i in range(-100,0):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
print("==========step3")
print("PREPEND MASSIVE DATA ")
ret = tdSql.execute('create table t3 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
for i in range(-6000,0):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
print("==========step4")
print("APPEND DATA")
ret = tdSql.execute('create table t4 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
print("==========step5")
print("APPEND DATA")
ret = tdSql.execute('create table t5 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
for i in range(0,6000):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
print("==========step6")
print("UPDATE BLOCK IN TWO STEP")
ret = tdSql.execute('create table t6 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t6 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'300')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'300')
for i in range(0,200):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'400')
print("==========step7")
print("UPDATE LAST HALF AND INSERT LITTLE DATA")
ret = tdSql.execute('create table t7 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t7 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
for i in range(100,300):
ret = tdSql.execute(
'insert into t7 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'500')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'500')
print("==========step8")
print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
ret = tdSql.execute('create table t8 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t8 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
for i in range(6000):
ret = tdSql.execute(
'insert into t8 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'12000')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'12000')
print("==========step9")
print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
ret = tdSql.execute('create table t9 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t9 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
for i in range(-100,100):
ret = tdSql.execute(
'insert into t9 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'500')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'500')
print("==========step10")
print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
ret = tdSql.execute('create table t10 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t10 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
for i in range(-6000,100):
ret = tdSql.execute(
'insert into t10 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12300')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12300')
print("==========step11")
print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
ret = tdSql.execute('create table t11 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t11 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
for i in range(100):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
for i in range(200,6000):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11900')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11900')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

352
tests/pytest/update/merge_commit_data2.py Normal file
View File

@ -0,0 +1,352 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import time
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def restart_taosd(self,db):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use %s;" % db)
def date_to_timestamp_microseconds(self, date):
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
return obj_stamp
def timestamp_microseconds_to_date(self, timestamp):
d = datetime.datetime.fromtimestamp(timestamp/1000)
str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
return str1
def run(self):
print("==========step1")
print("create table && insert data")
sql = 'reset query cache'
tdSql.execute(sql)
sql = 'drop database if exists db'
tdSql.execute(sql)
sql = 'create database db update 1 days 30;'
tdSql.execute(sql)
sql = 'use db;'
tdSql.execute(sql)
tdSql.execute('create table t1 (ts timestamp, a int)')
print("==================================1 start")
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
print("==========step2")
print("restart to commit ")
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
print("==========step3")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==========step4")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 2)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t1;')
# print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 2)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==================================2 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t2 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for k in range(10):
for i in range(10):
tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
print("==================================3 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t3 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(5200):
tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t3;')
for i in range(5200):
tdSql.checkData(i, 1, 2)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t3;')
for i in range(5200):
tdSql.checkData(i, 1, 2)
print("==================================4 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t4 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(100):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t4;')
for i in range(100):
tdSql.checkData(i, 1, 2)
for i in range(100, 200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t4;')
for i in range(100):
tdSql.checkData(i, 1, 2)
for i in range(100, 200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================5 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t5 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t5;')
for i in range(100):
tdSql.checkData(i, 1, 1)
for i in range(100, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5100):
tdSql.checkData(i, 1, 1)
for i in range(5100, 5200):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t5;')
for i in range(100):
tdSql.checkData(i, 1, 1)
for i in range(100, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5100):
tdSql.checkData(i, 1, 1)
for i in range(5100, 5200):
tdSql.checkData(i, 1, 2)
print("==================================6 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t6 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print("==================================7 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t7 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

384
tests/pytest/update/merge_commit_data2_update0.py Normal file
View File

@ -0,0 +1,384 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import time
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def restart_taosd(self,db):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use %s;" % db)
def date_to_timestamp_microseconds(self, date):
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
return obj_stamp
def timestamp_microseconds_to_date(self, timestamp):
d = datetime.datetime.fromtimestamp(timestamp/1000)
str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
return str1
def run(self):
print("==========step1")
print("create table && insert data")
sql = 'reset query cache'
tdSql.execute(sql)
sql = 'drop database if exists db'
tdSql.execute(sql)
sql = 'create database db update 0 days 30;'
tdSql.execute(sql)
sql = 'use db;'
tdSql.execute(sql)
tdSql.execute('create table t1 (ts timestamp, a int)')
print("==================================1 start")
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
print("==========step2")
print("restart to commit ")
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
print("==========step3")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==========step4")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t1;')
# print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==================================2 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t2 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for k in range(10):
for i in range(10):
tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
# print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
print("==================================3 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t3 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(5200):
tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t3;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t3;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================4 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t4 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(100):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t4;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t4;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
#
print("==================================5 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t5 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t5;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t5;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================6 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t6 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print("==================================7 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t7 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

309
tests/pytest/update/merge_commit_last-0.py Normal file
View File

@ -0,0 +1,309 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1603152000000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use db")
def run(self):
tdSql.prepare()
print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(5):
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 0)
print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
tdSql.execute("create table t2 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
for i in range(50):
tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
print("==============step 3: UPDATE PART OF THE LAST BLOCK")
tdSql.execute("create table t3 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
for i in range(25):
tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
tdSql.execute("create table t4 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 50)
for i in range(25):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
for i in range(50, 60):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t4")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 70)
print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
tdSql.execute("create table t5 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 50)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
for i in range(25):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
for i in range(25, 50):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
print("==============step 6: INSERT AHEAD A LOT OF DATA")
tdSql.execute("create table t6 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 0):
tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
tdSql.execute("create table t7 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 25):
tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2050)
print("==============step 8: INSERT AFTER A LOT AND UPDATE")
tdSql.execute("create table t8 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 50)
for i in range(25, 6000):
tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11950)
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11950)
print("==============step 9: UPDATE ONLY MIDDLE")
tdSql.execute("create table t9 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
for i in range(20, 30):
tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
tdSql.execute("create table t10 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 50)
for i in range(-4000, 4000):
tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 15950)
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 15950)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

321
tests/pytest/update/merge_commit_last.py Normal file
View File

@ -0,0 +1,321 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1603152000000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
tdSql.execute("create database udb update 1 days 30")
tdSql.execute("use udb")
print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(5):
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(1)
tdSql.checkData(0, 1, i)
print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
tdSql.execute("create table t2 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(50):
tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t2")
for i in range(50):
tdSql.checkData(i, 1, 2)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 2)
print("==============step 3: UPDATE PART OF THE LAST BLOCK")
tdSql.execute("create table t3 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(25):
tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t3")
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
tdSql.execute("create table t4 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(25):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
for i in range(50, 60):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t4")
tdSql.checkRows(60)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
for i in range(50, 60):
tdSql.checkData(i, 1, 2)
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(60)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
for i in range(50, 60):
tdSql.checkData(i, 1, 2)
print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
tdSql.execute("create table t5 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
for i in range(25):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 95)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 95)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
for i in range(25, 50):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 155)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 155)
print("==============step 6: INSERT AHEAD A LOT OF DATA")
tdSql.execute("create table t6 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 0):
tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
tdSql.execute("create table t7 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 25):
tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2075)
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2075)
print("==============step 8: INSERT AFTER A LOT AND UPDATE")
tdSql.execute("create table t8 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 50)
for i in range(25, 6000):
tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11975)
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11975)
print("==============step 9: UPDATE ONLY MIDDLE")
tdSql.execute("create table t9 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
for i in range(20, 30):
tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 60)
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 60)
print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
tdSql.execute("create table t10 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 50)
for i in range(-4000, 4000):
tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 16000)
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 16000)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

10
tests/pytest/updatetest.sh Normal file
View File

@ -0,0 +1,10 @@
# update
python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py
python3 ./test.py -f update/append_commit_data.py
python3 ./test.py -f update/append_commit_last-0.py
python3 ./test.py -f update/append_commit_last.py
python3 ./test.py -f update/merge_commit_data.py
python3 ./test.py -f update/merge_commit_data2.py
python3 ./test.py -f update/merge_commit_last-0.py
python3 ./test.py -f update/merge_commit_last.py

2
tests/script/general/http/restful_full.sim
View File

@ -81,7 +81,7 @@ print =============== step2 - no db
#11
system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql
print 11-> $system_content
if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then
if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then
return -1
endi